diff --git a/blockstore/autobatch.go b/blockstore/autobatch.go new file mode 100644 index 00000000000..53764d15cae --- /dev/null +++ b/blockstore/autobatch.go @@ -0,0 +1,242 @@ +package blockstore + +import ( + "context" + "sync" + "time" + + "golang.org/x/xerrors" + + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" +) + +// autolog is a logger for the autobatching blockstore. It is subscoped from the +// blockstore logger. +var autolog = log.Named("auto") + +// contains the same set of blocks twice, once as an ordered list for flushing, and as a map for fast access +type blockBatch struct { + blockList []block.Block + blockMap map[cid.Cid]block.Block +} + +type AutobatchBlockstore struct { + // TODO: drop if memory consumption is too high + addedCids map[cid.Cid]struct{} + + stateLock sync.Mutex + doFlushLock sync.Mutex + bufferedBatch blockBatch + + flushingBatch blockBatch + flushErr error + flushWorkerDone bool + + flushCh chan struct{} + + flushRetryDelay time.Duration + flushCtx context.Context + shutdownCh chan struct{} + + backingBs Blockstore + + bufferCapacity int + bufferSize int +} + +func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore { + bs := &AutobatchBlockstore{ + addedCids: make(map[cid.Cid]struct{}), + backingBs: backingBs, + bufferCapacity: bufferCapacity, + flushCtx: ctx, + flushCh: make(chan struct{}, 1), + shutdownCh: make(chan struct{}), + // could be made configable + flushRetryDelay: time.Millisecond * 100, + flushWorkerDone: false, + } + + bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block) + + go bs.flushWorker() + + return bs +} + +func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error { + bs.stateLock.Lock() + defer bs.stateLock.Unlock() + + _, ok := bs.addedCids[blk.Cid()] + if !ok { + bs.addedCids[blk.Cid()] = struct{}{} + bs.bufferedBatch.blockList = append(bs.bufferedBatch.blockList, blk) + bs.bufferedBatch.blockMap[blk.Cid()] = blk + bs.bufferSize += len(blk.RawData()) + if bs.bufferSize >= bs.bufferCapacity { + // signal that a flush is appropriate, may be ignored + select { + case bs.flushCh <- struct{}{}: + default: + // do nothing + } + } + } + + return nil +} + +func (bs *AutobatchBlockstore) flushWorker() { + defer func() { + bs.stateLock.Lock() + bs.flushWorkerDone = true + bs.stateLock.Unlock() + }() + for { + select { + case <-bs.flushCh: + putErr := bs.doFlush(bs.flushCtx) + for putErr != nil { + select { + case <-bs.shutdownCh: + return + case <-time.After(bs.flushRetryDelay): + autolog.Errorf("FLUSH ERRORED: %w, retrying after %v", putErr, bs.flushRetryDelay) + putErr = bs.doFlush(bs.flushCtx) + } + } + case <-bs.shutdownCh: + return + } + } +} + +// caller must NOT hold stateLock +func (bs *AutobatchBlockstore) doFlush(ctx context.Context) error { + bs.doFlushLock.Lock() + defer bs.doFlushLock.Unlock() + if bs.flushErr == nil { + bs.stateLock.Lock() + // We do NOT clear addedCids here, because its purpose is to expedite Puts + bs.flushingBatch = bs.bufferedBatch + bs.bufferedBatch.blockList = make([]block.Block, 0, len(bs.flushingBatch.blockList)) + bs.bufferedBatch.blockMap = make(map[cid.Cid]block.Block, len(bs.flushingBatch.blockMap)) + bs.stateLock.Unlock() + } + + bs.flushErr = bs.backingBs.PutMany(ctx, bs.flushingBatch.blockList) + bs.stateLock.Lock() + bs.flushingBatch = blockBatch{} + bs.stateLock.Unlock() + + return bs.flushErr +} + +// caller must NOT hold stateLock +func (bs *AutobatchBlockstore) Flush(ctx context.Context) error { + return bs.doFlush(ctx) +} + +func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error { + bs.stateLock.Lock() + flushDone := bs.flushWorkerDone + bs.stateLock.Unlock() + if !flushDone { + // may racily block forever if Shutdown is called in parallel + bs.shutdownCh <- struct{}{} + } + + return bs.flushErr +} + +func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error) { + // may seem backward to check the backingBs first, but that is the likeliest case + blk, err := bs.backingBs.Get(ctx, c) + if err == nil { + return blk, nil + } + + if err != ErrNotFound { + return blk, err + } + + v, ok := bs.flushingBatch.blockMap[c] + if ok { + return v, nil + } + + v, ok = bs.bufferedBatch.blockMap[c] + if ok { + return v, nil + } + + return bs.Get(ctx, c) +} + +func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error { + // if we wanted to support this, we would have to: + // - flush + // - delete from the backingBs (if present) + // - remove from addedCids (if present) + // - if present in addedCids, also walk the ordered lists and remove if present + return xerrors.New("deletion is unsupported") +} + +func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error { + // see note in DeleteBlock() + return xerrors.New("deletion is unsupported") +} + +func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + _, err := bs.Get(ctx, c) + if err == nil { + return true, nil + } + if err == ErrNotFound { + return false, nil + } + + return false, err +} + +func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + blk, err := bs.Get(ctx, c) + if err != nil { + return 0, err + } + + return len(blk.RawData()), nil +} + +func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error { + for _, blk := range blks { + if err := bs.Put(ctx, blk); err != nil { + return err + } + } + + return nil +} + +func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + if err := bs.Flush(ctx); err != nil { + return nil, err + } + + return bs.backingBs.AllKeysChan(ctx) +} + +func (bs *AutobatchBlockstore) HashOnRead(enabled bool) { + bs.backingBs.HashOnRead(enabled) +} + +func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error { + blk, err := bs.Get(ctx, cid) + if err != nil { + return err + } + + return callback(blk.RawData()) +} diff --git a/blockstore/autobatch_test.go b/blockstore/autobatch_test.go new file mode 100644 index 00000000000..57a3b7d6cae --- /dev/null +++ b/blockstore/autobatch_test.go @@ -0,0 +1,34 @@ +package blockstore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAutobatchBlockstore(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ab := NewAutobatch(ctx, NewMemory(), len(b0.RawData())+len(b1.RawData())-1) + + require.NoError(t, ab.Put(ctx, b0)) + require.NoError(t, ab.Put(ctx, b1)) + require.NoError(t, ab.Put(ctx, b2)) + + v0, err := ab.Get(ctx, b0.Cid()) + require.NoError(t, err) + require.Equal(t, b0.RawData(), v0.RawData()) + + v1, err := ab.Get(ctx, b1.Cid()) + require.NoError(t, err) + require.Equal(t, b1.RawData(), v1.RawData()) + + v2, err := ab.Get(ctx, b2.Cid()) + require.NoError(t, err) + require.Equal(t, b2.RawData(), v2.RawData()) + + require.NoError(t, ab.Flush(ctx)) + require.NoError(t, ab.Shutdown(ctx)) +} diff --git a/chain/actors/builtin/paych/message.go.template b/chain/actors/builtin/paych/message.go.template index 4a5ea2331e5..99f64cabb91 100644 --- a/chain/actors/builtin/paych/message.go.template +++ b/chain/actors/builtin/paych/message.go.template @@ -39,7 +39,11 @@ func (m message{{.v}}) Create(to address.Address, initialAmount abi.TokenAmount) func (m message{{.v}}) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych{{.v}}.UpdateChannelStateParams{ + {{if (ge .v 7)}} + Sv: toV{{.v}}SignedVoucher(*sv), + {{else}} Sv: *sv, + {{end}} Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message0.go b/chain/actors/builtin/paych/message0.go index bfeb2731e2b..7cba977e357 100644 --- a/chain/actors/builtin/paych/message0.go +++ b/chain/actors/builtin/paych/message0.go @@ -39,7 +39,9 @@ func (m message0) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message0) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych0.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message2.go b/chain/actors/builtin/paych/message2.go index 2cf3ef22eef..60c7fe16eb1 100644 --- a/chain/actors/builtin/paych/message2.go +++ b/chain/actors/builtin/paych/message2.go @@ -39,7 +39,9 @@ func (m message2) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message2) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych2.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message3.go b/chain/actors/builtin/paych/message3.go index 50503a1409a..04fb35b5787 100644 --- a/chain/actors/builtin/paych/message3.go +++ b/chain/actors/builtin/paych/message3.go @@ -39,7 +39,9 @@ func (m message3) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message3) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych3.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message4.go b/chain/actors/builtin/paych/message4.go index b2c6b612e38..9f5e000d9f7 100644 --- a/chain/actors/builtin/paych/message4.go +++ b/chain/actors/builtin/paych/message4.go @@ -39,7 +39,9 @@ func (m message4) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message4) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych4.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message5.go b/chain/actors/builtin/paych/message5.go index 37a2b6f04af..71e6b679992 100644 --- a/chain/actors/builtin/paych/message5.go +++ b/chain/actors/builtin/paych/message5.go @@ -39,7 +39,9 @@ func (m message5) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message5) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych5.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message6.go b/chain/actors/builtin/paych/message6.go index aecf26983d4..7f80bc4a61c 100644 --- a/chain/actors/builtin/paych/message6.go +++ b/chain/actors/builtin/paych/message6.go @@ -39,7 +39,9 @@ func (m message6) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message6) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych6.UpdateChannelStateParams{ - Sv: *sv, + + Sv: *sv, + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/message7.go b/chain/actors/builtin/paych/message7.go index 41dfa1bdd18..e3ee0d77b52 100644 --- a/chain/actors/builtin/paych/message7.go +++ b/chain/actors/builtin/paych/message7.go @@ -39,7 +39,9 @@ func (m message7) Create(to address.Address, initialAmount abi.TokenAmount) (*ty func (m message7) Update(paych address.Address, sv *SignedVoucher, secret []byte) (*types.Message, error) { params, aerr := actors.SerializeParams(&paych7.UpdateChannelStateParams{ - Sv: *sv, + + Sv: toV7SignedVoucher(*sv), + Secret: secret, }) if aerr != nil { diff --git a/chain/actors/builtin/paych/state.go.template b/chain/actors/builtin/paych/state.go.template index 3e41f5be5f6..f114072020e 100644 --- a/chain/actors/builtin/paych/state.go.template +++ b/chain/actors/builtin/paych/state.go.template @@ -112,3 +112,21 @@ func (ls *laneState{{.v}}) Redeemed() (big.Int, error) { func (ls *laneState{{.v}}) Nonce() (uint64, error) { return ls.LaneState.Nonce, nil } + +{{if (ge .v 7)}} +func toV{{.v}}SignedVoucher(sv SignedVoucher) paych{{.v}}.SignedVoucher { + return paych{{.v}}.SignedVoucher{ + ChannelAddr: sv.ChannelAddr, + TimeLockMin: sv.TimeLockMin, + TimeLockMax: sv.TimeLockMax, + SecretHash: sv.SecretPreimage, + Extra: sv.Extra, + Lane: sv.Lane, + Nonce: sv.Nonce, + Amount: sv.Amount, + MinSettleHeight: sv.MinSettleHeight, + Merges: sv.Merges, + Signature: sv.Signature, + } +} +{{end}} \ No newline at end of file diff --git a/chain/actors/builtin/paych/v7.go b/chain/actors/builtin/paych/v7.go index ce09ea2e4be..19c801c8228 100644 --- a/chain/actors/builtin/paych/v7.go +++ b/chain/actors/builtin/paych/v7.go @@ -112,3 +112,19 @@ func (ls *laneState7) Redeemed() (big.Int, error) { func (ls *laneState7) Nonce() (uint64, error) { return ls.LaneState.Nonce, nil } + +func toV7SignedVoucher(sv SignedVoucher) paych7.SignedVoucher { + return paych7.SignedVoucher{ + ChannelAddr: sv.ChannelAddr, + TimeLockMin: sv.TimeLockMin, + TimeLockMax: sv.TimeLockMax, + SecretHash: sv.SecretPreimage, + Extra: sv.Extra, + Lane: sv.Lane, + Nonce: sv.Nonce, + Amount: sv.Amount, + MinSettleHeight: sv.MinSettleHeight, + Merges: sv.Merges, + Signature: sv.Signature, + } +} diff --git a/chain/consensus/filcns/upgrades.go b/chain/consensus/filcns/upgrades.go index 863193180cc..a21bdb05e23 100644 --- a/chain/consensus/filcns/upgrades.go +++ b/chain/consensus/filcns/upgrades.go @@ -5,6 +5,8 @@ import ( "runtime" "time" + "github.com/docker/go-units" + "github.com/filecoin-project/specs-actors/v6/actors/migration/nv14" "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" @@ -1245,8 +1247,15 @@ func PreUpgradeActorsV7(ctx context.Context, sm *stmgr.StateManager, cache stmgr workerCount /= 2 } - config := nv15.Config{MaxWorkers: uint(workerCount)} - _, err := upgradeActorsV7Common(ctx, sm, cache, root, epoch, ts, config) + lbts, lbRoot, err := stmgr.GetLookbackTipSetForRound(ctx, sm, ts, epoch) + if err != nil { + return xerrors.Errorf("error getting lookback ts for premigration: %w", err) + } + + config := nv15.Config{MaxWorkers: uint(workerCount), + ProgressLogPeriod: time.Minute * 5} + + _, err = upgradeActorsV7Common(ctx, sm, cache, lbRoot, epoch, lbts, config) return err } @@ -1255,9 +1264,10 @@ func upgradeActorsV7Common( root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet, config nv15.Config, ) (cid.Cid, error) { - buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), blockstore.NewMemorySync()) - store := store.ActorStore(ctx, buf) - + writeStore := blockstore.NewAutobatch(ctx, sm.ChainStore().StateBlockstore(), units.GiB) + // TODO: pretty sure we'd achieve nothing by doing this, confirm in review + //buf := blockstore.NewTieredBstore(sm.ChainStore().StateBlockstore(), writeStore) + store := store.ActorStore(ctx, writeStore) // Load the state root. var stateRoot types.StateRoot if err := store.Get(ctx, root, &stateRoot); err != nil { @@ -1287,15 +1297,13 @@ func upgradeActorsV7Common( return cid.Undef, xerrors.Errorf("failed to persist new state root: %w", err) } - // Persist the new tree. - - { - from := buf - to := buf.Read() + // Persists the new tree and shuts down the flush worker + if err := writeStore.Flush(ctx); err != nil { + return cid.Undef, xerrors.Errorf("writeStore flush failed: %w", err) + } - if err := vm.Copy(ctx, from, to, newRoot); err != nil { - return cid.Undef, xerrors.Errorf("copying migrated tree: %w", err) - } + if err := writeStore.Shutdown(ctx); err != nil { + return cid.Undef, xerrors.Errorf("writeStore shutdown failed: %w", err) } return newRoot, nil diff --git a/chain/stmgr/forks.go b/chain/stmgr/forks.go index 454f781c4ff..a83ffdf7aaf 100644 --- a/chain/stmgr/forks.go +++ b/chain/stmgr/forks.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -15,8 +17,6 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/network" - "github.com/filecoin-project/specs-actors/v3/actors/migration/nv10" - "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin" init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init" @@ -211,7 +211,7 @@ func (sm *StateManager) hasExpensiveFork(height abi.ChainEpoch) bool { return ok } -func runPreMigration(ctx context.Context, sm *StateManager, fn PreMigrationFunc, cache *nv10.MemMigrationCache, ts *types.TipSet) { +func runPreMigration(ctx context.Context, sm *StateManager, fn PreMigrationFunc, cache *nv15.MemMigrationCache, ts *types.TipSet) { height := ts.Height() parent := ts.ParentState() diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index fd3558a1cf4..45dd52ec88b 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -4,6 +4,8 @@ import ( "context" "sync" + "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" + "github.com/filecoin-project/lotus/chain/rand" "github.com/filecoin-project/lotus/chain/beacon" @@ -18,10 +20,6 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/go-state-types/network" - // Used for genesis. - msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" - "github.com/filecoin-project/specs-actors/v3/actors/migration/nv10" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" @@ -30,6 +28,9 @@ import ( "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + + // Used for genesis. + msig0 "github.com/filecoin-project/specs-actors/actors/builtin/multisig" ) const LookbackNoLimit = api.LookbackNoLimit @@ -53,7 +54,7 @@ type versionSpec struct { type migration struct { upgrade MigrationFunc preMigrations []PreMigration - cache *nv10.MemMigrationCache + cache *nv15.MemMigrationCache } type Executor interface { @@ -121,7 +122,7 @@ func NewStateManager(cs *store.ChainStore, exec Executor, sys vm.SyscallBuilder, migration := &migration{ upgrade: upgrade.Migration, preMigrations: upgrade.PreMigrations, - cache: nv10.NewMemMigrationCache(), + cache: nv15.NewMemMigrationCache(), } stateMigrations[upgrade.Height] = migration } diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index e150d8b41bb..9bcea72240e 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -67,6 +67,7 @@ func main() { balancerCmd, sendCsvCmd, terminationsCmd, + migrationsCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go new file mode 100644 index 00000000000..85987c65867 --- /dev/null +++ b/cmd/lotus-shed/migrations.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/specs-actors/v7/actors/migration/nv15" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lotus/chain/consensus/filcns" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" +) + +var migrationsCmd = &cli.Command{ + Name: "migrate-nv15", + Description: "Run the specified migration", + ArgsUsage: "[block to look back from]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + if cctx.NArg() != 1 { + return fmt.Errorf("must pass block cid") + } + + blkCid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return fmt.Errorf("failed to parse input: %w", err) + } + + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore) + if err != nil { + return fmt.Errorf("failed to open blockstore: %w", err) + } + + defer func() { + if c, ok := bs.(io.Closer); ok { + if err := c.Close(); err != nil { + log.Warnf("failed to close blockstore: %s", err) + } + } + }() + + mds, err := lkrepo.Datastore(context.Background(), "/metadata") + if err != nil { + return err + } + + cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil) + defer cs.Close() //nolint:errcheck + + sm, err := stmgr.NewStateManager(cs, filcns.NewTipSetExecutor(), vm.Syscalls(ffiwrapper.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil) + if err != nil { + return err + } + + cache := nv15.NewMemMigrationCache() + + blk, err := cs.GetBlock(ctx, blkCid) + if err != nil { + return err + } + + migrationTs, err := cs.LoadTipSet(ctx, types.NewTipSetKey(blk.Parents...)) + if err != nil { + return err + } + + ts1, err := cs.GetTipsetByHeight(ctx, blk.Height-240, migrationTs, false) + if err != nil { + return err + } + + startTime := time.Now() + + err = filcns.PreUpgradeActorsV7(ctx, sm, cache, ts1.ParentState(), ts1.Height()-1, ts1) + if err != nil { + return err + } + + fmt.Println("completed round 1, took ", time.Since(startTime)) + startTime = time.Now() + + newCid1, err := filcns.UpgradeActorsV7(ctx, sm, cache, nil, blk.ParentStateRoot, blk.Height-1, migrationTs) + if err != nil { + return err + } + fmt.Println("completed round actual (with cache), took ", time.Since(startTime)) + + fmt.Println("new cid", newCid1) + + newCid2, err := filcns.UpgradeActorsV7(ctx, sm, nv15.NewMemMigrationCache(), nil, blk.ParentStateRoot, blk.Height-1, migrationTs) + if err != nil { + return err + } + fmt.Println("completed round actual (without cache), took ", time.Since(startTime)) + + fmt.Println("new cid", newCid2) + return nil + }, +} diff --git a/go.mod b/go.mod index 97c8134f42a..5e401a3e67e 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.3-0.20220111000201-e42866db1a53 - github.com/filecoin-project/go-state-types v0.1.1 + github.com/filecoin-project/go-state-types v0.1.3 github.com/filecoin-project/go-statemachine v1.0.1 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -50,7 +50,7 @@ require ( github.com/filecoin-project/specs-actors/v4 v4.0.1 github.com/filecoin-project/specs-actors/v5 v5.0.4 github.com/filecoin-project/specs-actors/v6 v6.0.1 - github.com/filecoin-project/specs-actors/v7 v7.0.0-20211230214648-aeae366b083a + github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1 github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.1 diff --git a/go.sum b/go.sum index 36302c0d7bd..bf84d7b43e3 100644 --- a/go.sum +++ b/go.sum @@ -300,6 +300,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoC github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgqMbJsMBLM0fIuLUUk= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk= @@ -346,8 +348,9 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.1.1 h1:LR260vya4p++atgf256W6yV3Lxl5mKrBFcEZePWQrdg= github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0= +github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.1 h1:LQ60+JDVjMdLxXmVFM2jjontzOYnfVE7u02CXV3WKSw= github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -375,8 +378,8 @@ github.com/filecoin-project/specs-actors/v6 v6.0.1 h1:laxvHNsvrq83Y9n+W7znVCePi3 github.com/filecoin-project/specs-actors/v6 v6.0.1/go.mod h1:V1AYfi5GkHXipx1mnVivoICZh3wtwPxDVuds+fbfQtk= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211117170924-fd07a4c7dff9/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211222192039-c83bea50c402/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= -github.com/filecoin-project/specs-actors/v7 v7.0.0-20211230214648-aeae366b083a h1:MS1mtAhZh0iSE7OxP1bb6+UNyYKsxg8n51FpHlX1d54= -github.com/filecoin-project/specs-actors/v7 v7.0.0-20211230214648-aeae366b083a/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= +github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1 h1:FuDaXIbcw2hRsFI8SDTmsGGCE+NumpF6aiBoU/2X5W4= +github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M= github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9 h1:oUYOvF7EvdXS0Zmk9mNkaB6Bu0l+WXBYPzVodKMiLug= github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= diff --git a/lotuspond/front/src/chain/methods.json b/lotuspond/front/src/chain/methods.json index 1f6191a9425..15c04ca2834 100644 --- a/lotuspond/front/src/chain/methods.json +++ b/lotuspond/front/src/chain/methods.json @@ -712,7 +712,7 @@ "CreateMiner", "UpdateClaimedPower", "EnrollCronEvent", - "OnEpochTickEnd", + "CronTick", "UpdatePledgeTotal", "SubmitPoRepForBulkVerify", "CurrentTotalPower" @@ -728,6 +728,7 @@ "RemoveVerifier", "AddVerifiedClient", "UseBytes", - "RestoreBytes" + "RestoreBytes", + "RemoveVerifiedClientDataCap" ] } \ No newline at end of file diff --git a/testplans/lotus-soup/go.mod b/testplans/lotus-soup/go.mod index 308ee5140c7..6b4be1d9716 100644 --- a/testplans/lotus-soup/go.mod +++ b/testplans/lotus-soup/go.mod @@ -11,7 +11,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.12.1 github.com/filecoin-project/go-fil-markets v1.14.1 github.com/filecoin-project/go-jsonrpc v0.1.5 - github.com/filecoin-project/go-state-types v0.1.1 + github.com/filecoin-project/go-state-types v0.1.3 github.com/filecoin-project/go-storedcounter v0.1.0 github.com/filecoin-project/lotus v0.0.0-00010101000000-000000000000 github.com/filecoin-project/specs-actors v0.9.14 diff --git a/testplans/lotus-soup/go.sum b/testplans/lotus-soup/go.sum index 47e65fd549c..e6e4149c8d2 100644 --- a/testplans/lotus-soup/go.sum +++ b/testplans/lotus-soup/go.sum @@ -391,6 +391,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoC github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 h1:ZNJ9tEG5bE72vBWYiuh5bkxJVM3ViHNOmQ7qew9n6RE= github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgqMbJsMBLM0fIuLUUk= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk= @@ -437,8 +439,9 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.1.1 h1:LR260vya4p++atgf256W6yV3Lxl5mKrBFcEZePWQrdg= github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0= +github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.1 h1:LQ60+JDVjMdLxXmVFM2jjontzOYnfVE7u02CXV3WKSw= github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -466,8 +469,8 @@ github.com/filecoin-project/specs-actors/v6 v6.0.1 h1:laxvHNsvrq83Y9n+W7znVCePi3 github.com/filecoin-project/specs-actors/v6 v6.0.1/go.mod h1:V1AYfi5GkHXipx1mnVivoICZh3wtwPxDVuds+fbfQtk= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211117170924-fd07a4c7dff9/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211222192039-c83bea50c402/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= -github.com/filecoin-project/specs-actors/v7 v7.0.0-20211230214648-aeae366b083a h1:MS1mtAhZh0iSE7OxP1bb6+UNyYKsxg8n51FpHlX1d54= -github.com/filecoin-project/specs-actors/v7 v7.0.0-20211230214648-aeae366b083a/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= +github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1 h1:FuDaXIbcw2hRsFI8SDTmsGGCE+NumpF6aiBoU/2X5W4= +github.com/filecoin-project/specs-actors/v7 v7.0.0-rc1/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M= github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9 h1:oUYOvF7EvdXS0Zmk9mNkaB6Bu0l+WXBYPzVodKMiLug= github.com/filecoin-project/specs-storage v0.1.1-0.20211228030229-6d460d25a0c9/go.mod h1:Tb88Zq+IBJbvAn3mS89GYj3jdRThBTE/771HCVZdRJU= github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=