From 970a53f827ded21b27525f6b0042bbc124c62d48 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 12 Feb 2023 03:06:55 +0000 Subject: [PATCH 01/20] feat(cosmic-swingset): add after-commit action --- golang/cosmos/app/app.go | 10 ++++++- golang/cosmos/x/swingset/abci.go | 20 ++++++++++++- packages/cosmic-swingset/src/launch-chain.js | 31 +++++++++++++++----- packages/internal/src/action-types.js | 1 + 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/golang/cosmos/app/app.go b/golang/cosmos/app/app.go index 7ea23afd232..0f79ae21fa8 100644 --- a/golang/cosmos/app/app.go +++ b/golang/cosmos/app/app.go @@ -852,7 +852,15 @@ func (app *GaiaApp) Commit() abci.ResponseCommit { if err != nil { panic(err.Error()) } - return app.BaseApp.Commit() + + res := app.BaseApp.Commit() + + err = swingset.AfterCommitBlock(app.SwingSetKeeper) + if err != nil { + panic(err.Error()) + } + + return res } // LoadHeight loads a particular height diff --git a/golang/cosmos/x/swingset/abci.go b/golang/cosmos/x/swingset/abci.go index c7359fcb04a..96f00a3011f 100644 --- a/golang/cosmos/x/swingset/abci.go +++ b/golang/cosmos/x/swingset/abci.go @@ -1,8 +1,8 @@ package swingset import ( - // "fmt" // "os" + "fmt" "time" "github.com/cosmos/cosmos-sdk/telemetry" @@ -101,3 +101,21 @@ func CommitBlock(keeper Keeper) error { } return err } + +func AfterCommitBlock(keeper Keeper) error { + // defer telemetry.ModuleMeasureSince(types.ModuleName, time.Now(), "commit_blocker") + + action := &commitBlockAction{ + Type: "AFTER_COMMIT_BLOCK", + BlockHeight: endBlockHeight, + BlockTime: endBlockTime, + } + _, err := keeper.BlockingSend(sdk.Context{}, action) + + // fmt.Fprintf(os.Stderr, "AFTER_COMMIT_BLOCK Returned from SwingSet: %s, %v\n", out, err) + if err != nil { + // Panic here, in the hopes that a replay from scratch will fix the problem. + panic(fmt.Errorf("AFTER_COMMIT_BLOCK failed: %w. Swingset is in an irrecoverable inconsistent state", err)) + } + return err +} diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index acfe78208b0..be8c7af3659 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -15,6 +15,7 @@ import { loadBasedir, loadSwingsetConfigFile, } from '@agoric/swingset-vat'; +import { waitUntilQuiescent } from '@agoric/swingset-vat/src/lib-nodejs/waitUntilQuiescent.js'; import { assert, Fail } from '@agoric/assert'; import { openSwingStore } from '@agoric/swing-store'; import { BridgeId as BRIDGE_ID } from '@agoric/internal'; @@ -375,6 +376,8 @@ export async function launch({ let savedBlockTime = Number(kvStore.get(getHostKey('blockTime')) || 0); let runTime = 0; let chainTime; + let saveTime = 0; + let endBlockFinish = 0; let blockParams; let decohered; let afterCommitWorkDone = Promise.resolve(); @@ -594,7 +597,7 @@ export async function launch({ } async function afterCommit(blockHeight, blockTime) { - await Promise.resolve() + await waitUntilQuiescent() .then(afterCommitCallback) .then((afterCommitStats = {}) => { controller.writeSlogObject({ @@ -672,12 +675,7 @@ export async function launch({ // Save the kernel's computed state just before the chain commits. const start2 = Date.now(); await saveOutsideState(savedHeight, blockTime); - const saveTime = Date.now() - start2; - controller.writeSlogObject({ - type: 'cosmic-swingset-commit-block-finish', - blockHeight, - blockTime, - }); + saveTime = Date.now() - start2; blockParams = undefined; @@ -685,6 +683,23 @@ export async function launch({ `wrote SwingSet checkpoint [run=${runTime}ms, chainSave=${chainTime}ms, kernelSave=${saveTime}ms]`, ); + return undefined; + } + + case ActionType.AFTER_COMMIT_BLOCK: { + const { blockHeight, blockTime } = action; + + const fullSaveTime = Date.now() - endBlockFinish; + + controller.writeSlogObject({ + type: 'cosmic-swingset-commit-block-finish', + blockHeight, + blockTime, + saveTime: saveTime / 1000, + chainTime: chainTime / 1000, + fullSaveTime: fullSaveTime / 1000, + }); + afterCommitWorkDone = afterCommit(blockHeight, blockTime); return undefined; @@ -759,6 +774,8 @@ export async function launch({ actionQueueStats: inboundQueueMetrics.getStats(), }); + endBlockFinish = Date.now(); + return undefined; } diff --git a/packages/internal/src/action-types.js b/packages/internal/src/action-types.js index 4e9d5d6e1c4..c24c700c4f0 100644 --- a/packages/internal/src/action-types.js +++ b/packages/internal/src/action-types.js @@ -5,6 +5,7 @@ export const CORE_EVAL = 'CORE_EVAL'; export const DELIVER_INBOUND = 'DELIVER_INBOUND'; export const END_BLOCK = 'END_BLOCK'; export const COMMIT_BLOCK = 'COMMIT_BLOCK'; +export const AFTER_COMMIT_BLOCK = 'AFTER_COMMIT_BLOCK'; export const IBC_EVENT = 'IBC_EVENT'; export const PLEASE_PROVISION = 'PLEASE_PROVISION'; export const VBANK_BALANCE_UPDATE = 'VBANK_BALANCE_UPDATE'; From 37858cba6b6799782d544c2317a860bf355f741f Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Wed, 8 Mar 2023 16:59:22 +0000 Subject: [PATCH 02/20] fix(cosmic-swingset): remove unused saved blockTime --- packages/cosmic-swingset/src/launch-chain.js | 8 ++------ packages/cosmic-swingset/src/sim-chain.js | 10 ++++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index be8c7af3659..d6e657372e5 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -297,10 +297,9 @@ export async function launch({ await mailboxStorage.commit(); } - async function saveOutsideState(blockHeight, blockTime) { + async function saveOutsideState(blockHeight) { const chainSends = clearChainSends(); kvStore.set(getHostKey('height'), `${blockHeight}`); - kvStore.set(getHostKey('blockTime'), `${blockTime}`); kvStore.set(getHostKey('chainSends'), JSON.stringify(chainSends)); await commit(); @@ -373,7 +372,6 @@ export async function launch({ } let savedHeight = Number(kvStore.get(getHostKey('height')) || 0); - let savedBlockTime = Number(kvStore.get(getHostKey('blockTime')) || 0); let runTime = 0; let chainTime; let saveTime = 0; @@ -674,7 +672,7 @@ export async function launch({ // Save the kernel's computed state just before the chain commits. const start2 = Date.now(); - await saveOutsideState(savedHeight, blockTime); + await saveOutsideState(savedHeight); saveTime = Date.now() - start2; blockParams = undefined; @@ -765,7 +763,6 @@ export async function launch({ // Advance our saved state variables. savedHeight = blockHeight; - savedBlockTime = blockTime; } controller.writeSlogObject({ type: 'cosmic-swingset-end-block-finish', @@ -793,7 +790,6 @@ export async function launch({ blockingSend, shutdown, savedHeight, - savedBlockTime, savedChainSends: JSON.parse(kvStore.get(getHostKey('chainSends')) || '[]'), }; } diff --git a/packages/cosmic-swingset/src/sim-chain.js b/packages/cosmic-swingset/src/sim-chain.js index eb391f17bfc..accf8bbb0dd 100644 --- a/packages/cosmic-swingset/src/sim-chain.js +++ b/packages/cosmic-swingset/src/sim-chain.js @@ -113,10 +113,9 @@ export async function connectToFakeChain(basedir, GCI, delay, inbound) { slogSender, }); - const { blockingSend, savedHeight, savedBlockTime } = s; + const { blockingSend, savedHeight } = s; let blockHeight = savedHeight; - let blockTime = savedBlockTime || scaleBlockTime(Date.now()); const intoChain = []; let nextBlockTimeout = 0; @@ -125,7 +124,7 @@ export async function connectToFakeChain(basedir, GCI, delay, inbound) { const withBlockQueue = makeWithQueue(); const unhandledSimulateBlock = withBlockQueue( async function unqueuedSimulateBlock() { - blockTime = scaleBlockTime(Date.now()); + const blockTime = scaleBlockTime(Date.now()); blockHeight += 1; const params = DEFAULT_SIM_SWINGSET_PARAMS; @@ -203,7 +202,10 @@ export async function connectToFakeChain(basedir, GCI, delay, inbound) { // The before-first-block is special... do it now. // This emulates what x/swingset does to run a BOOTSTRAP_BLOCK // before continuing with the real initialHeight. - await blockingSend({ type: 'BOOTSTRAP_BLOCK', blockTime }); + await blockingSend({ + type: 'BOOTSTRAP_BLOCK', + blockTime: scaleBlockTime(Date.now()), + }); blockHeight = initialHeight; }; From be684315dc68ecf0cb603a8eb38ddd5418e996a6 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 10 Feb 2023 20:38:58 +0000 Subject: [PATCH 03/20] feat(cosmic-swingset): export swingStore kvData to vstorage --- golang/cosmos/x/swingset/keeper/keeper.go | 1 + packages/cosmic-swingset/src/chain-main.js | 39 +++++++++++++++++++- packages/cosmic-swingset/src/launch-chain.js | 19 +++++++++- packages/internal/src/chain-storage-paths.js | 1 + 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/keeper.go b/golang/cosmos/x/swingset/keeper/keeper.go index 25ea4d997ce..f498a6a69cf 100644 --- a/golang/cosmos/x/swingset/keeper/keeper.go +++ b/golang/cosmos/x/swingset/keeper/keeper.go @@ -32,6 +32,7 @@ const ( StoragePathMailbox = "mailbox" StoragePathCustom = "published" StoragePathBundles = "bundles" + StoragePathSwingStore = "swingStore" ) // 2 ** 256 - 1 diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 8183cdc2613..a83316f0066 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -14,7 +14,7 @@ import { exportMailbox, } from '@agoric/swingset-vat/src/devices/mailbox/mailbox.js'; -import { Fail } from '@agoric/assert'; +import { Fail, q } from '@agoric/assert'; import { makeSlogSender, tryFlushSlogSender } from '@agoric/telemetry'; import { @@ -243,6 +243,42 @@ export default async function main(progname, args, { env, homedir, agcc }) { commit: actionQueueRawStorage.commit, abort: actionQueueRawStorage.abort, }); + /** + * Callback invoked during SwingSet execution when new "export data" is + * generated by swingStore to be saved in the host's verified DB. In our + * case, we publish these entries in vstorage under a dedicated prefix. + * This effectively shadows the "export data" of the swingStore so that + * processes like state-sync can generate a verified "root of trust" to + * restore SwingSet state. + * + * @param {ReadonlyArray<[path: string, value?: string | null]>} updates + */ + const swingStoreExportCallback = async updates => { + // Allow I/O to proceed first + await waitUntilQuiescent(); + + const entries = updates.map(([key, value]) => { + if (typeof key !== 'string') { + throw Fail`Unexpected swingStore exported key ${q(key)}`; + } + const path = `${STORAGE_PATH.SWING_STORE}.${key}`; + if (value == null) { + return [path]; + } + if (typeof value !== 'string') { + throw Fail`Unexpected ${typeof value} value for swingStore exported key ${q( + key, + )}`; + } + return [path, value]; + }); + sendToChain( + stringify({ + method: 'setWithoutNotify', + args: entries, + }), + ); + }; function setActivityhash(activityhash) { const entry = [STORAGE_PATH.ACTIVITYHASH, activityhash]; const msg = stringify({ @@ -399,6 +435,7 @@ export default async function main(progname, args, { env, homedir, agcc }) { verboseBlocks: true, metricsProvider, slogSender, + swingStoreExportCallback, swingStoreTraceFile, keepSnapshots, afterCommitCallback, diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index d6e657372e5..033ab11bb16 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -19,6 +19,7 @@ import { waitUntilQuiescent } from '@agoric/swingset-vat/src/lib-nodejs/waitUnti import { assert, Fail } from '@agoric/assert'; import { openSwingStore } from '@agoric/swing-store'; import { BridgeId as BRIDGE_ID } from '@agoric/internal'; +import { makeWithQueue } from '@agoric/internal/src/queue.js'; import * as ActionType from '@agoric/internal/src/action-types.js'; import { extractCoreProposalBundles } from '@agoric/deploy-script-support/src/extract-proposal.js'; @@ -226,13 +227,28 @@ export async function launch({ metricsProvider = makeDefaultMeterProvider(), slogSender, swingStoreTraceFile, + swingStoreExportCallback, keepSnapshots, afterCommitCallback = async () => ({}), }) { console.info('Launching SwingSet kernel'); + // The swingStore's exportCallback is synchronous, however we allow the + // callback provided to launch-chain to be asynchronous. The callbacks are + // invoked sequentially like if they were awaited, and the block manager + // synchronizes before finishing END_BLOCK + let pendingSwingStoreExport = Promise.resolve(); + const swingStoreExportCallbackWithQueue = + swingStoreExportCallback && makeWithQueue()(swingStoreExportCallback); + const swingStoreExportSyncCallback = + swingStoreExportCallback && + (updates => { + pendingSwingStoreExport = swingStoreExportCallbackWithQueue(updates); + }); + const { kernelStorage, hostStorage } = openSwingStore(kernelStateDBDir, { traceFile: swingStoreTraceFile, + exportCallback: swingStoreExportSyncCallback, keepSnapshots, }); const { kvStore, commit } = hostStorage; @@ -647,6 +663,7 @@ export async function launch({ blockHeight, runNum, }); + await pendingSwingStoreExport; controller.writeSlogObject({ type: 'cosmic-swingset-bootstrap-block-finish', blockTime, @@ -758,7 +775,7 @@ export async function launch({ // We write out our on-chain state as a number of chainSends. const start = Date.now(); - await saveChainState(); + await Promise.all([saveChainState(), pendingSwingStoreExport]); chainTime = Date.now() - start; // Advance our saved state variables. diff --git a/packages/internal/src/chain-storage-paths.js b/packages/internal/src/chain-storage-paths.js index c2d82a595ea..93ff21367bd 100644 --- a/packages/internal/src/chain-storage-paths.js +++ b/packages/internal/src/chain-storage-paths.js @@ -11,3 +11,4 @@ export const EGRESS = 'egress'; export const MAILBOX = 'mailbox'; export const BUNDLES = 'bundles'; export const CUSTOM = 'published'; +export const SWING_STORE = 'swingStore'; From 5dcc44d31be0c8a95a5749d768791fa35b72dbd3 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sat, 11 Mar 2023 00:45:41 +0000 Subject: [PATCH 04/20] feat(cosmic-swingset): remove unnecessary explicit activityhash The activityhash is now part of the swingstore export --- golang/cosmos/x/swingset/keeper/keeper.go | 15 ++-- packages/cosmic-swingset/src/chain-main.js | 9 --- packages/cosmic-swingset/src/launch-chain.js | 72 ++++++++------------ packages/internal/src/chain-storage-paths.js | 3 +- 4 files changed, 38 insertions(+), 61 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/keeper.go b/golang/cosmos/x/swingset/keeper/keeper.go index f498a6a69cf..2221b886ea2 100644 --- a/golang/cosmos/x/swingset/keeper/keeper.go +++ b/golang/cosmos/x/swingset/keeper/keeper.go @@ -25,14 +25,13 @@ import ( // Top-level paths for chain storage should remain synchronized with // packages/internal/src/chain-storage-paths.js const ( - StoragePathActionQueue = "actionQueue" - StoragePathActivityhash = "activityhash" - StoragePathBeansOwing = "beansOwing" - StoragePathEgress = "egress" - StoragePathMailbox = "mailbox" - StoragePathCustom = "published" - StoragePathBundles = "bundles" - StoragePathSwingStore = "swingStore" + StoragePathActionQueue = "actionQueue" + StoragePathBeansOwing = "beansOwing" + StoragePathEgress = "egress" + StoragePathMailbox = "mailbox" + StoragePathCustom = "published" + StoragePathBundles = "bundles" + StoragePathSwingStore = "swingStore" ) // 2 ** 256 - 1 diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index a83316f0066..f8c08054ad4 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -279,14 +279,6 @@ export default async function main(progname, args, { env, homedir, agcc }) { }), ); }; - function setActivityhash(activityhash) { - const entry = [STORAGE_PATH.ACTIVITYHASH, activityhash]; - const msg = stringify({ - method: 'set', - args: [entry], - }); - chainSend(portNums.storage, msg); - } function doOutboundBridge(dstID, msg) { const portNum = portNums[dstID]; if (portNum === undefined) { @@ -427,7 +419,6 @@ export default async function main(progname, args, { env, homedir, agcc }) { mailboxStorage, clearChainSends, replayChainSends, - setActivityhash, bridgeOutbound: doOutboundBridge, vatconfig, argv, diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 033ab11bb16..1bba4acbc8f 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -216,7 +216,6 @@ export async function launch({ mailboxStorage, clearChainSends, replayChainSends, - setActivityhash, bridgeOutbound, makeInstallationPublisher, vatconfig, @@ -303,9 +302,6 @@ export async function launch({ // entire bootstrap before opening for business. const policy = neverStop(); await crankScheduler(policy); - if (setActivityhash) { - setActivityhash(controller.getActivityhash()); - } } async function saveChainState() { @@ -482,47 +478,39 @@ export async function launch({ return runPolicy.shouldRun(); } - async function doRunKernel() { - // First, complete leftover work, if any - let keepGoing = await runSwingset(); - if (!keepGoing) return; - - // Then, update the timer device with the new external time, which might - // push work onto the kernel run-queue (if any timers were ready to wake). - const addedToQueue = timer.poll(blockTime); - console.debug( - `polled; blockTime:${blockTime}, h:${blockHeight}; ADDED =`, - addedToQueue, - ); - // We must run the kernel even if nothing was added since the kernel - // only notes state exports and updates consistency hashes when attempting - // to perform a crank. + // First, complete leftover work, if any + let keepGoing = await runSwingset(); + if (!keepGoing) return; + + // Then, update the timer device with the new external time, which might + // push work onto the kernel run-queue (if any timers were ready to wake). + const addedToQueue = timer.poll(blockTime); + console.debug( + `polled; blockTime:${blockTime}, h:${blockHeight}; ADDED =`, + addedToQueue, + ); + // We must run the kernel even if nothing was added since the kernel + // only notes state exports and updates consistency hashes when attempting + // to perform a crank. + keepGoing = await runSwingset(); + if (!keepGoing) return; + + // Finally, process as much as we can from the actionQueue, which contains + // first the old actions followed by the newActions, running the + // kernel to completion after each. + for (const { action, context } of actionQueue.consumeAll()) { + const inboundNum = `${context.blockHeight}-${context.txHash}-${context.msgIdx}`; + inboundQueueMetrics.decStat(); + // eslint-disable-next-line no-await-in-loop + await performAction(action, inboundNum); + // eslint-disable-next-line no-await-in-loop keepGoing = await runSwingset(); - if (!keepGoing) return; - - // Finally, process as much as we can from the actionQueue, which contains - // first the old actions followed by the newActions, running the - // kernel to completion after each. - for (const { action, context } of actionQueue.consumeAll()) { - const inboundNum = `${context.blockHeight}-${context.txHash}-${context.msgIdx}`; - inboundQueueMetrics.decStat(); - // eslint-disable-next-line no-await-in-loop - await performAction(action, inboundNum); - // eslint-disable-next-line no-await-in-loop - keepGoing = await runSwingset(); - if (!keepGoing) { - // any leftover actions will remain on the actionQueue for possible - // processing in the next block - return; - } + if (!keepGoing) { + // any leftover actions will remain on the actionQueue for possible + // processing in the next block + return; } } - - await doRunKernel(); - - if (setActivityhash) { - setActivityhash(controller.getActivityhash()); - } } async function endBlock(blockHeight, blockTime, params) { diff --git a/packages/internal/src/chain-storage-paths.js b/packages/internal/src/chain-storage-paths.js index 93ff21367bd..f5f0d1796f6 100644 --- a/packages/internal/src/chain-storage-paths.js +++ b/packages/internal/src/chain-storage-paths.js @@ -1,11 +1,10 @@ /** * These identify top-level paths for SwingSet chain storage - * (and serve as prefixes, with the exception of ACTIVITYHASH). + * (and serve as prefixes). * To avoid collisions, they should remain synchronized with * golang/cosmos/x/swingset/keeper/keeper.go */ export const ACTION_QUEUE = 'actionQueue'; -export const ACTIVITYHASH = 'activityhash'; export const BEANSOWING = 'beansOwing'; export const EGRESS = 'egress'; export const MAILBOX = 'mailbox'; From 726e188481f4286583e48e6c3d87b654a7dd8d73 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sat, 11 Feb 2023 22:58:24 +0000 Subject: [PATCH 05/20] feat(vstorage): Export sub tree --- golang/cosmos/x/vstorage/keeper/keeper.go | 23 +++++++++++++++++++ .../cosmos/x/vstorage/keeper/keeper_test.go | 18 +++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/golang/cosmos/x/vstorage/keeper/keeper.go b/golang/cosmos/x/vstorage/keeper/keeper.go index b8728d14bc6..8505d1eb5d3 100644 --- a/golang/cosmos/x/vstorage/keeper/keeper.go +++ b/golang/cosmos/x/vstorage/keeper/keeper.go @@ -120,8 +120,27 @@ func NewKeeper(storeKey sdk.StoreKey) Keeper { // ExportStorage fetches all storage func (k Keeper) ExportStorage(ctx sdk.Context) []*types.DataEntry { + return k.ExportStorageFromPrefix(ctx, "") +} + +// ExportStorageFromPrefix fetches storage only under the supplied pathPrefix. +func (k Keeper) ExportStorageFromPrefix(ctx sdk.Context, pathPrefix string) []*types.DataEntry { store := ctx.KVStore(k.storeKey) + if len(pathPrefix) > 0 { + if err := types.ValidatePath(pathPrefix); err != nil { + panic(err) + } + pathPrefix = pathPrefix + types.PathSeparator + } + + // since vstorage encodes keys with a prefix indicating the number of path + // elements, exporting all entries under a given path cannot use a prefix + // iterator. Instead we iterate over the whole vstorage content and check + // whether each entry matches the path prefix. This choice assumes most + // entries will be exported. An alternative implementation would be to + // recursively list all children under the pathPrefix, and export them. + iterator := sdk.KVStorePrefixIterator(store, nil) exported := []*types.DataEntry{} @@ -135,10 +154,14 @@ func (k Keeper) ExportStorage(ctx sdk.Context) []*types.DataEntry { continue } path := types.EncodedKeyToPath(iterator.Key()) + if !strings.HasPrefix(path, pathPrefix) { + continue + } value, hasPrefix := cutPrefix(rawValue, types.EncodedDataPrefix) if !hasPrefix { panic(fmt.Errorf("value at path %q starts with unexpected prefix", path)) } + path = path[len(pathPrefix):] entry := types.DataEntry{Path: path, Value: string(value)} exported = append(exported, &entry) } diff --git a/golang/cosmos/x/vstorage/keeper/keeper_test.go b/golang/cosmos/x/vstorage/keeper/keeper_test.go index b07bf81fbda..d53c67ee842 100644 --- a/golang/cosmos/x/vstorage/keeper/keeper_test.go +++ b/golang/cosmos/x/vstorage/keeper/keeper_test.go @@ -170,11 +170,21 @@ func TestStorage(t *testing.T) { {Path: "key2.child2.grandchild2", Value: "value2grandchild"}, {Path: "key2.child2.grandchild2a", Value: "value2grandchilda"}, } - got := keeper.ExportStorage(ctx) - if !reflect.DeepEqual(got, expectedExport) { - t.Errorf("got export %q, want %q", got, expectedExport) + gotExport := keeper.ExportStorage(ctx) + if !reflect.DeepEqual(gotExport, expectedExport) { + t.Errorf("got export %q, want %q", gotExport, expectedExport) } - keeper.ImportStorage(ctx, got) + + // Check the export. + expectedKey2Export := []*types.DataEntry{ + {Path: "child2.grandchild2", Value: "value2grandchild"}, + {Path: "child2.grandchild2a", Value: "value2grandchilda"}, + } + if got := keeper.ExportStorageFromPrefix(ctx, "key2"); !reflect.DeepEqual(got, expectedKey2Export) { + t.Errorf("got export %q, want %q", got, expectedKey2Export) + } + + keeper.ImportStorage(ctx, gotExport) } func TestStorageNotify(t *testing.T) { From b1072d8b1ddabbb5f2835eb503c945fed3b6b080 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 12 Feb 2023 03:21:25 +0000 Subject: [PATCH 06/20] feat(cosmic-swingset): basic snapshot wiring --- go.mod | 2 +- go.sum | 4 +- golang/cosmos/app/app.go | 42 ++- golang/cosmos/x/swingset/alias.go | 1 + golang/cosmos/x/swingset/keeper/keeper.go | 4 + .../cosmos/x/swingset/keeper/snapshotter.go | 268 ++++++++++++++++++ packages/cosmic-swingset/src/chain-main.js | 52 +++- packages/cosmic-swingset/src/launch-chain.js | 10 +- packages/internal/src/action-types.js | 1 + packages/telemetry/src/slog-to-otel.js | 14 + 10 files changed, 384 insertions(+), 14 deletions(-) create mode 100644 golang/cosmos/x/swingset/keeper/snapshotter.go diff --git a/go.mod b/go.mod index fb583a93081..a28f1050a37 100644 --- a/go.mod +++ b/go.mod @@ -145,7 +145,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.33.2 replace github.com/tendermint/tendermint => github.com/agoric-labs/tendermint v0.34.23-alpha.agoric.3 // We need a fork of cosmos-sdk until all of the differences are merged. -replace github.com/cosmos/cosmos-sdk => github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835 +replace github.com/cosmos/cosmos-sdk => github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db replace github.com/cosmos/gaia/v7 => github.com/Agoric/ag0/v7 v7.0.2-alpha.agoric.1 diff --git a/go.sum b/go.sum index 97620c6dc68..767f82486e6 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,8 @@ github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1: github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835 h1:Mmw52cHAUNwtaNXpk7b3lTeoCRd5Vw9Fdrly5ABIxCA= -github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835/go.mod h1:fdXvzy+wmYB+W+N139yb0+szbT7zAGgUjmxm5DBrjto= +github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db h1:FwZL+MsLF3sLqawUJI3etRSqrxxPEvqdrWqTEQTO18Y= +github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db/go.mod h1:fdXvzy+wmYB+W+N139yb0+szbT7zAGgUjmxm5DBrjto= github.com/agoric-labs/cosmos-sdk/ics23/go v0.8.0-alpha.agoric.1 h1:2jvHI/2d+psWAZy6FQ0vXJCHUtfU3ZbbW+pQFL04arQ= github.com/agoric-labs/cosmos-sdk/ics23/go v0.8.0-alpha.agoric.1/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= github.com/agoric-labs/tendermint v0.34.23-alpha.agoric.3 h1:aq6F1r3RQkKUYNeMNjRxgGn3dayvKnDK/R6gQF0WoFs= diff --git a/golang/cosmos/app/app.go b/golang/cosmos/app/app.go index 0f79ae21fa8..a8d9982b094 100644 --- a/golang/cosmos/app/app.go +++ b/golang/cosmos/app/app.go @@ -109,6 +109,7 @@ import ( "github.com/Agoric/agoric-sdk/golang/cosmos/x/lien" "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset" swingsetclient "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/client" + swingsetkeeper "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/keeper" swingsettypes "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types" "github.com/Agoric/agoric-sdk/golang/cosmos/x/vbank" vbanktypes "github.com/Agoric/agoric-sdk/golang/cosmos/x/vbank/types" @@ -227,11 +228,12 @@ type GaiaApp struct { // nolint: golint FeeGrantKeeper feegrantkeeper.Keeper AuthzKeeper authzkeeper.Keeper - SwingSetKeeper swingset.Keeper - VstorageKeeper vstorage.Keeper - VibcKeeper vibc.Keeper - VbankKeeper vbank.Keeper - LienKeeper lien.Keeper + SwingSetKeeper swingset.Keeper + SwingSetSnapshotter swingset.Snapshotter + VstorageKeeper vstorage.Keeper + VibcKeeper vibc.Keeper + VbankKeeper vbank.Keeper + LienKeeper lien.Keeper // make scoped keepers public for test purposes ScopedIBCKeeper capabilitykeeper.ScopedKeeper @@ -447,6 +449,12 @@ func NewAgoricApp( callToController, ) + app.SwingSetSnapshotter = swingsetkeeper.NewSwingsetSnapshotter( + bApp, + app.SwingSetKeeper, + sendToController, + ) + app.VibcKeeper = vibc.NewKeeper( appCodec, keys[vibc.StoreKey], app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper, @@ -753,6 +761,12 @@ func NewAgoricApp( app.ScopedVibcKeeper = scopedVibcKeeper app.ScopedTransferKeeper = scopedTransferKeeper app.ScopedICAHostKeeper = scopedICAHostKeeper + snapshotManager := app.SnapshotManager() + if snapshotManager != nil { + if err = snapshotManager.RegisterExtensions(&app.SwingSetSnapshotter); err != nil { + panic(fmt.Errorf("failed to register snapshot extension: %s", err)) + } + } return app } @@ -847,19 +861,33 @@ func (app *GaiaApp) InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci // Commit tells the controller that the block is commited func (app *GaiaApp) Commit() abci.ResponseCommit { + err := app.SwingSetSnapshotter.WaitUntilSnapshotStarted() + + if err != nil { + app.Logger().Error("swingset snapshot failed to start", "err", err) + } + // Frontrun the BaseApp's Commit method - err := swingset.CommitBlock(app.SwingSetKeeper) + err = swingset.CommitBlock(app.SwingSetKeeper) if err != nil { panic(err.Error()) } - res := app.BaseApp.Commit() + res, snapshotHeight := app.BaseApp.CommitWithoutSnapshot() err = swingset.AfterCommitBlock(app.SwingSetKeeper) if err != nil { panic(err.Error()) } + if snapshotHeight > 0 { + err = app.SwingSetSnapshotter.InitiateSnapshot(snapshotHeight) + + if err != nil { + app.Logger().Error("failed to initiate swingset snapshot", "err", err) + } + } + return res } diff --git a/golang/cosmos/x/swingset/alias.go b/golang/cosmos/x/swingset/alias.go index 25d3d958c5c..9c4b80c96e8 100644 --- a/golang/cosmos/x/swingset/alias.go +++ b/golang/cosmos/x/swingset/alias.go @@ -22,6 +22,7 @@ var ( type ( Keeper = keeper.Keeper + Snapshotter = keeper.SwingsetSnapshotter Egress = types.Egress MsgDeliverInbound = types.MsgDeliverInbound MsgProvision = types.MsgProvision diff --git a/golang/cosmos/x/swingset/keeper/keeper.go b/golang/cosmos/x/swingset/keeper/keeper.go index 2221b886ea2..087ad7b538f 100644 --- a/golang/cosmos/x/swingset/keeper/keeper.go +++ b/golang/cosmos/x/swingset/keeper/keeper.go @@ -443,6 +443,10 @@ func (k Keeper) SetMailbox(ctx sdk.Context, peer string, mailbox string) { k.vstorageKeeper.LegacySetStorageAndNotify(ctx, vstoragetypes.NewStorageEntry(path, mailbox)) } +func (k Keeper) ExportSwingStore(ctx sdk.Context) []*vstoragetypes.DataEntry { + return k.vstorageKeeper.ExportStorageFromPrefix(ctx, StoragePathSwingStore) +} + func (k Keeper) PathToEncodedKey(path string) []byte { return k.vstorageKeeper.PathToEncodedKey(path) } diff --git a/golang/cosmos/x/swingset/keeper/snapshotter.go b/golang/cosmos/x/swingset/keeper/snapshotter.go new file mode 100644 index 00000000000..810097eaf00 --- /dev/null +++ b/golang/cosmos/x/swingset/keeper/snapshotter.go @@ -0,0 +1,268 @@ +package keeper + +import ( + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/Agoric/agoric-sdk/golang/cosmos/vm" + "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types" + vstoragetypes "github.com/Agoric/agoric-sdk/golang/cosmos/x/vstorage/types" + "github.com/cosmos/cosmos-sdk/baseapp" + snapshots "github.com/cosmos/cosmos-sdk/snapshots/types" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" +) + +var _ snapshots.ExtensionSnapshotter = &SwingsetSnapshotter{} + +const SnapshotFormat = 1 + +type activeSnapshot struct { + // The block height of the snapshot in progress + height int64 + // The logger for this snapshot + logger log.Logger + // Use to synchronize the commit boundary + startedResult chan error + // Internal flag indicating whether the cosmos driven snapshot process completed + retrieved bool +} + +type SwingStoreExporter interface { + ExportSwingStore(ctx sdk.Context) []*vstoragetypes.DataEntry +} + +type SwingsetSnapshotter struct { + app *baseapp.BaseApp + logger log.Logger + exporter SwingStoreExporter + blockingSend func(action vm.Jsonable) (string, error) + activeSnapshot *activeSnapshot +} + +type snapshotAction struct { + Type string `json:"type"` // COSMOS_SNAPSHOT + BlockHeight int64 `json:"blockHeight"` + Request string `json:"request"` // "initiate", "discard", "retrieve", or "restore" + Args []json.RawMessage `json:"args,omitempty"` +} + +func NewSwingsetSnapshotter(app *baseapp.BaseApp, exporter SwingStoreExporter, sendToController func(bool, string) (string, error)) SwingsetSnapshotter { + // The sendToController performed by this submodule are non-deterministic. + // This submodule will send messages to JS from goroutines at unpredictable + // times, but this is safe because when handling the messages, the JS side + // does not perform operations affecting consensus and ignores state changes + // since committing the previous block. + // Since this submodule implements block level commit synchronization, the + // processing and results are both insensitive to sub-block timing of messages. + + blockingSend := func(action vm.Jsonable) (string, error) { + bz, err := json.Marshal(action) + if err != nil { + return "", err + } + return sendToController(true, string(bz)) + } + + return SwingsetSnapshotter{ + app: app, + logger: app.Logger().With("module", fmt.Sprintf("x/%s", types.ModuleName), "submodule", "snapshotter"), + exporter: exporter, + blockingSend: blockingSend, + activeSnapshot: nil, + } +} + +// InitiateSnapshot synchronously initiates a snapshot for the given height. +// If a snapshot is already in progress, or if no snapshot manager is configured, +// this will fail. +// The snapshot operation is performed in a goroutine, and synchronized with the +// main thread through the `WaitUntilSnapshotStarted` method. +func (snapshotter *SwingsetSnapshotter) InitiateSnapshot(height int64) error { + if snapshotter.activeSnapshot != nil { + return fmt.Errorf("snapshot already in progress for height %d", snapshotter.activeSnapshot.height) + } + + if snapshotter.app.SnapshotManager() == nil { + return fmt.Errorf("snapshot manager not configured") + } + + logger := snapshotter.logger.With("height", height) + + // Indicate that a snapshot has been initiated by setting `activeSnapshot`. + // This structure is used to synchronize with the goroutine spawned below. + // It's nilled-out before exiting (and is the only code that does so). + snapshotter.activeSnapshot = &activeSnapshot{ + height: height, + logger: logger, + startedResult: make(chan error, 1), + retrieved: false, + } + + go func(startedResult chan error) { + action := &snapshotAction{ + Type: "COSMOS_SNAPSHOT", + BlockHeight: snapshotter.activeSnapshot.height, + Request: "initiate", + } + + // blockingSend for COSMOS_SNAPSHOT action is safe to call from a goroutine + _, err := snapshotter.blockingSend(action) + + if err != nil { + // First indicate a snapshot is no longer in progress if the call to + // `WaitUntilSnapshotStarted` has't happened yet. + snapshotter.activeSnapshot = nil + // Then signal the current snapshot operation if a call to + // `WaitUntilSnapshotStarted` was already waiting. + startedResult <- err + close(startedResult) + logger.Error("failed to initiate swingset snapshot", "err", err) + return + } + + // Signal that the snapshot operation has started in the goroutine. Calls to + // `WaitUntilSnapshotStarted` will no longer block. + close(startedResult) + + snapshotter.app.Snapshot(height) + + // Check whether the cosmos Snapshot() method successfully handled our extension + if snapshotter.activeSnapshot.retrieved { + snapshotter.activeSnapshot = nil + return + } + + logger.Error("failed to make swingset snapshot") + action = &snapshotAction{ + Type: "COSMOS_SNAPSHOT", + BlockHeight: snapshotter.activeSnapshot.height, + Request: "discard", + } + _, err = snapshotter.blockingSend(action) + + if err != nil { + logger.Error("failed to discard of swingset snapshot", "err", err) + } + + snapshotter.activeSnapshot = nil + }(snapshotter.activeSnapshot.startedResult) + + return nil +} + +// WaitUntilSnapshotStarted synchronizes with a snapshot in progress, if any. +// The JS SwingStore export must have started before a new block is committed. +// The app must call this method before sending a commit action to SwingSet. +// +// Waits for a just initiated snapshot to have started in its goroutine. +// If no snapshot is in progress (`InitiateSnapshot` hasn't been called or +// already completed), or if we previously checked if the snapshot had started, +// returns immediately. +func (snapshotter *SwingsetSnapshotter) WaitUntilSnapshotStarted() error { + // First, copy the synchronization structure in case the snapshot operation + // completes while we check its status + activeSnapshot := snapshotter.activeSnapshot + + if activeSnapshot == nil { + return nil + } + + // The snapshot goroutine only produces a value in case of an error, + // and closes the channel once the snapshot has started or failed. + // Only the first call after a snapshot was initiated will report an error. + return <-activeSnapshot.startedResult +} + +// SnapshotName returns the name of snapshotter, it should be unique in the manager. +// Implements ExtensionSnapshotter +func (snapshotter *SwingsetSnapshotter) SnapshotName() string { + return types.ModuleName +} + +// SnapshotFormat returns the default format the extension snapshotter uses to encode the +// payloads when taking a snapshot. +// It's defined within the extension, different from the global format for the whole state-sync snapshot. +// Implements ExtensionSnapshotter +func (snapshotter *SwingsetSnapshotter) SnapshotFormat() uint32 { + return SnapshotFormat +} + +// SupportedFormats returns a list of formats it can restore from. +// Implements ExtensionSnapshotter +func (snapshotter *SwingsetSnapshotter) SupportedFormats() []uint32 { + return []uint32{SnapshotFormat} +} + +// SnapshotExtension writes extension payloads into the underlying protobuf stream. +// This operation is invoked by the snapshot manager in the goroutine started by +// `InitiateSnapshot`. +// Implements ExtensionSnapshotter +func (snapshotter *SwingsetSnapshotter) SnapshotExtension(height uint64, payloadWriter snapshots.ExtensionPayloadWriter) (err error) { + defer func() { + // Since the cosmos layers do a poor job of reporting errors, do our own reporting + // `err` will be set correctly regardless if it was explicitly assigned or + // a value was provided to a `return` statement. + // See https://go.dev/blog/defer-panic-and-recover for details + if err != nil { + var logger log.Logger + if snapshotter.activeSnapshot != nil { + logger = snapshotter.activeSnapshot.logger + } else { + logger = snapshotter.logger + } + + logger.Error("swingset snapshot extension failed", "err", err) + } + }() + + if snapshotter.activeSnapshot == nil { + return errors.New("no active swingset snapshot") + } + + if snapshotter.activeSnapshot.height != int64(height) { + return fmt.Errorf("swingset snapshot requested for unexpected height %d (expected %d)", height, snapshotter.activeSnapshot.height) + } + + action := &snapshotAction{ + Type: "COSMOS_SNAPSHOT", + BlockHeight: snapshotter.activeSnapshot.height, + Request: "retrieve", + } + out, err := snapshotter.blockingSend(action) + + if err != nil { + return err + } + + var exportDir string + err = json.Unmarshal([]byte(out), &exportDir) + if err != nil { + return err + } + + defer os.RemoveAll(exportDir) + snapshotter.activeSnapshot.retrieved = true + + // FIXME: actually do something with the snapshot + + return errors.New("not implemented") +} + +// RestoreExtension restores an extension state snapshot, +// the payload reader returns `io.EOF` when it reaches the extension boundaries. +// Implements ExtensionSnapshotter +func (snapshotter *SwingsetSnapshotter) RestoreExtension(height uint64, format uint32, payloadReader snapshots.ExtensionPayloadReader) error { + if format != SnapshotFormat { + return snapshots.ErrUnknownFormat + } + + ctx := snapshotter.app.NewUncachedContext(false, tmproto.Header{Height: int64(height)}) + + _ = snapshotter.exporter.ExportSwingStore(ctx) + + return errors.New("not implemented") +} diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index f8c08054ad4..5fc211e6463 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -24,6 +24,7 @@ import { import { makeMarshal } from '@endo/marshal'; import * as STORAGE_PATH from '@agoric/internal/src/chain-storage-paths.js'; +import * as ActionType from '@agoric/internal/src/action-types.js'; import { BridgeId as BRIDGE_ID } from '@agoric/internal'; import { makeBufferedStorage, @@ -209,6 +210,11 @@ export default async function main(progname, args, { env, homedir, agcc }) { } } + /** @type {Awaited>['blockingSend'] | undefined} */ + let blockingSend; + /** @type {((obj: object) => void) | undefined} */ + let writeSlogObject; + // this storagePort changes for every single message. We define it out here // so the 'externalStorage' object can close over the single mutable // instance, and we update the 'portNums.storage' value each time toSwingSet is called @@ -432,12 +438,15 @@ export default async function main(progname, args, { env, homedir, agcc }) { afterCommitCallback, }); - savedChainSends = s.savedChainSends; + ({ blockingSend, writeSlogObject, savedChainSends } = s); - return s.blockingSend; + return blockingSend; + } + + async function handleCosmosSnapshot(_blockHeight, _request, _requestArgs) { + throw Fail`Not implemented`; } - let blockingSend; async function toSwingSet(action, _replier) { // console.log(`toSwingSet`, action); if (action.vibcPort) { @@ -458,6 +467,42 @@ export default async function main(progname, args, { env, homedir, agcc }) { portNums.lien = action.lienPort; } + // Snapshot actions are specific to cosmos chains and handled here + if (action.type === ActionType.COSMOS_SNAPSHOT) { + const { blockHeight, request, args: requestArgs } = action; + writeSlogObject?.({ + type: 'cosmic-swingset-snapshot-start', + blockHeight, + request, + args: requestArgs, + }); + + const resultP = handleCosmosSnapshot(blockHeight, request, requestArgs); + + resultP.then( + result => { + writeSlogObject?.({ + type: 'cosmic-swingset-snapshot-finish', + blockHeight, + request, + args: requestArgs, + result, + }); + }, + error => { + writeSlogObject?.({ + type: 'cosmic-swingset-snapshot-finish', + blockHeight, + request, + args: requestArgs, + error, + }); + }, + ); + + return resultP; + } + // Ensure that initialization has completed. blockingSend = await (blockingSend || launchAndInitializeSwingSet(action)); @@ -466,6 +511,7 @@ export default async function main(progname, args, { env, homedir, agcc }) { return true; } + // Block related actions are processed by `blockingSend` return blockingSend(action); } } diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 1bba4acbc8f..480b2dde981 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -611,6 +611,9 @@ export async function launch({ }); } + // Handle block related actions + // Some actions that are integration specific may be handled by the caller + // For example COSMOS_SNAPSHOT and AG_COSMOS_INIT are handled in chain-main.js async function blockingSend(action) { if (decohered) { throw decohered; @@ -787,13 +790,18 @@ export async function launch({ } } - function shutdown() { + async function shutdown() { return controller.shutdown(); } + function writeSlogObject(obj) { + controller.writeSlogObject(obj); + } + return { blockingSend, shutdown, + writeSlogObject, savedHeight, savedChainSends: JSON.parse(kvStore.get(getHostKey('chainSends')) || '[]'), }; diff --git a/packages/internal/src/action-types.js b/packages/internal/src/action-types.js index c24c700c4f0..f0e0d178ad9 100644 --- a/packages/internal/src/action-types.js +++ b/packages/internal/src/action-types.js @@ -1,4 +1,5 @@ export const BOOTSTRAP_BLOCK = 'BOOTSTRAP_BLOCK'; +export const COSMOS_SNAPSHOT = 'COSMOS_SNAPSHOT'; export const BEGIN_BLOCK = 'BEGIN_BLOCK'; export const CALCULATE_FEES_IN_BEANS = 'CALCULATE_FEES_IN_BEANS'; export const CORE_EVAL = 'CORE_EVAL'; diff --git a/packages/telemetry/src/slog-to-otel.js b/packages/telemetry/src/slog-to-otel.js index 4d3665837c7..5e42756a8d0 100644 --- a/packages/telemetry/src/slog-to-otel.js +++ b/packages/telemetry/src/slog-to-otel.js @@ -983,6 +983,20 @@ export const makeSlogToOtelKit = (tracer, overrideAttrs = {}) => { spans.pop(`swingset-run`); break; } + case 'cosmic-swingset-snapshot-start': { + // TODO: start a span on the root block span + spans + .top() + ?.addEvent('state-sync-snapshot-start', cleanAttrs(slogAttrs), now); + break; + } + case 'cosmic-swingset-snapshot-finish': { + // TODO: end the span started on the root block span + spans + .top() + ?.addEvent('state-sync-snapshot-finish', cleanAttrs(slogAttrs), now); + break; + } case 'heap-snapshot-save': { // Add an event to whatever the top span is for now (likely crank) spans.top()?.addEvent('snapshot-save', cleanAttrs(slogAttrs), now); From 30340fbf14caaa0decb1dc1c0dd2ecec150c38d6 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 17 Feb 2023 14:55:39 +0000 Subject: [PATCH 07/20] Revert "fix: disable state-sync by default, until we've implemented it" This reverts commit fedb533ee3e6a989131b1c1a8992072b562d4fa9. --- golang/cosmos/daemon/cmd/root.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index a0441d521f3..8426cd6cacb 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -232,19 +232,14 @@ func (ac appCreator) newApp( panic(err) } - // FIXME: Actually use the snapshotStore once we have a way to put SwingSet - // state into it. - var snapshotStore *snapshots.Store - if false { - snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") - snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) - if err != nil { - panic(err) - } - snapshotStore, err = snapshots.NewStore(snapshotDB, snapshotDir) - if err != nil { - panic(err) - } + snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots") + snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir) + if err != nil { + panic(err) + } + snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) + if err != nil { + panic(err) } return gaia.NewAgoricApp( From c5410ddf8d4355433ec099317f91a19b61b954d7 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 24 Feb 2023 15:32:53 +0000 Subject: [PATCH 08/20] fix(cosmic-swingset): shutdown controller on exit --- packages/cosmic-swingset/src/chain-main.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 5fc211e6463..fe74534be59 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -22,6 +22,7 @@ import { makeSerializeToStorage, } from '@agoric/internal/src/lib-chainStorage.js'; import { makeMarshal } from '@endo/marshal'; +import { makeShutdown } from '@agoric/internal/src/node/shutdown.js'; import * as STORAGE_PATH from '@agoric/internal/src/chain-storage-paths.js'; import * as ActionType from '@agoric/internal/src/action-types.js'; @@ -333,6 +334,9 @@ export default async function main(progname, args, { env, homedir, agcc }) { ); const vatconfig = new URL(vatHref).pathname; + // Delay makeShutdown to override the golang interrupts + const { registerShutdown } = makeShutdown(); + const { metricsProvider } = getTelemetryProviders({ console, env, @@ -438,7 +442,10 @@ export default async function main(progname, args, { env, homedir, agcc }) { afterCommitCallback, }); - ({ blockingSend, writeSlogObject, savedChainSends } = s); + let shutdown; + ({ blockingSend, shutdown, writeSlogObject, savedChainSends } = s); + + registerShutdown(shutdown); return blockingSend; } From df304d585928bfe3e7b9bc12ed9b1668726f54ec Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Thu, 16 Feb 2023 20:14:16 +0000 Subject: [PATCH 09/20] feat(cosmic-swingset): add kernel-db exporter --- packages/cosmic-swingset/package.json | 1 + .../cosmic-swingset/src/export-kernel-db.js | 288 ++++++++++++++++++ .../src/helpers/process-value.js | 99 ++++-- 3 files changed, 371 insertions(+), 17 deletions(-) create mode 100755 packages/cosmic-swingset/src/export-kernel-db.js diff --git a/packages/cosmic-swingset/package.json b/packages/cosmic-swingset/package.json index f7bbf360e20..4f0e1df9686 100644 --- a/packages/cosmic-swingset/package.json +++ b/packages/cosmic-swingset/package.json @@ -35,6 +35,7 @@ "@endo/import-bundle": "^0.3.3", "@endo/init": "^0.5.55", "@endo/marshal": "^0.8.4", + "@endo/promise-kit": "^0.2.55", "@iarna/toml": "^2.2.3", "@opentelemetry/sdk-metrics": "^0.32.0", "anylogger": "^0.21.0", diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js new file mode 100755 index 00000000000..567de6c4229 --- /dev/null +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -0,0 +1,288 @@ +#! /usr/bin/env node + +// @ts-check +import '@endo/init'; + +import os from 'os'; +import process from 'process'; +import fsPower from 'fs/promises'; +import pathPower from 'path'; + +import { makePromiseKit } from '@endo/promise-kit'; +import { Fail } from '@agoric/assert'; +import { makeAggregateError } from '@agoric/internal'; +import { makeShutdown } from '@agoric/internal/src/node/shutdown.js'; +import { openSwingStore, makeSwingStoreExporter } from '@agoric/swing-store'; + +import { isEntrypoint } from './helpers/is-entrypoint.js'; +import { makeProcessValue } from './helpers/process-value.js'; + +/** @typedef {'current' | 'archival' | 'debug'} SwingStoreExportMode */ + +// eslint-disable-next-line jsdoc/require-returns-check +/** + * @param {string | undefined} mode + * @returns {asserts mode is SwingStoreExportMode | undefined} + */ +const checkExportMode = mode => { + switch (mode) { + case 'current': + case 'archival': + case 'debug': + case undefined: + break; + default: + throw Fail`Invalid value ${mode} for "export-mode"`; + } +}; + +/** + * A state-sync manifest is a representation of the information contained in a + * swingStore export for a given block. + * + * The `data` field is the name of a file containing the swingStore's KV + * "export data". + * The `artifacts` field is a list of [artifactName, fileName] pairs + * (where the content of each artifact is stored in the corresponding file). + * For more details, see packages/swing-store/docs/data-export.md + * + * @typedef {object} StateSyncManifest + * @property {number} blockHeight the block height corresponding to this export + * @property {SwingStoreExportMode} [mode] + * @property {string} [data] file name containing the swingStore "export data" + * @property {Array<[artifactName: string, fileName: string]>} artifacts + * List of swingStore export artifacts which can be validated by the export data + */ + +/** + * @typedef {object} StateSyncExporter + * @property {() => number | undefined} getBlockHeight + * @property {() => Promise} onStarted + * @property {() => Promise} onDone + * @property {() => Promise} stop + */ + +/** + * @typedef {object} StateSyncExporterOptions + * @property {string} stateDir the directory containing the SwingStore to export + * @property {string} exportDir the directory in which to place the exported artifacts and manifest + * @property {number} [blockHeight] block height to check for + * @property {SwingStoreExportMode} [exportMode] whether to include historical or debug artifacts in the export + * @property {boolean} [includeExportData] whether to include an artifact for the export data in the export + */ + +/** + * @param {StateSyncExporterOptions} options + * @param {object} powers + * @param {Pick} powers.fs + * @param {import('path')['resolve']} powers.pathResolve + * @param {typeof import('@agoric/swing-store')['makeSwingStoreExporter']} [powers.makeSwingStoreExporter] + * @param {typeof import('@agoric/swing-store')['openSwingStore']} [powers.openSwingStore] + * @param {(...args: any[]) => void} [powers.log] + * @returns {StateSyncExporter} + */ +export const initiateSwingStoreExport = ( + { stateDir, exportDir, blockHeight, exportMode, includeExportData }, + { + fs: { open, writeFile }, + pathResolve, + makeSwingStoreExporter: makeExporter = makeSwingStoreExporter, + openSwingStore: openDB = openSwingStore, + log = console.log, + }, +) => { + const effectiveExportMode = exportMode ?? 'current'; + if (effectiveExportMode !== 'current' && !includeExportData) { + throw Fail`Must include export data if export mode not "current"`; + } + + /** @type {number | undefined} */ + let savedBlockHeight; + + /** @type {import('@endo/promise-kit').PromiseKit} */ + const startedKit = makePromiseKit(); + + /** @type {Error | undefined} */ + let stopped; + const abortIfStopped = () => { + if (stopped) throw stopped; + }; + /** @type {Array<() => Promise>} */ + const cleanup = []; + + const exportDone = (async () => { + const manifestPath = pathResolve(exportDir, 'export-manifest.json'); + const manifestFile = await open(manifestPath, 'wx'); + cleanup.push(async () => manifestFile.close()); + + const swingStoreExporter = makeExporter(stateDir, exportMode); + cleanup.push(async () => swingStoreExporter.close()); + + const { hostStorage } = openDB(stateDir); + + savedBlockHeight = Number(hostStorage.kvStore.get('host.height')) || 0; + await hostStorage.close(); + + if (blockHeight) { + blockHeight === savedBlockHeight || + Fail`DB at unexpected block height ${savedBlockHeight} (expected ${blockHeight})`; + } + + abortIfStopped(); + startedKit.resolve(); + log?.(`Starting DB export at block height ${savedBlockHeight}`); + + /** @type {StateSyncManifest} */ + const manifest = { + blockHeight: savedBlockHeight, + mode: exportMode, + artifacts: [], + }; + + if (includeExportData) { + log?.(`Writing Export Data`); + const fileName = `export-data.jsonl`; + // eslint-disable-next-line @jessie.js/no-nested-await + const exportDataFile = await open(pathResolve(exportDir, fileName), 'wx'); + cleanup.push(async () => exportDataFile.close()); + + // eslint-disable-next-line @jessie.js/no-nested-await + for await (const entry of swingStoreExporter.getExportData()) { + await exportDataFile.write(`${JSON.stringify(entry)}\n`); + } + + manifest.data = fileName; + } + abortIfStopped(); + + for await (const artifactName of swingStoreExporter.getArtifactNames()) { + abortIfStopped(); + log?.(`Writing artifact: ${artifactName}`); + const artifactData = swingStoreExporter.getArtifact(artifactName); + // Use artifactName as the file name as we trust swingStore to generate + // artifact names that are valid file names. + await writeFile(pathResolve(exportDir, artifactName), artifactData); + manifest.artifacts.push([artifactName, artifactName]); + } + + await manifestFile.write(JSON.stringify(manifest, null, 2)); + log?.(`Saved export manifest: ${manifestPath}`); + })(); + + const doCleanup = async opErr => { + const errors = []; + for await (const cleaner of cleanup.reverse()) { + await Promise.resolve() + .then(cleaner) + .catch(err => errors.push(err)); + } + if (errors.length) { + const error = makeAggregateError(errors, 'Errors while cleaning up'); + if (opErr) { + Object.defineProperty(error, 'cause', { value: opErr }); + } + throw error; + } else if (opErr) { + throw opErr; + } + }; + + const done = exportDone.then(doCleanup, doCleanup).catch(err => { + startedKit.reject(err); + throw err; + }); + + return { + getBlockHeight: () => blockHeight || savedBlockHeight, + onStarted: async () => startedKit.promise, + onDone: async () => done, + stop: async () => { + stopped ||= new Error(`Interrupted`); + return done.catch(err => { + if (err !== stopped) { + throw err; + } + }); + }, + }; +}; + +/** + * @param {string[]} args + * @param {object} powers + * @param {Partial>} powers.env + * @param {string} powers.homedir + * @param {Console} powers.console + * @param {import('fs/promises')} powers.fs + * @param {import('path')['resolve']} powers.pathResolve + */ +export const main = async ( + args, + { env, homedir, console, fs, pathResolve }, +) => { + const processValue = makeProcessValue({ env, args }); + + const stateDir = + processValue.getFlag('state-dir') || + // We try to find the actual cosmos state directory (default=~/.ag-chain-cosmos) + `${processValue.getFlag( + 'home', + `${homedir}/.ag-chain-cosmos`, + )}/data/ag-cosmos-chain-state`; + + const stateDirStat = await fs.stat(stateDir); + if (!stateDirStat.isDirectory()) { + throw new Error('state-dir must be an exiting directory'); + } + + const exportDir = pathResolve( + /** @type {string} */ (processValue.getFlag('export-dir', '.')), + ); + + const includeExportData = processValue.getBoolean({ + flagName: 'include-export-data', + }); + const exportMode = processValue.getFlag('export-mode'); + checkExportMode(exportMode); + + const checkBlockHeight = processValue.getInteger({ + flagName: 'check-block-height', + }); + + const { registerShutdown } = makeShutdown(false); + + const exporter = initiateSwingStoreExport( + { + stateDir, + exportDir, + blockHeight: checkBlockHeight, + exportMode, + includeExportData, + }, + { + fs, + pathResolve, + log: console.log, + }, + ); + + registerShutdown(() => exporter.stop()); + + await exporter.onDone(); +}; + +if (isEntrypoint(import.meta.url)) { + main(process.argv.splice(2), { + homedir: os.homedir(), + env: process.env, + console, + fs: fsPower, + pathResolve: pathPower.resolve, + }).then( + _res => 0, + rej => { + console.error(`error running export-kernel-db:`, rej); + process.exit(process.exitCode || rej.exitCode || 1); + }, + ); +} diff --git a/packages/cosmic-swingset/src/helpers/process-value.js b/packages/cosmic-swingset/src/helpers/process-value.js index c10e48e45c2..d55c2033463 100644 --- a/packages/cosmic-swingset/src/helpers/process-value.js +++ b/packages/cosmic-swingset/src/helpers/process-value.js @@ -1,3 +1,5 @@ +// @ts-check + import { resolve as pathResolve } from 'path'; export const makeProcessValue = ({ env, args }) => { @@ -18,8 +20,12 @@ export const makeProcessValue = ({ env, args }) => { for (let i = 0; i < args.length; i += 1) { const arg = args[i]; if (arg === flag) { - i += 1; - flagValue = args[i]; + if (args.length > i + 1 && !args[i + 1].startsWith('--')) { + i += 1; + flagValue = args[i]; + } else { + flagValue = ''; + } } else if (arg.startsWith(flagEquals)) { flagValue = arg.substr(flagEquals.length); } @@ -31,15 +37,15 @@ export const makeProcessValue = ({ env, args }) => { * @param {object} options * @param {string} [options.envName] * @param {string} [options.flagName] - * @param {string} options.trueValue - * @returns {string | undefined} + * @param {boolean} [options.emptyFlagIsTrue] + * @returns {boolean | undefined} */ - const getPath = ({ envName, flagName, trueValue }) => { + const getBoolean = ({ envName, flagName, emptyFlagIsTrue = true }) => { let option; - if (envName) { - option = env[envName]; - } else if (flagName) { + if (flagName) { option = getFlag(flagName); + } else if (envName) { + option = env[envName]; } else { return undefined; } @@ -48,21 +54,80 @@ export const makeProcessValue = ({ env, args }) => { case '0': case 'false': case false: - return undefined; + return false; case '1': case 'true': case true: - return trueValue; + return true; default: - if (option) { - return pathResolve(option); - } else if (envName && flagName) { - return getPath({ flagName, trueValue }); - } else { - return undefined; + if (option === '' && flagName && emptyFlagIsTrue) { + return true; + } + if (option === undefined && envName && flagName) { + return getBoolean({ envName, emptyFlagIsTrue }); } + return undefined; + } + }; + + /** + * @param {object} options + * @param {string} [options.envName] + * @param {string} [options.flagName] + * @returns {number | undefined} + */ + const getInteger = ({ envName, flagName }) => { + let option; + if (flagName) { + option = getFlag(flagName); + } else if (envName) { + option = env[envName]; + } else { + return undefined; + } + + if (option) { + const value = Number.parseInt(option, 10); + return String(value) === option ? value : undefined; + } else if (option === undefined && flagName && envName) { + return getInteger({ envName }); + } + + return undefined; + }; + + /** + * @param {object} options + * @param {string} [options.envName] + * @param {string} [options.flagName] + * @param {string} [options.trueValue] + * @returns {string | undefined} + */ + const getPath = ({ envName, flagName, trueValue }) => { + if (trueValue !== undefined) { + const boolValue = getBoolean({ envName, flagName }); + if (boolValue !== undefined) { + return boolValue ? trueValue : undefined; + } + } + + let option; + if (flagName) { + option = getFlag(flagName); + } else if (envName) { + option = env[envName]; + } else { + return undefined; + } + + if (option) { + return pathResolve(option); + } else if (trueValue && envName && flagName) { + return getPath({ envName, trueValue }); + } else { + return undefined; } }; - return harden({ getFlag, getPath }); + return harden({ getFlag, getBoolean, getInteger, getPath }); }; From 22bc1d54a7582d9490959dfe204838293412b537 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 19 Feb 2023 01:06:39 +0000 Subject: [PATCH 10/20] feat(cosmic-swingset): wire snapshot taking in chain-main --- packages/cosmic-swingset/src/chain-main.js | 127 ++++++++++++++++++- packages/cosmic-swingset/src/launch-chain.js | 2 +- packages/cosmic-swingset/src/sim-chain.js | 2 +- 3 files changed, 125 insertions(+), 6 deletions(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index fe74534be59..a67e7cbf77e 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -4,8 +4,11 @@ import { resolve as pathResolve } from 'path'; import v8 from 'node:v8'; import process from 'node:process'; import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; import { performance } from 'perf_hooks'; import { resolve as importMetaResolve } from 'import-meta-resolve'; +import tmpfs from 'tmp'; + import { E } from '@endo/far'; import engineGC from '@agoric/swingset-vat/src/lib-nodejs/engine-gc.js'; import { waitUntilQuiescent } from '@agoric/swingset-vat/src/lib-nodejs/waitUntilQuiescent.js'; @@ -35,6 +38,7 @@ import stringify from './helpers/json-stable-stringify.js'; import { launch } from './launch-chain.js'; import { getTelemetryProviders } from './kernel-stats.js'; import { makeProcessValue } from './helpers/process-value.js'; +import { initiateSwingStoreExport } from './export-kernel-db.js'; // eslint-disable-next-line no-unused-vars let whenHellFreezesOver = null; @@ -174,6 +178,23 @@ export default async function main(progname, args, { env, homedir, agcc }) { whenHellFreezesOver = new Promise(() => {}); agcc.runAgCosmosDaemon(nodePort, fromGo, [progname, ...args]); + /** + * @type {undefined | { + * blockHeight: number, + * exporter?: import('./export-kernel-db.js').StateSyncExporter, + * exportDir?: string, + * cleanup?: () => Promise, + * }} + */ + let stateSyncExport; + + async function discardStateSyncExport() { + const exportData = stateSyncExport; + stateSyncExport = undefined; + await exportData?.exporter?.stop(); + await exportData?.cleanup?.(); + } + let savedChainSends = []; // Send a chain downcall, recording what we sent and received. @@ -183,7 +204,10 @@ export default async function main(progname, args, { env, homedir, agcc }) { return ret; } - const clearChainSends = () => { + const clearChainSends = async () => { + // Cosmos should have blocked before calling commit, but wait just in case + await stateSyncExport?.exporter?.onStarted().catch(() => {}); + const chainSends = savedChainSends; savedChainSends = []; return chainSends; @@ -445,13 +469,108 @@ export default async function main(progname, args, { env, homedir, agcc }) { let shutdown; ({ blockingSend, shutdown, writeSlogObject, savedChainSends } = s); - registerShutdown(shutdown); + registerShutdown(async () => + Promise.all([shutdown(), discardStateSyncExport()]).then(() => {}), + ); return blockingSend; } - async function handleCosmosSnapshot(_blockHeight, _request, _requestArgs) { - throw Fail`Not implemented`; + async function handleCosmosSnapshot(blockHeight, request, _requestArgs) { + switch (request) { + case 'initiate': { + !stateSyncExport || + Fail`Snapshot already in progress for ${stateSyncExport.blockHeight}`; + + const exportData = + /** @type {Required>} */ ({ + blockHeight, + }); + stateSyncExport = exportData; + + // eslint-disable-next-line @jessie.js/no-nested-await + await new Promise((resolve, reject) => { + tmpfs.dir( + { + prefix: `agd-state-sync-${blockHeight}-`, + unsafeCleanup: true, + }, + (err, exportDir, cleanup) => { + if (err) { + reject(err); + return; + } + exportData.exportDir = exportDir; + /** @type {Promise | undefined} */ + let cleanupResult; + exportData.cleanup = async () => { + cleanupResult ||= new Promise(cleanupDone => { + // If the exporter is still the same, then the retriever + // is in charge of cleanups + if (stateSyncExport !== exportData) { + // @ts-expect-error wrong type definitions + cleanup(cleanupDone); + } else { + console.warn('unexpected call of state-sync cleanup'); + cleanupDone(); + } + }); + await cleanupResult; + }; + resolve(null); + }, + ); + }); + + exportData.exporter = initiateSwingStoreExport( + { + stateDir: stateDBDir, + exportDir: exportData.exportDir, + blockHeight, + }, + { + fs: fsPromises, + pathResolve, + }, + ); + + exportData.exporter.onDone().catch(() => { + if (exportData === stateSyncExport) { + stateSyncExport = undefined; + } + exportData.cleanup(); + }); + + return exportData.exporter.onStarted().catch(err => { + console.warn( + `State-sync export failed for block ${blockHeight}`, + err, + ); + throw err; + }); + } + case 'discard': { + return discardStateSyncExport(); + } + case 'retrieve': { + const exportData = stateSyncExport; + if (!exportData || !exportData.exporter) { + throw Fail`No snapshot in progress`; + } + + return exportData.exporter.onDone().then(() => { + if (exportData === stateSyncExport) { + // Don't cleanup, cosmos is now in charge + stateSyncExport = undefined; + return exportData.exportDir; + } else { + throw Fail`Snapshot was discarded`; + } + }); + } + default: + throw Fail`Unknown cosmos snapshot request ${request}`; + } } async function toSwingSet(action, _replier) { diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 480b2dde981..cc1c4cb0688 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -310,7 +310,7 @@ export async function launch({ } async function saveOutsideState(blockHeight) { - const chainSends = clearChainSends(); + const chainSends = await clearChainSends(); kvStore.set(getHostKey('height'), `${blockHeight}`); kvStore.set(getHostKey('chainSends'), JSON.stringify(chainSends)); diff --git a/packages/cosmic-swingset/src/sim-chain.js b/packages/cosmic-swingset/src/sim-chain.js index accf8bbb0dd..66bb3206853 100644 --- a/packages/cosmic-swingset/src/sim-chain.js +++ b/packages/cosmic-swingset/src/sim-chain.js @@ -80,7 +80,7 @@ export async function connectToFakeChain(basedir, GCI, delay, inbound) { function replayChainSends() { Fail`Replay not implemented`; } - function clearChainSends() { + async function clearChainSends() { return []; } From 05e2b15a3f5b749ae95d0e1f3eb96fc0ec0d7467 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Tue, 7 Mar 2023 22:42:31 +0000 Subject: [PATCH 11/20] feat(cosmic-swingset): execute export in a subprocess --- packages/cosmic-swingset/src/chain-main.js | 9 +- .../cosmic-swingset/src/export-kernel-db.js | 135 +++++++++++++++++- 2 files changed, 137 insertions(+), 7 deletions(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index a67e7cbf77e..e8b09e3a81a 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -4,10 +4,10 @@ import { resolve as pathResolve } from 'path'; import v8 from 'node:v8'; import process from 'node:process'; import fs from 'node:fs'; -import fsPromises from 'node:fs/promises'; import { performance } from 'perf_hooks'; import { resolve as importMetaResolve } from 'import-meta-resolve'; import tmpfs from 'tmp'; +import { fork } from 'node:child_process'; import { E } from '@endo/far'; import engineGC from '@agoric/swingset-vat/src/lib-nodejs/engine-gc.js'; @@ -38,7 +38,7 @@ import stringify from './helpers/json-stable-stringify.js'; import { launch } from './launch-chain.js'; import { getTelemetryProviders } from './kernel-stats.js'; import { makeProcessValue } from './helpers/process-value.js'; -import { initiateSwingStoreExport } from './export-kernel-db.js'; +import { spawnSwingStoreExport } from './export-kernel-db.js'; // eslint-disable-next-line no-unused-vars let whenHellFreezesOver = null; @@ -522,15 +522,14 @@ export default async function main(progname, args, { env, homedir, agcc }) { ); }); - exportData.exporter = initiateSwingStoreExport( + exportData.exporter = spawnSwingStoreExport( { stateDir: stateDBDir, exportDir: exportData.exportDir, blockHeight, }, { - fs: fsPromises, - pathResolve, + fork, }, ); diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js index 567de6c4229..bcc318a7ccc 100755 --- a/packages/cosmic-swingset/src/export-kernel-db.js +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -7,6 +7,7 @@ import os from 'os'; import process from 'process'; import fsPower from 'fs/promises'; import pathPower from 'path'; +import { fileURLToPath } from 'url'; import { makePromiseKit } from '@endo/promise-kit'; import { Fail } from '@agoric/assert'; @@ -207,18 +208,25 @@ export const initiateSwingStoreExport = ( }; }; +/** + * @typedef {{type: 'started', blockHeight: number}} ExportMessageStarted + * @typedef {{type: 'done', error?: Error}} ExportMessageDone + * @typedef {ExportMessageStarted | ExportMessageDone} ExportMessage + */ + /** * @param {string[]} args * @param {object} powers * @param {Partial>} powers.env * @param {string} powers.homedir + * @param {((msg: ExportMessage) => void) | null} powers.send * @param {Console} powers.console * @param {import('fs/promises')} powers.fs * @param {import('path')['resolve']} powers.pathResolve */ export const main = async ( args, - { env, homedir, console, fs, pathResolve }, + { env, homedir, send, console, fs, pathResolve }, ) => { const processValue = makeProcessValue({ env, args }); @@ -268,13 +276,136 @@ export const main = async ( registerShutdown(() => exporter.stop()); - await exporter.onDone(); + exporter.onStarted().then( + () => + send?.({ + type: 'started', + blockHeight: /** @type {number} */ (exporter.getBlockHeight()), + }), + () => {}, + ); + + await exporter.onDone().then( + () => send?.({ type: 'done' }), + error => { + if (send) { + send({ type: 'done', error }); + } else { + throw error; + } + }, + ); +}; + +/** + * @param {StateSyncExporterOptions} options + * @param {object} powers + * @param {typeof import('child_process')['fork']} powers.fork + * @returns {StateSyncExporter} + */ +export const spawnSwingStoreExport = ( + { stateDir, exportDir, blockHeight, exportMode, includeExportData }, + { fork }, +) => { + const args = ['--state-dir', stateDir, '--export-dir', exportDir]; + + if (blockHeight !== undefined) { + args.push('--check-block-height', String(blockHeight)); + } + + if (exportMode) { + args.push('--export-mode', exportMode); + } + + if (includeExportData) { + args.push('--include-export-data'); + } + + const cp = fork(fileURLToPath(import.meta.url), args, { + serialization: 'advanced', // To get error objects serialized + }); + + const kits = harden({ + /** @type {import('@endo/promise-kit').PromiseKit} */ + started: makePromiseKit(), + /** @type {import('@endo/promise-kit').PromiseKit} */ + done: makePromiseKit(), + }); + + let exited = false; + + /** + * @param {number | null} code + * @param {NodeJS.Signals | null} signal + */ + const onExit = (code, signal) => { + exited = true; + kits.done.reject( + new Error(`Process exited before done. code=${code}, signal=${signal}`), + ); + }; + + /** @type {number | undefined} */ + let exportBlockHeight; + + /** @param {import('./export-kernel-db.js').ExportMessage} msg */ + const onMessage = msg => { + switch (msg.type) { + case 'started': { + exportBlockHeight = msg.blockHeight; + kits.started.resolve(); + break; + } + case 'done': { + if (msg.error) { + kits.done.reject(msg.error); + } else { + kits.done.resolve(); + } + break; + } + default: { + // @ts-expect-error exhaustive check + Fail`Unexpected ${msg.type} message`; + } + } + }; + + cp.on('error', kits.done.reject).on('exit', onExit).on('message', onMessage); + + kits.done.promise + .catch(err => { + kits.started.reject(err); + }) + .then(() => { + cp.off('error', kits.done.reject) + .off('exit', onExit) + .off('message', onMessage); + }); + + if (cp.exitCode != null) { + onExit(cp.exitCode, null); + } + return harden({ + getBlockHeight: () => blockHeight || exportBlockHeight, + onStarted: async () => kits.started.promise, + onDone: async () => kits.done.promise, + stop: async () => { + if (!exited) { + cp.kill(); + } + return kits.done.promise; + }, + }); }; if (isEntrypoint(import.meta.url)) { main(process.argv.splice(2), { homedir: os.homedir(), env: process.env, + send: process.send + ? Function.prototype.bind.call(process.send, process) + : undefined, console, fs: fsPower, pathResolve: pathPower.resolve, From a2dabd1672c580e1f421336f2ab34e2694ed5557 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 10 Mar 2023 23:00:44 +0000 Subject: [PATCH 12/20] feat(cosmic-swingset): explicit verbose option for export db --- packages/cosmic-swingset/src/export-kernel-db.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js index bcc318a7ccc..3464dd4da90 100755 --- a/packages/cosmic-swingset/src/export-kernel-db.js +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -257,6 +257,10 @@ export const main = async ( flagName: 'check-block-height', }); + const verbose = processValue.getBoolean({ + flagName: 'verbose', + }); + const { registerShutdown } = makeShutdown(false); const exporter = initiateSwingStoreExport( @@ -270,7 +274,7 @@ export const main = async ( { fs, pathResolve, - log: console.log, + log: verbose ? console.log : undefined, }, ); @@ -288,6 +292,7 @@ export const main = async ( await exporter.onDone().then( () => send?.({ type: 'done' }), error => { + verbose && console.error('state-sync export failed', error); if (send) { send({ type: 'done', error }); } else { @@ -301,11 +306,12 @@ export const main = async ( * @param {StateSyncExporterOptions} options * @param {object} powers * @param {typeof import('child_process')['fork']} powers.fork + * @param {boolean} [powers.verbose] * @returns {StateSyncExporter} */ export const spawnSwingStoreExport = ( { stateDir, exportDir, blockHeight, exportMode, includeExportData }, - { fork }, + { fork, verbose }, ) => { const args = ['--state-dir', stateDir, '--export-dir', exportDir]; @@ -321,6 +327,10 @@ export const spawnSwingStoreExport = ( args.push('--include-export-data'); } + if (verbose) { + args.push('--verbose'); + } + const cp = fork(fileURLToPath(import.meta.url), args, { serialization: 'advanced', // To get error objects serialized }); From 4cc25f56ba9e967039c2dff2cbb566eafb37aaea Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 24 Feb 2023 20:59:31 +0000 Subject: [PATCH 13/20] feat(cosmic-proto): add state-sync artifacts proto --- .../proto/agoric/swingset/swingset.proto | 14 + golang/cosmos/x/swingset/types/swingset.pb.go | 331 +++++++++++++++--- .../dist/agoric/swingset/swingset.d.ts | 30 ++ .../dist/agoric/swingset/swingset.js | 57 +++ packages/cosmic-proto/package.json | 4 +- 5 files changed, 383 insertions(+), 53 deletions(-) diff --git a/golang/cosmos/proto/agoric/swingset/swingset.proto b/golang/cosmos/proto/agoric/swingset/swingset.proto index 592646fb5c7..991738b9569 100644 --- a/golang/cosmos/proto/agoric/swingset/swingset.proto +++ b/golang/cosmos/proto/agoric/swingset/swingset.proto @@ -149,3 +149,17 @@ message Egress { (gogoproto.moretags) = "yaml:\"powerFlags\"" ]; } + +// The payload messages used by swingset state-sync +message ExtensionSnapshotterArtifactPayload { + option (gogoproto.equal) = false; + string name = 1 [ + (gogoproto.jsontag) = "name", + (gogoproto.moretags) = "yaml:\"name\"" + ]; + + bytes data = 2 [ + (gogoproto.jsontag) = "data", + (gogoproto.moretags) = "yaml:\"data\"" + ]; +} diff --git a/golang/cosmos/x/swingset/types/swingset.pb.go b/golang/cosmos/x/swingset/types/swingset.pb.go index 23fbb38244a..940e367d844 100644 --- a/golang/cosmos/x/swingset/types/swingset.pb.go +++ b/golang/cosmos/x/swingset/types/swingset.pb.go @@ -230,6 +230,7 @@ func (m *Params) GetQueueMax() []QueueSize { return nil } +// The current state of the module. type State struct { // The allowed number of items to add to queues, as determined by SwingSet. // Transactions which attempt to enqueue more should be rejected. @@ -494,6 +495,59 @@ func (m *Egress) GetPowerFlags() []string { return nil } +// The payload messages used by swingset state-sync +type ExtensionSnapshotterArtifactPayload struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name" yaml:"name"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data" yaml:"data"` +} + +func (m *ExtensionSnapshotterArtifactPayload) Reset() { *m = ExtensionSnapshotterArtifactPayload{} } +func (m *ExtensionSnapshotterArtifactPayload) String() string { return proto.CompactTextString(m) } +func (*ExtensionSnapshotterArtifactPayload) ProtoMessage() {} +func (*ExtensionSnapshotterArtifactPayload) Descriptor() ([]byte, []int) { + return fileDescriptor_ff9c341e0de15f8b, []int{8} +} +func (m *ExtensionSnapshotterArtifactPayload) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ExtensionSnapshotterArtifactPayload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ExtensionSnapshotterArtifactPayload.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ExtensionSnapshotterArtifactPayload) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExtensionSnapshotterArtifactPayload.Merge(m, src) +} +func (m *ExtensionSnapshotterArtifactPayload) XXX_Size() int { + return m.Size() +} +func (m *ExtensionSnapshotterArtifactPayload) XXX_DiscardUnknown() { + xxx_messageInfo_ExtensionSnapshotterArtifactPayload.DiscardUnknown(m) +} + +var xxx_messageInfo_ExtensionSnapshotterArtifactPayload proto.InternalMessageInfo + +func (m *ExtensionSnapshotterArtifactPayload) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ExtensionSnapshotterArtifactPayload) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + func init() { proto.RegisterType((*CoreEvalProposal)(nil), "agoric.swingset.CoreEvalProposal") proto.RegisterType((*CoreEval)(nil), "agoric.swingset.CoreEval") @@ -503,62 +557,67 @@ func init() { proto.RegisterType((*PowerFlagFee)(nil), "agoric.swingset.PowerFlagFee") proto.RegisterType((*QueueSize)(nil), "agoric.swingset.QueueSize") proto.RegisterType((*Egress)(nil), "agoric.swingset.Egress") + proto.RegisterType((*ExtensionSnapshotterArtifactPayload)(nil), "agoric.swingset.ExtensionSnapshotterArtifactPayload") } func init() { proto.RegisterFile("agoric/swingset/swingset.proto", fileDescriptor_ff9c341e0de15f8b) } var fileDescriptor_ff9c341e0de15f8b = []byte{ - // 790 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x54, 0xcd, 0x6b, 0x3b, 0x45, - 0x18, 0xce, 0x9a, 0x0f, 0x93, 0x49, 0x7e, 0x6d, 0x1d, 0x0b, 0xc6, 0x62, 0x77, 0xca, 0x5e, 0x2c, - 0x48, 0x77, 0x5b, 0x45, 0x84, 0x14, 0x0f, 0xd9, 0x90, 0x52, 0x10, 0x25, 0x6e, 0xa9, 0x07, 0x51, - 0x96, 0xc9, 0x66, 0xb2, 0x4e, 0xbb, 0xd9, 0xd9, 0xee, 0x4c, 0xd2, 0x8f, 0x7f, 0x40, 0x2f, 0x82, - 0x78, 0xf2, 0xd8, 0xb3, 0x7f, 0x49, 0x8f, 0x3d, 0x8a, 0x87, 0x55, 0xd2, 0x8b, 0xf4, 0x98, 0xa3, - 0x20, 0xc8, 0xcc, 0x6c, 0xb6, 0xc1, 0x0a, 0xf6, 0xe2, 0x69, 0x67, 0xde, 0x8f, 0xe7, 0x7d, 0xde, - 0xf7, 0x99, 0x7d, 0x81, 0x89, 0x43, 0x96, 0xd2, 0xc0, 0xe1, 0x97, 0x34, 0x0e, 0x39, 0x11, 0xc5, - 0xc1, 0x4e, 0x52, 0x26, 0x18, 0x5c, 0xd7, 0x7e, 0x7b, 0x69, 0xde, 0xda, 0x0c, 0x59, 0xc8, 0x94, - 0xcf, 0x91, 0x27, 0x1d, 0xb6, 0x65, 0x06, 0x8c, 0x4f, 0x18, 0x77, 0x86, 0x98, 0x13, 0x67, 0x76, - 0x30, 0x24, 0x02, 0x1f, 0x38, 0x01, 0xa3, 0xb1, 0xf6, 0x5b, 0xdf, 0x1a, 0x60, 0xa3, 0xc7, 0x52, - 0xd2, 0x9f, 0xe1, 0x68, 0x90, 0xb2, 0x84, 0x71, 0x1c, 0xc1, 0x4d, 0x50, 0x15, 0x54, 0x44, 0xa4, - 0x6d, 0xec, 0x18, 0xbb, 0x0d, 0x4f, 0x5f, 0xe0, 0x0e, 0x68, 0x8e, 0x08, 0x0f, 0x52, 0x9a, 0x08, - 0xca, 0xe2, 0xf6, 0x6b, 0xca, 0xb7, 0x6a, 0x82, 0x1f, 0x82, 0x2a, 0x99, 0xe1, 0x88, 0xb7, 0xcb, - 0x3b, 0xe5, 0xdd, 0xe6, 0xfb, 0x6f, 0xdb, 0xff, 0xe0, 0x68, 0x2f, 0x2b, 0xb9, 0x95, 0xbb, 0x0c, - 0x95, 0x3c, 0x1d, 0xdd, 0xa9, 0x7c, 0x77, 0x8b, 0x4a, 0x16, 0x07, 0xf5, 0xa5, 0x1b, 0x76, 0x40, - 0xeb, 0x8c, 0xb3, 0xd8, 0x4f, 0x48, 0x3a, 0xa1, 0x82, 0x6b, 0x1e, 0xee, 0x5b, 0x8b, 0x0c, 0xbd, - 0x79, 0x8d, 0x27, 0x51, 0xc7, 0x5a, 0xf5, 0x5a, 0x5e, 0x53, 0x5e, 0x07, 0xfa, 0x06, 0xdf, 0x03, - 0xaf, 0x9f, 0x71, 0x3f, 0x60, 0x23, 0xa2, 0x29, 0xba, 0x70, 0x91, 0xa1, 0xb5, 0x65, 0x9a, 0x72, - 0x58, 0x5e, 0xed, 0x8c, 0xf7, 0xe4, 0xe1, 0xfb, 0x32, 0xa8, 0x0d, 0x70, 0x8a, 0x27, 0x1c, 0x1e, - 0x83, 0xb5, 0x21, 0xc1, 0x31, 0x97, 0xb0, 0xfe, 0x34, 0xa6, 0xa2, 0x6d, 0xa8, 0x2e, 0xde, 0x79, - 0xd6, 0xc5, 0x89, 0x48, 0x69, 0x1c, 0xba, 0x32, 0x38, 0x6f, 0xa4, 0xa5, 0x32, 0x07, 0x24, 0x3d, - 0x8d, 0xa9, 0x80, 0x17, 0x60, 0x6d, 0x4c, 0x88, 0xc2, 0xf0, 0x93, 0x94, 0x06, 0x92, 0x88, 0x9e, - 0x87, 0x16, 0xc3, 0x96, 0x62, 0xd8, 0xb9, 0x18, 0x76, 0x8f, 0xd1, 0xd8, 0xdd, 0x97, 0x30, 0x3f, - 0xff, 0x86, 0x76, 0x43, 0x2a, 0xbe, 0x99, 0x0e, 0xed, 0x80, 0x4d, 0x9c, 0x5c, 0x39, 0xfd, 0xd9, - 0xe3, 0xa3, 0x73, 0x47, 0x5c, 0x27, 0x84, 0xab, 0x04, 0xee, 0xb5, 0xc6, 0x84, 0xc8, 0x6a, 0x03, - 0x59, 0x00, 0xee, 0x83, 0xcd, 0x21, 0x63, 0x82, 0x8b, 0x14, 0x27, 0xfe, 0x0c, 0x0b, 0x3f, 0x60, - 0xf1, 0x98, 0x86, 0xed, 0xb2, 0x12, 0x09, 0x16, 0xbe, 0x2f, 0xb0, 0xe8, 0x29, 0x0f, 0xfc, 0x04, - 0xac, 0x27, 0xec, 0x92, 0xa4, 0xfe, 0x38, 0xc2, 0xa1, 0x3f, 0x26, 0x84, 0xb7, 0x2b, 0x8a, 0xe5, - 0xf6, 0xb3, 0x7e, 0x07, 0x32, 0xee, 0x28, 0xc2, 0xe1, 0x11, 0x21, 0x79, 0xc3, 0xaf, 0x92, 0x15, - 0x1b, 0x87, 0x1f, 0x83, 0xc6, 0xc5, 0x94, 0x4c, 0x89, 0x3f, 0xc1, 0x57, 0xed, 0xaa, 0x82, 0xd9, - 0x7a, 0x06, 0xf3, 0xb9, 0x8c, 0x38, 0xa1, 0x37, 0x4b, 0x8c, 0xba, 0x4a, 0xf9, 0x14, 0x5f, 0x75, - 0xea, 0x3f, 0xdd, 0xa2, 0xd2, 0x1f, 0xb7, 0xc8, 0xb0, 0x3e, 0x03, 0xd5, 0x13, 0x81, 0x05, 0x81, - 0x7d, 0xf0, 0x4a, 0x23, 0xe2, 0x28, 0x62, 0x97, 0x64, 0x94, 0x8b, 0xf1, 0xdf, 0xa8, 0x2d, 0x95, - 0xd6, 0xd5, 0x59, 0x56, 0x04, 0x9a, 0x2b, 0x6a, 0xc1, 0x0d, 0x50, 0x3e, 0x27, 0xd7, 0xf9, 0xb3, - 0x96, 0x47, 0xd8, 0x07, 0x55, 0xa5, 0x5d, 0xfe, 0x56, 0x1c, 0x89, 0xf1, 0x6b, 0x86, 0xde, 0x7d, - 0x81, 0x0e, 0xa7, 0x34, 0x16, 0x9e, 0xce, 0xee, 0x54, 0x14, 0xfb, 0x1f, 0x0d, 0xd0, 0x5a, 0x1d, - 0x16, 0xdc, 0x06, 0xe0, 0x69, 0xc8, 0x79, 0xd9, 0x46, 0x31, 0x3a, 0xf8, 0x35, 0x28, 0x8f, 0xc9, - 0xff, 0xf2, 0x3a, 0x24, 0x6e, 0x4e, 0xea, 0x23, 0xd0, 0x28, 0x66, 0xf4, 0x2f, 0x03, 0x80, 0xa0, - 0xc2, 0xe9, 0x8d, 0xfe, 0x57, 0xaa, 0x9e, 0x3a, 0xe7, 0x89, 0x7f, 0x19, 0xa0, 0xd6, 0x0f, 0x53, - 0xc2, 0x39, 0x3c, 0x04, 0xf5, 0x98, 0x06, 0xe7, 0x31, 0x9e, 0xe4, 0x3b, 0xc1, 0x45, 0x8f, 0x19, - 0x2a, 0x6c, 0x8b, 0x0c, 0xad, 0xeb, 0x1f, 0x6c, 0x69, 0xb1, 0xbc, 0xc2, 0x09, 0xbf, 0x02, 0x95, - 0x84, 0x90, 0x54, 0x55, 0x68, 0xb9, 0xc7, 0x8f, 0x19, 0x52, 0xf7, 0x45, 0x86, 0x9a, 0x3a, 0x49, - 0xde, 0xac, 0x3f, 0x33, 0xb4, 0xf7, 0x82, 0xf6, 0xba, 0x41, 0xd0, 0x1d, 0x8d, 0x24, 0x29, 0x4f, - 0xa1, 0x40, 0x0f, 0x34, 0x9f, 0x46, 0xac, 0x37, 0x4f, 0xc3, 0x3d, 0x98, 0x67, 0x08, 0x14, 0x4a, - 0xf0, 0xc7, 0x0c, 0x81, 0x62, 0xea, 0x7c, 0x91, 0xa1, 0x37, 0xf2, 0xc2, 0x85, 0xcd, 0xf2, 0x56, - 0x02, 0x54, 0xff, 0x25, 0xf7, 0xf4, 0x6e, 0x6e, 0x1a, 0xf7, 0x73, 0xd3, 0xf8, 0x7d, 0x6e, 0x1a, - 0x3f, 0x3c, 0x98, 0xa5, 0xfb, 0x07, 0xb3, 0xf4, 0xcb, 0x83, 0x59, 0xfa, 0xf2, 0x70, 0x85, 0x68, - 0x57, 0xaf, 0x69, 0xfd, 0x2c, 0x15, 0xd1, 0x90, 0x45, 0x38, 0x0e, 0x97, 0x1d, 0x5c, 0x3d, 0x6d, - 0x70, 0xd5, 0xc1, 0xb0, 0xa6, 0x16, 0xef, 0x07, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x71, 0x8e, - 0x36, 0xaa, 0xe1, 0x05, 0x00, 0x00, + // 858 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0xbd, 0x6f, 0x23, 0x45, + 0x14, 0xf7, 0x62, 0x3b, 0xc4, 0xcf, 0xbe, 0xe4, 0x18, 0x22, 0x9d, 0x89, 0x38, 0x4f, 0xb4, 0x14, + 0x44, 0x3a, 0x9d, 0x7d, 0x01, 0x21, 0x24, 0x9f, 0x28, 0xbc, 0x91, 0x4f, 0x27, 0x21, 0x90, 0xd9, + 0x28, 0x14, 0x08, 0xb4, 0x1a, 0xaf, 0xc7, 0x7b, 0x93, 0xac, 0x67, 0xf6, 0x66, 0x26, 0x5f, 0xd7, + 0x23, 0x68, 0x90, 0x10, 0x15, 0x65, 0x6a, 0xfe, 0x92, 0x2b, 0xaf, 0x44, 0x14, 0x0b, 0x4a, 0x1a, + 0x94, 0xd2, 0x25, 0x12, 0x12, 0x9a, 0x99, 0xf5, 0xc6, 0x22, 0x48, 0xa4, 0xb9, 0x6a, 0xdf, 0xe7, + 0xef, 0xbd, 0xf7, 0x7b, 0x33, 0x3b, 0xd0, 0x21, 0x89, 0x90, 0x2c, 0xee, 0xa9, 0x13, 0xc6, 0x13, + 0x45, 0x75, 0x29, 0x74, 0x33, 0x29, 0xb4, 0x40, 0xeb, 0xce, 0xdf, 0x5d, 0x98, 0x37, 0x37, 0x12, + 0x91, 0x08, 0xeb, 0xeb, 0x19, 0xc9, 0x85, 0x6d, 0x76, 0x62, 0xa1, 0x66, 0x42, 0xf5, 0xc6, 0x44, + 0xd1, 0xde, 0xf1, 0xce, 0x98, 0x6a, 0xb2, 0xd3, 0x8b, 0x05, 0xe3, 0xce, 0xef, 0x7f, 0xe7, 0xc1, + 0xdd, 0x5d, 0x21, 0xe9, 0xf0, 0x98, 0xa4, 0x23, 0x29, 0x32, 0xa1, 0x48, 0x8a, 0x36, 0xa0, 0xae, + 0x99, 0x4e, 0x69, 0xdb, 0xdb, 0xf2, 0xb6, 0x1b, 0xa1, 0x53, 0xd0, 0x16, 0x34, 0x27, 0x54, 0xc5, + 0x92, 0x65, 0x9a, 0x09, 0xde, 0x7e, 0xc3, 0xfa, 0x96, 0x4d, 0xe8, 0x23, 0xa8, 0xd3, 0x63, 0x92, + 0xaa, 0x76, 0x75, 0xab, 0xba, 0xdd, 0xfc, 0xe0, 0x9d, 0xee, 0xbf, 0x7a, 0xec, 0x2e, 0x2a, 0x05, + 0xb5, 0x97, 0x39, 0xae, 0x84, 0x2e, 0xba, 0x5f, 0xfb, 0xfe, 0x1c, 0x57, 0x7c, 0x05, 0xab, 0x0b, + 0x37, 0xea, 0x43, 0xeb, 0x40, 0x09, 0x1e, 0x65, 0x54, 0xce, 0x98, 0x56, 0xae, 0x8f, 0xe0, 0xde, + 0x3c, 0xc7, 0x6f, 0x9f, 0x91, 0x59, 0xda, 0xf7, 0x97, 0xbd, 0x7e, 0xd8, 0x34, 0xea, 0xc8, 0x69, + 0xe8, 0x01, 0xbc, 0x79, 0xa0, 0xa2, 0x58, 0x4c, 0xa8, 0x6b, 0x31, 0x40, 0xf3, 0x1c, 0xaf, 0x2d, + 0xd2, 0xac, 0xc3, 0x0f, 0x57, 0x0e, 0xd4, 0xae, 0x11, 0x7e, 0xa8, 0xc2, 0xca, 0x88, 0x48, 0x32, + 0x53, 0xe8, 0x29, 0xac, 0x8d, 0x29, 0xe1, 0xca, 0xc0, 0x46, 0x47, 0x9c, 0xe9, 0xb6, 0x67, 0xa7, + 0x78, 0xf7, 0xc6, 0x14, 0x7b, 0x5a, 0x32, 0x9e, 0x04, 0x26, 0xb8, 0x18, 0xa4, 0x65, 0x33, 0x47, + 0x54, 0xee, 0x73, 0xa6, 0xd1, 0x73, 0x58, 0x9b, 0x52, 0x6a, 0x31, 0xa2, 0x4c, 0xb2, 0xd8, 0x34, + 0xe2, 0xf8, 0x70, 0xcb, 0xe8, 0x9a, 0x65, 0x74, 0x8b, 0x65, 0x74, 0x77, 0x05, 0xe3, 0xc1, 0x23, + 0x03, 0xf3, 0xcb, 0xef, 0x78, 0x3b, 0x61, 0xfa, 0xd9, 0xd1, 0xb8, 0x1b, 0x8b, 0x59, 0xaf, 0xd8, + 0x9c, 0xfb, 0x3c, 0x54, 0x93, 0xc3, 0x9e, 0x3e, 0xcb, 0xa8, 0xb2, 0x09, 0x2a, 0x6c, 0x4d, 0x29, + 0x35, 0xd5, 0x46, 0xa6, 0x00, 0x7a, 0x04, 0x1b, 0x63, 0x21, 0xb4, 0xd2, 0x92, 0x64, 0xd1, 0x31, + 0xd1, 0x51, 0x2c, 0xf8, 0x94, 0x25, 0xed, 0xaa, 0x5d, 0x12, 0x2a, 0x7d, 0x5f, 0x12, 0xbd, 0x6b, + 0x3d, 0xe8, 0x53, 0x58, 0xcf, 0xc4, 0x09, 0x95, 0xd1, 0x34, 0x25, 0x49, 0x34, 0xa5, 0x54, 0xb5, + 0x6b, 0xb6, 0xcb, 0xfb, 0x37, 0xe6, 0x1d, 0x99, 0xb8, 0x27, 0x29, 0x49, 0x9e, 0x50, 0x5a, 0x0c, + 0x7c, 0x27, 0x5b, 0xb2, 0x29, 0xf4, 0x09, 0x34, 0x9e, 0x1f, 0xd1, 0x23, 0x1a, 0xcd, 0xc8, 0x69, + 0xbb, 0x6e, 0x61, 0x36, 0x6f, 0xc0, 0x7c, 0x61, 0x22, 0xf6, 0xd8, 0x8b, 0x05, 0xc6, 0xaa, 0x4d, + 0xf9, 0x8c, 0x9c, 0xf6, 0x57, 0x7f, 0x3e, 0xc7, 0x95, 0x3f, 0xcf, 0xb1, 0xe7, 0x7f, 0x0e, 0xf5, + 0x3d, 0x4d, 0x34, 0x45, 0x43, 0xb8, 0xe3, 0x10, 0x49, 0x9a, 0x8a, 0x13, 0x3a, 0x29, 0x96, 0xf1, + 0xff, 0xa8, 0x2d, 0x9b, 0x36, 0x70, 0x59, 0x7e, 0x0a, 0xcd, 0xa5, 0x6d, 0xa1, 0xbb, 0x50, 0x3d, + 0xa4, 0x67, 0xc5, 0xb1, 0x36, 0x22, 0x1a, 0x42, 0xdd, 0xee, 0xae, 0x38, 0x2b, 0x3d, 0x83, 0xf1, + 0x5b, 0x8e, 0xdf, 0xbf, 0xc5, 0x1e, 0xf6, 0x19, 0xd7, 0xa1, 0xcb, 0xee, 0xd7, 0x6c, 0xf7, 0x3f, + 0x79, 0xd0, 0x5a, 0x26, 0x0b, 0xdd, 0x07, 0xb8, 0x26, 0xb9, 0x28, 0xdb, 0x28, 0xa9, 0x43, 0xdf, + 0x40, 0x75, 0x4a, 0x5f, 0xcb, 0xe9, 0x30, 0xb8, 0x45, 0x53, 0x1f, 0x43, 0xa3, 0xe4, 0xe8, 0x3f, + 0x08, 0x40, 0x50, 0x53, 0xec, 0x85, 0xbb, 0x2b, 0xf5, 0xd0, 0xca, 0x45, 0xe2, 0xdf, 0x1e, 0xac, + 0x0c, 0x13, 0x49, 0x95, 0x42, 0x8f, 0x61, 0x95, 0xb3, 0xf8, 0x90, 0x93, 0x59, 0xf1, 0x4f, 0x08, + 0xf0, 0x55, 0x8e, 0x4b, 0xdb, 0x3c, 0xc7, 0xeb, 0xee, 0x82, 0x2d, 0x2c, 0x7e, 0x58, 0x3a, 0xd1, + 0xd7, 0x50, 0xcb, 0x28, 0x95, 0xb6, 0x42, 0x2b, 0x78, 0x7a, 0x95, 0x63, 0xab, 0xcf, 0x73, 0xdc, + 0x74, 0x49, 0x46, 0xf3, 0xff, 0xca, 0xf1, 0xc3, 0x5b, 0x8c, 0x37, 0x88, 0xe3, 0xc1, 0x64, 0x62, + 0x9a, 0x0a, 0x2d, 0x0a, 0x0a, 0xa1, 0x79, 0x4d, 0xb1, 0xfb, 0xf3, 0x34, 0x82, 0x9d, 0x8b, 0x1c, + 0x43, 0xb9, 0x09, 0x75, 0x95, 0x63, 0x28, 0x59, 0x57, 0xf3, 0x1c, 0xbf, 0x55, 0x14, 0x2e, 0x6d, + 0x7e, 0xb8, 0x14, 0x60, 0xe7, 0xaf, 0xf8, 0xdf, 0x7a, 0xf0, 0xde, 0xf0, 0x54, 0x53, 0xae, 0x98, + 0xe0, 0x7b, 0x9c, 0x64, 0xea, 0x99, 0xd0, 0x9a, 0xca, 0x81, 0xd4, 0x6c, 0x4a, 0x62, 0x3d, 0x22, + 0x67, 0xa9, 0x20, 0x13, 0xf4, 0x00, 0x6a, 0x4b, 0xc4, 0xdc, 0x33, 0xf3, 0x15, 0xa4, 0x14, 0xf3, + 0x39, 0x42, 0xac, 0xd1, 0x04, 0x4f, 0x88, 0x26, 0x05, 0x19, 0x36, 0xd8, 0xe8, 0xd7, 0xc1, 0x46, + 0xf3, 0x43, 0x6b, 0x74, 0x7d, 0x04, 0xfb, 0x2f, 0x2f, 0x3a, 0xde, 0xab, 0x8b, 0x8e, 0xf7, 0xc7, + 0x45, 0xc7, 0xfb, 0xf1, 0xb2, 0x53, 0x79, 0x75, 0xd9, 0xa9, 0xfc, 0x7a, 0xd9, 0xa9, 0x7c, 0xf5, + 0x78, 0x89, 0xb0, 0x81, 0x7b, 0x2e, 0xdc, 0xf5, 0xb0, 0x84, 0x25, 0x22, 0x25, 0x3c, 0x59, 0x30, + 0x79, 0x7a, 0xfd, 0x92, 0x58, 0x26, 0xc7, 0x2b, 0xf6, 0x01, 0xf8, 0xf0, 0x9f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xd2, 0xc0, 0xc3, 0x71, 0x69, 0x06, 0x00, 0x00, } func (this *Params) Equal(that interface{}) bool { @@ -1079,6 +1138,43 @@ func (m *Egress) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ExtensionSnapshotterArtifactPayload) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExtensionSnapshotterArtifactPayload) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ExtensionSnapshotterArtifactPayload) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = encodeVarintSwingset(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0x12 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintSwingset(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintSwingset(dAtA []byte, offset int, v uint64) int { offset -= sovSwingset(v) base := offset @@ -1255,6 +1351,23 @@ func (m *Egress) Size() (n int) { return n } +func (m *ExtensionSnapshotterArtifactPayload) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovSwingset(uint64(l)) + } + l = len(m.Data) + if l > 0 { + n += 1 + l + sovSwingset(uint64(l)) + } + return n +} + func sovSwingset(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -2306,6 +2419,122 @@ func (m *Egress) Unmarshal(dAtA []byte) error { } return nil } +func (m *ExtensionSnapshotterArtifactPayload) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSwingset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExtensionSnapshotterArtifactPayload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExtensionSnapshotterArtifactPayload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSwingset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthSwingset + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthSwingset + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSwingset + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthSwingset + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthSwingset + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSwingset(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSwingset + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipSwingset(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/packages/cosmic-proto/dist/agoric/swingset/swingset.d.ts b/packages/cosmic-proto/dist/agoric/swingset/swingset.d.ts index 7acac814aed..9b2e99c6a60 100644 --- a/packages/cosmic-proto/dist/agoric/swingset/swingset.d.ts +++ b/packages/cosmic-proto/dist/agoric/swingset/swingset.d.ts @@ -111,6 +111,11 @@ export interface Egress { /** TODO: Remove these power flags as they are deprecated and have no effect. */ powerFlags: string[]; } +/** The payload messages used by swingset state-sync */ +export interface ExtensionSnapshotterArtifactPayload { + name: string; + data: Uint8Array; +} export declare const CoreEvalProposal: { encode(message: CoreEvalProposal, writer?: _m0.Writer): _m0.Writer; decode(input: _m0.Reader | Uint8Array, length?: number): CoreEvalProposal; @@ -509,6 +514,31 @@ export declare const Egress: { object: I, ): Egress; }; +export declare const ExtensionSnapshotterArtifactPayload: { + encode( + message: ExtensionSnapshotterArtifactPayload, + writer?: _m0.Writer, + ): _m0.Writer; + decode( + input: _m0.Reader | Uint8Array, + length?: number, + ): ExtensionSnapshotterArtifactPayload; + fromJSON(object: any): ExtensionSnapshotterArtifactPayload; + toJSON(message: ExtensionSnapshotterArtifactPayload): unknown; + fromPartial< + I extends { + name?: string | undefined; + data?: Uint8Array | undefined; + } & { + name?: string | undefined; + data?: Uint8Array | undefined; + } & { + [K in Exclude]: never; + }, + >( + object: I, + ): ExtensionSnapshotterArtifactPayload; +}; declare type Builtin = | Date | Function diff --git a/packages/cosmic-proto/dist/agoric/swingset/swingset.js b/packages/cosmic-proto/dist/agoric/swingset/swingset.js index 75793e0066c..d425ff80482 100644 --- a/packages/cosmic-proto/dist/agoric/swingset/swingset.js +++ b/packages/cosmic-proto/dist/agoric/swingset/swingset.js @@ -540,6 +540,63 @@ export const Egress = { return message; }, }; +function createBaseExtensionSnapshotterArtifactPayload() { + return { name: '', data: new Uint8Array() }; +} +export const ExtensionSnapshotterArtifactPayload = { + encode(message, writer = _m0.Writer.create()) { + if (message.name !== '') { + writer.uint32(10).string(message.name); + } + if (message.data.length !== 0) { + writer.uint32(18).bytes(message.data); + } + return writer; + }, + decode(input, length) { + const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseExtensionSnapshotterArtifactPayload(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.name = reader.string(); + break; + case 2: + message.data = reader.bytes(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }, + fromJSON(object) { + return { + name: isSet(object.name) ? String(object.name) : '', + data: isSet(object.data) + ? bytesFromBase64(object.data) + : new Uint8Array(), + }; + }, + toJSON(message) { + const obj = {}; + message.name !== undefined && (obj.name = message.name); + message.data !== undefined && + (obj.data = base64FromBytes( + message.data !== undefined ? message.data : new Uint8Array(), + )); + return obj; + }, + fromPartial(object) { + const message = createBaseExtensionSnapshotterArtifactPayload(); + message.name = object.name ?? ''; + message.data = object.data ?? new Uint8Array(); + return message; + }, +}; var globalThis = (() => { if (typeof globalThis !== 'undefined') { return globalThis; diff --git a/packages/cosmic-proto/package.json b/packages/cosmic-proto/package.json index b70c2c27267..fd9383c5cfd 100644 --- a/packages/cosmic-proto/package.json +++ b/packages/cosmic-proto/package.json @@ -31,8 +31,8 @@ }, "scripts": { "build": "echo Use yarn rebuild to update dist output", - "rebuild": "rm -rf gen && mkdir -p gen && yarn generate && tsc --build && yarn prettier -w swingset", - "generate": "protoc-gen-ts_proto --plugin=node_modules/.bin/protoc-gen-ts_proto --ts_proto_opt='esModuleInterop=true,forceLong=long,useOptionals=messages' --ts_proto_out=./gen --ts_proto_opt=importSuffix=.js ./proto/agoric/swingset/msgs.proto ./proto/agoric/swingset/query.proto -I./proto", + "rebuild": "rm -rf gen && mkdir -p gen && yarn generate && tsc --build && yarn prettier -w dist/agoric && rm -rf gen", + "generate": "protoc --plugin=node_modules/.bin/protoc-gen-ts_proto --ts_proto_opt='esModuleInterop=true,forceLong=long,useOptionals=messages' --ts_proto_out=./gen --ts_proto_opt=importSuffix=.js ./proto/agoric/swingset/msgs.proto ./proto/agoric/swingset/query.proto -I./proto", "test": "node test/sanity-test.js", "test:xs": "exit 0", "lint": "exit 0", From 7b0d99ed1db8fa85757de994b7cf3a82fb967244 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Fri, 24 Feb 2023 22:02:08 +0000 Subject: [PATCH 14/20] feat(cosmos): wire swingset SnapshotExtension --- .../cosmos/x/swingset/keeper/snapshotter.go | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/snapshotter.go b/golang/cosmos/x/swingset/keeper/snapshotter.go index 810097eaf00..9326f3d365f 100644 --- a/golang/cosmos/x/swingset/keeper/snapshotter.go +++ b/golang/cosmos/x/swingset/keeper/snapshotter.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "github.com/Agoric/agoric-sdk/golang/cosmos/vm" "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types" @@ -18,8 +19,12 @@ import ( var _ snapshots.ExtensionSnapshotter = &SwingsetSnapshotter{} +// SnapshotFormat 1 is a proto message containing an artifact name, and the binary artifact data const SnapshotFormat = 1 +// The manifest filename must be synchronized with the JS export/import tooling +const ExportManifestFilename = "export-manifest.json" + type activeSnapshot struct { // The block height of the snapshot in progress height int64 @@ -31,6 +36,14 @@ type activeSnapshot struct { retrieved bool } +type exportManifest struct { + BlockHeight uint64 `json:"blockHeight,omitempty"` + // The filename of the export data + Data string `json:"data,omitempty"` + // The list of artifact names and their corresponding filenames + Artifacts [][2]string `json:"artifacts"` +} + type SwingStoreExporter interface { ExportSwingStore(ctx sdk.Context) []*vstoragetypes.DataEntry } @@ -245,11 +258,57 @@ func (snapshotter *SwingsetSnapshotter) SnapshotExtension(height uint64, payload } defer os.RemoveAll(exportDir) + + rawManifest, err := os.ReadFile(filepath.Join(exportDir, ExportManifestFilename)) + if err != nil { + return err + } + + var manifest exportManifest + err = json.Unmarshal(rawManifest, &manifest) + if err != nil { + return err + } + + if manifest.BlockHeight != height { + return fmt.Errorf("snapshot manifest blockHeight (%d) doesn't match (%d)", manifest.BlockHeight, height) + } + + writeFileToPayload := func(fileName string, artifactName string) error { + payload := types.ExtensionSnapshotterArtifactPayload{Name: artifactName} + + payload.Data, err = os.ReadFile(filepath.Join(exportDir, fileName)) + if err != nil { + return err + } + + payloadBytes, err := payload.Marshal() + if err != nil { + return err + } + + err = payloadWriter(payloadBytes) + if err != nil { + return err + } + + return nil + } + + for _, artifactInfo := range manifest.Artifacts { + artifactName := artifactInfo[0] + fileName := artifactInfo[1] + err = writeFileToPayload(fileName, artifactName) + if err != nil { + return err + } + } + snapshotter.activeSnapshot.retrieved = true - // FIXME: actually do something with the snapshot + snapshotter.activeSnapshot.logger.Info("fully retrieved snapshot from", "exportDir", exportDir) - return errors.New("not implemented") + return nil } // RestoreExtension restores an extension state snapshot, From 00fab12e464e5604cb3e5eb697fd02565ea78fe7 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Wed, 8 Mar 2023 17:22:38 +0000 Subject: [PATCH 15/20] feat(cosmic-swingset): add kernel-db importer --- .../cosmic-swingset/src/import-kernel-db.js | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100755 packages/cosmic-swingset/src/import-kernel-db.js diff --git a/packages/cosmic-swingset/src/import-kernel-db.js b/packages/cosmic-swingset/src/import-kernel-db.js new file mode 100755 index 00000000000..dd208736928 --- /dev/null +++ b/packages/cosmic-swingset/src/import-kernel-db.js @@ -0,0 +1,206 @@ +#! /usr/bin/env node + +// @ts-check +import '@endo/init'; + +import os from 'os'; +import process from 'process'; +import { Transform } from 'stream'; +import fsPower from 'fs'; +import fsPromisesPower from 'fs/promises'; +import pathPower from 'path'; + +import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform.js'; +import { Fail } from '@agoric/assert'; +import { importSwingStore } from '@agoric/swing-store'; + +import { isEntrypoint } from './helpers/is-entrypoint.js'; +import { makeProcessValue } from './helpers/process-value.js'; + +/** + * @typedef {object} StateSyncImporterOptions + * @property {string} stateDir the directory containing the SwingStore to export + * @property {string} exportDir the directory where to place the exported artifacts and manifest + * @property {number} [blockHeight] block height to check for + * @property {boolean} [includeHistorical] whether to include historical artifacts in the export + */ + +/** + * @param {StateSyncImporterOptions} options + * @param {object} powers + * @param {Pick & Pick} powers.fs + * @param {import('path')['resolve']} powers.pathResolve + * @param {typeof import('@agoric/swing-store')['importSwingStore']} [powers.importSwingStore] + * @param {(...args: any[]) => void} [powers.log] + * @returns {Promise} + */ +export const performStateSyncImport = async ( + { stateDir, exportDir, blockHeight, includeHistorical }, + { + fs: { createReadStream, readFile }, + pathResolve, + importSwingStore: importDB = importSwingStore, + log, + }, +) => { + /** @param {string} allegedRelativeFilename */ + const safeExportFileResolve = allegedRelativeFilename => { + const resolvedPath = pathResolve(exportDir, allegedRelativeFilename); + resolvedPath.startsWith(exportDir) || + Fail`Exported file ${allegedRelativeFilename} must be in export dir ${exportDir}`; + return resolvedPath; + }; + + const manifestPath = safeExportFileResolve('export-manifest.json'); + /** @type {Readonly} */ + const manifest = await readFile(manifestPath, { encoding: 'utf-8' }).then( + data => JSON.parse(data), + ); + + if (blockHeight !== undefined && manifest.blockHeight !== blockHeight) { + Fail`State-sync manifest for unexpected block height ${manifest.blockHeight} (expected ${blockHeight})`; + } + + if (!manifest.data) { + throw Fail`State-sync manifest missing export data`; + } + + if (!manifest.artifacts) { + throw Fail`State-sync manifest missing required artifacts`; + } + + const artifacts = harden(Object.fromEntries(manifest.artifacts)); + + if ( + includeHistorical && + manifest.mode !== 'archival' && + manifest.mode !== 'debug' + ) { + throw Fail`State-sync manifest missing historical artifacts`; + } + + // Represent the data in `exportDir` as a SwingSetExporter object. + /** @type {import('@agoric/swing-store').SwingStoreExporter} */ + const exporter = harden({ + async *getExportData() { + log?.('importing export data'); + const exportData = createReadStream( + safeExportFileResolve(/** @type {string} */ (manifest.data)), + ); + const exportDataEntries = exportData + .pipe(new BufferLineTransform()) + .setEncoding('utf8') + .pipe( + new Transform({ + objectMode: true, + transform(data, encoding, callback) { + try { + callback(null, JSON.parse(data)); + } catch (error) { + callback(error); + } + }, + }), + ); + yield* exportDataEntries; + }, + async *getArtifact(name) { + log?.(`importing artifact ${name}`); + const fileName = artifacts[name]; + if (!fileName) { + Fail`invalid artifact ${name}`; + } + const stream = createReadStream(safeExportFileResolve(fileName)); + yield* stream; + }, + async *getArtifactNames() { + yield* Object.keys(artifacts); + }, + async close() { + // TODO + }, + }); + + const swingstore = await importDB(exporter, stateDir, { includeHistorical }); + + const { hostStorage } = swingstore; + + hostStorage.kvStore.set('host.height', String(manifest.blockHeight)); + await hostStorage.commit(); + await hostStorage.close(); +}; + +/** + * @param {string[]} args + * @param {object} powers + * @param {Partial>} powers.env + * @param {string} powers.homedir + * @param {Console} powers.console + * @param {Pick & Pick} powers.fs + * @param {import('path')['resolve']} powers.pathResolve + */ +export const main = async ( + args, + { env, homedir, console, fs, pathResolve }, +) => { + const processValue = makeProcessValue({ env, args }); + + const stateDir = + processValue.getFlag('state-dir') || + // We try to find the actual cosmos state directory (default=~/.ag-chain-cosmos) + `${processValue.getFlag( + 'home', + `${homedir}/.ag-chain-cosmos`, + )}/data/ag-cosmos-chain-state`; + + const stateDirStat = await fs.stat(stateDir); + if (!stateDirStat.isDirectory()) { + throw new Error('state-dir must be an exiting directory'); + } + + const exportDir = pathResolve( + /** @type {string} */ (processValue.getFlag('export-dir', '.')), + ); + + const includeHistorical = processValue.getBoolean({ + flagName: 'include-historical', + }); + + const checkBlockHeight = processValue.getInteger({ + flagName: 'check-block-height', + }); + + const verbose = processValue.getBoolean({ + flagName: 'verbose', + }); + + await performStateSyncImport( + { + stateDir, + exportDir, + blockHeight: checkBlockHeight, + includeHistorical, + }, + { + fs, + pathResolve, + log: verbose ? console.log : undefined, + }, + ); +}; + +if (isEntrypoint(import.meta.url)) { + main(process.argv.splice(2), { + homedir: os.homedir(), + env: process.env, + console, + fs: { ...fsPower, ...fsPromisesPower }, + pathResolve: pathPower.resolve, + }).then( + _res => 0, + rej => { + console.error(`error running export-kernel-db:`, rej); + process.exit(process.exitCode || rej.exitCode || 1); + }, + ); +} From c94e49dbca2cfa4fc2e1ceb3617a5f1b6489a0a1 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Wed, 8 Mar 2023 17:25:11 +0000 Subject: [PATCH 16/20] feat(cosmos): wire swingset RestoreExtension --- .../cosmos/x/swingset/keeper/snapshotter.go | 114 +++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/snapshotter.go b/golang/cosmos/x/swingset/keeper/snapshotter.go index 9326f3d365f..474b041637c 100644 --- a/golang/cosmos/x/swingset/keeper/snapshotter.go +++ b/golang/cosmos/x/swingset/keeper/snapshotter.go @@ -4,8 +4,10 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" + "regexp" "github.com/Agoric/agoric-sdk/golang/cosmos/vm" "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types" @@ -24,6 +26,17 @@ const SnapshotFormat = 1 // The manifest filename must be synchronized with the JS export/import tooling const ExportManifestFilename = "export-manifest.json" +const ExportDataFilename = "export-data.jsonl" +const ExportedFilesMode = 0644 + +var disallowedArtifactNameChar = regexp.MustCompile(`[^-_.a-zA-Z0-9]`) + +// sanitizeArtifactName searches a string for all characters +// other than ASCII alphanumerics, hyphens, underscores, and dots, +// and replaces each of them with a hyphen. +func sanitizeArtifactName(name string) string { + return disallowedArtifactNameChar.ReplaceAllString(name, "-") +} type activeSnapshot struct { // The block height of the snapshot in progress @@ -321,7 +334,104 @@ func (snapshotter *SwingsetSnapshotter) RestoreExtension(height uint64, format u ctx := snapshotter.app.NewUncachedContext(false, tmproto.Header{Height: int64(height)}) - _ = snapshotter.exporter.ExportSwingStore(ctx) + exportDir, err := os.MkdirTemp("", fmt.Sprintf("agd-state-sync-restore-%d-*", height)) + if err != nil { + return err + } + defer os.RemoveAll(exportDir) + + manifest := exportManifest{ + BlockHeight: height, + Data: ExportDataFilename, + } + + exportDataFile, err := os.OpenFile(filepath.Join(exportDir, ExportDataFilename), os.O_CREATE|os.O_WRONLY, ExportedFilesMode) + if err != nil { + return err + } + defer exportDataFile.Close() + + // Retrieve the SwingStore "ExportData" from the verified vstorage data. + // At this point the content of the cosmos DB has been verified against the + // AppHash, which means the SwingStore data it contains can be used as the + // trusted root against which to validate the artifacts. + swingStoreEntries := snapshotter.exporter.ExportSwingStore(ctx) + + if len(swingStoreEntries) > 0 { + encoder := json.NewEncoder(exportDataFile) + encoder.SetEscapeHTML(false) + for _, dataEntry := range swingStoreEntries { + entry := []string{dataEntry.Path, dataEntry.Value} + err := encoder.Encode(entry) + if err != nil { + return err + } + } + } + + writeExportFile := func(filename string, data []byte) error { + return os.WriteFile(filepath.Join(exportDir, filename), data, ExportedFilesMode) + } + + for { + payloadBytes, err := payloadReader() + if err == io.EOF { + break + } else if err != nil { + return err + } + + payload := types.ExtensionSnapshotterArtifactPayload{} + if err = payload.Unmarshal(payloadBytes); err != nil { + return err + } + + // Since we cannot trust the state-sync payload at this point, we generate + // a safe and unique filename from the artifact name we received, by + // substituting any non letters-digits-hyphen-underscore-dot by a hyphen, + // and prefixing with an incremented id. + // The filename is not used for any purpose in the snapshotting logic. + filename := sanitizeArtifactName(payload.Name) + filename = fmt.Sprintf("%d-%s", len(manifest.Artifacts), filename) + manifest.Artifacts = append(manifest.Artifacts, [2]string{payload.Name, filename}) + err = writeExportFile(filename, payload.Data) + + if err != nil { + return err + } + } + + err = exportDataFile.Sync() + if err != nil { + return err + } + exportDataFile.Close() + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + err = writeExportFile(ExportManifestFilename, manifestBytes) + if err != nil { + return err + } - return errors.New("not implemented") + encodedExportDir, err := json.Marshal(exportDir) + if err != nil { + return err + } + + action := &snapshotAction{ + Type: "COSMOS_SNAPSHOT", + BlockHeight: int64(height), + Request: "restore", + Args: []json.RawMessage{encodedExportDir}, + } + + _, err = snapshotter.blockingSend(action) + if err != nil { + return err + } + + return nil } From 9d053b7381280c8d7afc35fb1bcbf6dd18886738 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Wed, 8 Mar 2023 17:24:47 +0000 Subject: [PATCH 17/20] feat(cosmic-swingset): wire snapshot restoring in chain-main --- packages/cosmic-swingset/src/chain-main.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index e8b09e3a81a..55e913e457a 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -4,6 +4,7 @@ import { resolve as pathResolve } from 'path'; import v8 from 'node:v8'; import process from 'node:process'; import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; import { performance } from 'perf_hooks'; import { resolve as importMetaResolve } from 'import-meta-resolve'; import tmpfs from 'tmp'; @@ -39,6 +40,7 @@ import { launch } from './launch-chain.js'; import { getTelemetryProviders } from './kernel-stats.js'; import { makeProcessValue } from './helpers/process-value.js'; import { spawnSwingStoreExport } from './export-kernel-db.js'; +import { performStateSyncImport } from './import-kernel-db.js'; // eslint-disable-next-line no-unused-vars let whenHellFreezesOver = null; @@ -476,8 +478,18 @@ export default async function main(progname, args, { env, homedir, agcc }) { return blockingSend; } - async function handleCosmosSnapshot(blockHeight, request, _requestArgs) { + async function handleCosmosSnapshot(blockHeight, request, requestArgs) { switch (request) { + case 'restore': { + const exportDir = requestArgs[0]; + if (typeof exportDir !== 'string') { + throw Fail`Invalid exportDir argument ${q(exportDir)}`; + } + return performStateSyncImport( + { exportDir, stateDir: stateDBDir, blockHeight }, + { fs: { ...fs, ...fsPromises }, pathResolve }, + ); + } case 'initiate': { !stateSyncExport || Fail`Snapshot already in progress for ${stateSyncExport.blockHeight}`; From 2faa1fbd17d5cee552bfec8fd396b96d252bca1c Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Tue, 14 Mar 2023 14:01:40 +0000 Subject: [PATCH 18/20] feat(deployment): Enable state-sync on validator nodes --- .../roles/install-cosmos/tasks/main.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/deployment/ansible/roles/install-cosmos/tasks/main.yml b/packages/deployment/ansible/roles/install-cosmos/tasks/main.yml index 2a6a2164518..8cfc55aa15e 100644 --- a/packages/deployment/ansible/roles/install-cosmos/tasks/main.yml +++ b/packages/deployment/ansible/roles/install-cosmos/tasks/main.yml @@ -93,3 +93,31 @@ section: api option: swagger value: 'true' + +- name: Set custom pruning + ini_file: + path: '/home/{{ service }}/.{{ service }}/config/app.toml' + section: + option: '{{ item.option }}' + value: '{{ item.value }}' + with_items: + - { option: 'pruning', value: '"custom"' } + - { option: 'pruning-interval', value: '10' } + - { option: 'pruning-keep-recent', value: '10' } + - { + option: 'pruning-keep-every', + value: "{{lookup('env', 'AG_SETUP_COSMOS_STATE_SYNC_INTERVAL') or 500}}", + } + +- name: Set state-sync + ini_file: + path: '/home/{{ service }}/.{{ service }}/config/app.toml' + section: state-sync + option: '{{ item.option }}' + value: '{{ item.value }}' + with_items: + - { option: 'snapshot-keep-recent', value: '2' } + - { + option: 'snapshot-interval', + value: "{{lookup('env', 'AG_SETUP_COSMOS_STATE_SYNC_INTERVAL') or 500}}", + } From 27b97c1a6fa6e702fa72f1d215bb4d42f403ed3e Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Tue, 14 Mar 2023 14:02:29 +0000 Subject: [PATCH 19/20] feat(ci): Use state-sync in deployment loadgen test --- packages/deployment/scripts/integration-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/deployment/scripts/integration-test.sh b/packages/deployment/scripts/integration-test.sh index 0cabe964565..542322aa13b 100755 --- a/packages/deployment/scripts/integration-test.sh +++ b/packages/deployment/scripts/integration-test.sh @@ -20,6 +20,7 @@ cd "$NETWORK_NAME/setup" export AG_SETUP_COSMOS_HOME=${AG_SETUP_COSMOS_HOME-$PWD} +export AG_SETUP_COSMOS_STATE_SYNC_INTERVAL=20 AGORIC_SDK_PATH=${AGORIC_SDK_PATH-$(cd "$thisdir/../../.." > /dev/null && pwd -P)} if [ -d /usr/src/testnet-load-generator ] @@ -57,6 +58,6 @@ then --no-stage.save-storage \ --stages=3 --stage.duration=10 --stage.loadgen.cycles=4 \ --stage.loadgen.faucet.interval=6 --stage.loadgen.faucet.limit=4 \ - --profile=testnet "--testnet-origin=file://$RESULTSDIR" \ + --profile=testnet "--testnet-origin=file://$RESULTSDIR" --use-state-sync \ --no-reset --custom-bootstrap fi From 283fded7ceedace3c916cf1e02eb25d1e4f87a88 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Mon, 27 Feb 2023 14:52:54 +0000 Subject: [PATCH 20/20] feat(cosmos): Export KVData as artifact --- .../cosmos/x/swingset/keeper/snapshotter.go | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/golang/cosmos/x/swingset/keeper/snapshotter.go b/golang/cosmos/x/swingset/keeper/snapshotter.go index 474b041637c..4c06f0b516d 100644 --- a/golang/cosmos/x/swingset/keeper/snapshotter.go +++ b/golang/cosmos/x/swingset/keeper/snapshotter.go @@ -27,6 +27,8 @@ const SnapshotFormat = 1 // The manifest filename must be synchronized with the JS export/import tooling const ExportManifestFilename = "export-manifest.json" const ExportDataFilename = "export-data.jsonl" +const UntrustedExportDataArtifactName = "UNTRUSTED-EXPORT-DATA" +const UntrustedExportDataFilename = "untrusted-export-data.jsonl" const ExportedFilesMode = 0644 var disallowedArtifactNameChar = regexp.MustCompile(`[^-_.a-zA-Z0-9]`) @@ -308,9 +310,19 @@ func (snapshotter *SwingsetSnapshotter) SnapshotExtension(height uint64, payload return nil } + if manifest.Data != "" { + err = writeFileToPayload(manifest.Data, UntrustedExportDataArtifactName) + if err != nil { + return err + } + } + for _, artifactInfo := range manifest.Artifacts { artifactName := artifactInfo[0] fileName := artifactInfo[1] + if artifactName == UntrustedExportDataArtifactName { + return fmt.Errorf("unexpected artifact name %s", artifactName) + } err = writeFileToPayload(fileName, artifactName) if err != nil { return err @@ -386,15 +398,30 @@ func (snapshotter *SwingsetSnapshotter) RestoreExtension(height uint64, format u return err } - // Since we cannot trust the state-sync payload at this point, we generate - // a safe and unique filename from the artifact name we received, by - // substituting any non letters-digits-hyphen-underscore-dot by a hyphen, - // and prefixing with an incremented id. - // The filename is not used for any purpose in the snapshotting logic. - filename := sanitizeArtifactName(payload.Name) - filename = fmt.Sprintf("%d-%s", len(manifest.Artifacts), filename) - manifest.Artifacts = append(manifest.Artifacts, [2]string{payload.Name, filename}) - err = writeExportFile(filename, payload.Data) + switch { + case payload.Name != UntrustedExportDataArtifactName: + // Artifact verifiable on import from the export data + // Since we cannot trust the state-sync payload at this point, we generate + // a safe and unique filename from the artifact name we received, by + // substituting any non letters-digits-hyphen-underscore-dot by a hyphen, + // and prefixing with an incremented id. + // The filename is not used for any purpose in the snapshotting logic. + filename := sanitizeArtifactName(payload.Name) + filename = fmt.Sprintf("%d-%s", len(manifest.Artifacts), filename) + manifest.Artifacts = append(manifest.Artifacts, [2]string{payload.Name, filename}) + err = writeExportFile(filename, payload.Data) + + case len(swingStoreEntries) > 0: + // Pseudo artifact containing untrusted export data which may have been + // saved separately for debugging purposes (not referenced from the manifest) + err = writeExportFile(UntrustedExportDataFilename, payload.Data) + + default: + // There is no trusted export data + err = errors.New("cannot restore from untrusted export data") + // snapshotter.logger.Info("using untrusted export data for swingstore restore") + // _, err = exportDataFile.Write(payload.Data) + } if err != nil { return err