From c3c8b0015d438c422fd18a419b729856b589bca1 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Mon, 14 Oct 2024 18:55:43 -0300 Subject: [PATCH 1/3] feat: add support of incremental quicksync --- desktop/NodeManager.ts | 24 +++++++++++++++++------- node/use-version-quicksync | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/desktop/NodeManager.ts b/desktop/NodeManager.ts index 69f0e9ca4..3873b2c4a 100644 --- a/desktop/NodeManager.ts +++ b/desktop/NodeManager.ts @@ -778,6 +778,9 @@ class NodeManager extends AbstractManager { }; }; + private getNodeStateFile = (nodeData: string) => + path.resolve(nodeData, 'state.sql'); + private checkForPausedQuicksync = async ( nodeData: string ): Promise => { @@ -912,13 +915,20 @@ class NodeManager extends AbstractManager { } const bin = getQuicksyncPath(); - const args = [ - 'download', - '--node-data', - this.getNodeDataPath(), - '--go-spacemesh-path', - getNodePath(), - ]; + + const nodeDataPath = this.getNodeDataPath(); + const stateFile = this.getNodeStateFile(nodeDataPath); + const shouldDownloadEntire = !(await fse.pathExists(stateFile)); + + const args = shouldDownloadEntire + ? [ + 'download', + '--node-data', + nodeDataPath, + '--go-spacemesh-path', + getNodePath(), + ] + : ['partial', '-s', stateFile, '-j', '1']; logger.log('runQuicksync:download', `${bin} ${args.join(' ')}`); diff --git a/node/use-version-quicksync b/node/use-version-quicksync index 00a2c6f10..42cd91d18 100644 --- a/node/use-version-quicksync +++ b/node/use-version-quicksync @@ -1 +1 @@ -v0.1.16 +v0.2.0-alpha From a07a3b3a6abea08429e3c1afcca4b4e58ba0edcc Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Tue, 15 Oct 2024 13:06:53 -0300 Subject: [PATCH 2/3] tweak: run partial update only if possible and handle corner cases --- desktop/NodeManager.ts | 153 ++++++++++++++++++++++++++++++++----- node/use-version-quicksync | 2 +- shared/types/quicksync.ts | 4 + 3 files changed, 139 insertions(+), 20 deletions(-) diff --git a/desktop/NodeManager.ts b/desktop/NodeManager.ts index 3873b2c4a..ea2fee07f 100644 --- a/desktop/NodeManager.ts +++ b/desktop/NodeManager.ts @@ -329,11 +329,33 @@ class NodeManager extends AbstractManager { isLocalStateFarBehind(qsStatus, 100) ) { await this.runQuicksyncPrompt(qsStatus); + } else if ( + qsStatus.partial && + qsStatus.db >= qsStatus.partial.from && + qsStatus.partial.to >= qsStatus.current - 10 + ) { + hideGenericModal(this.mainWindow.webContents); + const isForce = await showGenericPrompt(this.mainWindow.webContents, { + title: 'Quick sync is not needed', + message: [ + `Partial quicksync available from layer ${qsStatus.partial.from} to ${qsStatus.partial.to}`, + `Latest layer in your database: ${qsStatus.db}`, + `Current layer in the network: ${qsStatus.current}`, + '', + 'Your node is nearly synced. We recommend you to keep using your local state.', + 'Do you want to partially update to the trusted state anyway?', + ].join('\n'), + confirmTitle: 'Update partially!', + cancelTitle: 'No, thanks!', + cancelTimeout: 60, + }); + isForce && (await this.runQuicksyncDownload()); } else { hideGenericModal(this.mainWindow.webContents); const isForce = await showGenericPrompt(this.mainWindow.webContents, { title: 'Quick sync is not needed', message: [ + 'Partial update is not available.', `Latest layer in your database: ${qsStatus.db}`, `Latest layer in the trusted state: ${qsStatus.available}`, `Current layer in the network: ${qsStatus.current}`, @@ -341,11 +363,11 @@ class NodeManager extends AbstractManager { 'Your node is nearly synced. We recommend you to keep using your local state.', 'Do you want to download the trusted state anyway?', ].join('\n'), - confirmTitle: 'Download anyway!', + confirmTitle: 'Download entire state!', cancelTitle: 'No, thanks!', cancelTimeout: 60, }); - isForce && (await this.runQuicksyncDownload()); + isForce && (await this.runQuicksyncDownload(isForce)); } } catch (err) { logger.error('runQuicksyncByButton', err); @@ -484,9 +506,19 @@ class NodeManager extends AbstractManager { try { hideGenericModal(this.mainWindow.webContents); + const isPartialAvailable = + qsStatus.partial !== null && + qsStatus.db >= qsStatus.partial.from && + qsStatus.db < qsStatus.partial.to; + const prompt = await showGenericPrompt(this.mainWindow.webContents, { title: 'Run a Quicksync?', message: [ + `Partial update is ${ + isPartialAvailable + ? `available from layer ${qsStatus.partial?.from} to ${qsStatus.partial?.to}` + : 'not available' + }`, `Latest layer in your database: ${qsStatus.db}`, `Latest layer in the trusted state: ${qsStatus.available}`, `Current layer in the network: ${qsStatus.current}`, @@ -497,7 +529,7 @@ class NodeManager extends AbstractManager { '', 'You can run the Quicksync from the settings page later.', ].join('\n'), - confirmTitle: 'Yes, download it!', + confirmTitle: 'Yes, run quicksync!', cancelTitle: 'Sync as usual', cancelTimeout: 60, }); @@ -517,24 +549,33 @@ class NodeManager extends AbstractManager { const resume = await showGenericPrompt(this.mainWindow.webContents, { title: 'Resume interrupted quicksync?', message: [ - 'You have an incomplete download:', + 'You have an incomplete download of the entire state:', `Layer: ${qsStatus.paused.layer}`, `Progress: ${downloaded} MB / ${total} MB (${percent}%)`, '', 'Do you want to resume downloading it?', - `Or download from scratch for layer: ${qsStatus.available}?`, + isPartialAvailable + ? 'Or delete downloaded data and run partial quicksync?' + : `Or download from scratch for layer: ${qsStatus.available}?`, ].join('\n'), confirmTitle: 'Yes, resume', - cancelTitle: 'Download from scratch', + cancelTitle: isPartialAvailable + ? 'Run partial quicksync' + : 'Download from scratch', }); - if (!resume) { - const { urlFile, downloadFile } = this.getQuicksyncFiles( - this.getNodeDataPath() - ); - await fse.remove(urlFile); - await fse.remove(downloadFile); + if (resume) { + // Continue downloading + await this.runQuicksyncDownload(true); + return; } + // Otherwise clean up downloaded files + const { urlFile, downloadFile } = this.getQuicksyncFiles( + this.getNodeDataPath() + ); + await fse.remove(urlFile); + await fse.remove(downloadFile); } + // And run quicksync as usual await this.runQuicksyncDownload(); } } catch (err) { @@ -824,10 +865,74 @@ class NodeManager extends AbstractManager { } }; + private checkForPartialQuicksync = async (nodeDataDir: string) => { + const bin = getQuicksyncPath(); + const stateFile = path.resolve(nodeDataDir, 'state.sql'); + + return new Promise((resolve, reject) => { + const args = ['partial-check', '--state-sql', stateFile]; + const process = spawn(bin, args); + logger.log('runQuicksync:check', `${bin} ${args.join(' ')}`); + + let stdout = ''; + let stderr = ''; + process.stdout.on('data', (data) => { + stdout += data; + }); + process.stderr.on('data', (data) => { + stderr += data; + }); + + process.on('error', (err) => { + reject( + new Error( + `Cannot run Quicksync tool to check the database: ${err.message}` + ) + ); + }); + + process.on('close', async (code) => { + if (code !== 0) { + // TODO: Better error for known codes + return reject( + new Error( + `Quicksync tool failed with exit code ${code}:\n${stderr}` + ) + ); + } + + const stats = stdout + .split('\n') + .filter((l) => l.startsWith('Possible to restore')); + + if (stats.length !== 2) { + return reject( + new Error( + `Unknown result of running Quicksync tool:\n${stdout}\n${stderr}` + ) + ); + } + + const result: QuicksyncStatus['partial'] = { + from: parseInt(stats[0].match(/(\d+)/)?.[0] || '0', 10), + to: parseInt(stats[1].match(/(\d+)/)?.[0] || '0', 10), + }; + + logger.log('runQuicksyncPartialCheck', result); + return resolve(result); + }); + }).catch(() => { + return null; + }); + }; + + private quicksyncStatus: QuicksyncStatus | null = null; + private runQuicksyncCheck = async () => { const bin = getQuicksyncPath(); const nodeDataDir = this.getNodeDataPath(); const paused = await this.checkForPausedQuicksync(nodeDataDir); + const partial = await this.checkForPartialQuicksync(nodeDataDir); return new Promise((resolve, reject) => { const args = [ 'check', @@ -889,8 +994,10 @@ class NodeManager extends AbstractManager { current, available, paused, + partial, }; + this.quicksyncStatus = result; this.sendToMainWindow(ipcConsts.UPDATE_QUICKSYNC_STATUS, result); logger.log('runQuicksyncCheck', result); return resolve(result); @@ -898,7 +1005,9 @@ class NodeManager extends AbstractManager { }); }; - private runQuicksyncDownload = async () => { + private runQuicksyncDownload = async ( + forceEntire = false + ): Promise => { if (this.nodeProcess) { showGenericModal(this.mainWindow.webContents, { title: 'Quicksyncing...', @@ -918,21 +1027,27 @@ class NodeManager extends AbstractManager { const nodeDataPath = this.getNodeDataPath(); const stateFile = this.getNodeStateFile(nodeDataPath); - const shouldDownloadEntire = !(await fse.pathExists(stateFile)); - const args = shouldDownloadEntire - ? [ + const isPartialAvailable = + this.quicksyncStatus?.partial && + this.quicksyncStatus.db >= this.quicksyncStatus.partial.from && + this.quicksyncStatus.db < this.quicksyncStatus.partial.to; + + const shouldQuicksyncPartially = !forceEntire && isPartialAvailable; + + const args = shouldQuicksyncPartially + ? ['partial', '--state-sql', stateFile] + : [ 'download', '--node-data', nodeDataPath, '--go-spacemesh-path', getNodePath(), - ] - : ['partial', '-s', stateFile, '-j', '1']; + ]; logger.log('runQuicksync:download', `${bin} ${args.join(' ')}`); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { if (!isFileExists(bin)) { reject(new Error(`Quicksync tool is not found: ${bin}`)); return; diff --git a/node/use-version-quicksync b/node/use-version-quicksync index 42cd91d18..cedb07d02 100644 --- a/node/use-version-quicksync +++ b/node/use-version-quicksync @@ -1 +1 @@ -v0.2.0-alpha +v0.2.0-alpha.1 diff --git a/shared/types/quicksync.ts b/shared/types/quicksync.ts index 6ff57a9ca..58c0c5d3a 100644 --- a/shared/types/quicksync.ts +++ b/shared/types/quicksync.ts @@ -9,4 +9,8 @@ export interface QuicksyncStatus { current: number; available: number; paused: PausedQuicksyncStatus | null; + partial: { + from: number; + to: number; + } | null; } From c4268e8f38ec9a7cdb573ddf67449c84e27bd981 Mon Sep 17 00:00:00 2001 From: brusher_ru Date: Wed, 16 Oct 2024 19:14:50 -0300 Subject: [PATCH 3/3] tweak: use renamed `incremental` term instead of `partial` --- desktop/NodeManager.ts | 64 +++++++++++++++++++------------------- node/use-version-quicksync | 2 +- shared/types/quicksync.ts | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/desktop/NodeManager.ts b/desktop/NodeManager.ts index ea2fee07f..49480fa98 100644 --- a/desktop/NodeManager.ts +++ b/desktop/NodeManager.ts @@ -330,22 +330,22 @@ class NodeManager extends AbstractManager { ) { await this.runQuicksyncPrompt(qsStatus); } else if ( - qsStatus.partial && - qsStatus.db >= qsStatus.partial.from && - qsStatus.partial.to >= qsStatus.current - 10 + qsStatus.incremental && + qsStatus.db >= qsStatus.incremental.from && + qsStatus.incremental.to >= qsStatus.current - 10 ) { hideGenericModal(this.mainWindow.webContents); const isForce = await showGenericPrompt(this.mainWindow.webContents, { title: 'Quick sync is not needed', message: [ - `Partial quicksync available from layer ${qsStatus.partial.from} to ${qsStatus.partial.to}`, + `Incremental quicksync available from layer ${qsStatus.incremental.from} to ${qsStatus.incremental.to}`, `Latest layer in your database: ${qsStatus.db}`, `Current layer in the network: ${qsStatus.current}`, '', 'Your node is nearly synced. We recommend you to keep using your local state.', - 'Do you want to partially update to the trusted state anyway?', + 'Do you want to incrementally update to the trusted state anyway?', ].join('\n'), - confirmTitle: 'Update partially!', + confirmTitle: 'Update incrementally!', cancelTitle: 'No, thanks!', cancelTimeout: 60, }); @@ -355,7 +355,7 @@ class NodeManager extends AbstractManager { const isForce = await showGenericPrompt(this.mainWindow.webContents, { title: 'Quick sync is not needed', message: [ - 'Partial update is not available.', + 'Incremental update is not available.', `Latest layer in your database: ${qsStatus.db}`, `Latest layer in the trusted state: ${qsStatus.available}`, `Current layer in the network: ${qsStatus.current}`, @@ -506,17 +506,17 @@ class NodeManager extends AbstractManager { try { hideGenericModal(this.mainWindow.webContents); - const isPartialAvailable = - qsStatus.partial !== null && - qsStatus.db >= qsStatus.partial.from && - qsStatus.db < qsStatus.partial.to; + const isIncrementalAvailable = + qsStatus.incremental !== null && + qsStatus.db >= qsStatus.incremental.from && + qsStatus.db < qsStatus.incremental.to; const prompt = await showGenericPrompt(this.mainWindow.webContents, { title: 'Run a Quicksync?', message: [ - `Partial update is ${ - isPartialAvailable - ? `available from layer ${qsStatus.partial?.from} to ${qsStatus.partial?.to}` + `Incremental update is ${ + isIncrementalAvailable + ? `available from layer ${qsStatus.incremental?.from} to ${qsStatus.incremental?.to}` : 'not available' }`, `Latest layer in your database: ${qsStatus.db}`, @@ -554,13 +554,13 @@ class NodeManager extends AbstractManager { `Progress: ${downloaded} MB / ${total} MB (${percent}%)`, '', 'Do you want to resume downloading it?', - isPartialAvailable - ? 'Or delete downloaded data and run partial quicksync?' + isIncrementalAvailable + ? 'Or delete downloaded data and run incremental quicksync?' : `Or download from scratch for layer: ${qsStatus.available}?`, ].join('\n'), confirmTitle: 'Yes, resume', - cancelTitle: isPartialAvailable - ? 'Run partial quicksync' + cancelTitle: isIncrementalAvailable + ? 'Run incremental quicksync' : 'Download from scratch', }); if (resume) { @@ -865,12 +865,12 @@ class NodeManager extends AbstractManager { } }; - private checkForPartialQuicksync = async (nodeDataDir: string) => { + private checkForIncrementalQuicksync = async (nodeDataDir: string) => { const bin = getQuicksyncPath(); const stateFile = path.resolve(nodeDataDir, 'state.sql'); - return new Promise((resolve, reject) => { - const args = ['partial-check', '--state-sql', stateFile]; + return new Promise((resolve, reject) => { + const args = ['incremental-check', '--state-sql', stateFile]; const process = spawn(bin, args); logger.log('runQuicksync:check', `${bin} ${args.join(' ')}`); @@ -913,12 +913,12 @@ class NodeManager extends AbstractManager { ); } - const result: QuicksyncStatus['partial'] = { + const result: QuicksyncStatus['incremental'] = { from: parseInt(stats[0].match(/(\d+)/)?.[0] || '0', 10), to: parseInt(stats[1].match(/(\d+)/)?.[0] || '0', 10), }; - logger.log('runQuicksyncPartialCheck', result); + logger.log('runQuicksyncIncrementalCheck', result); return resolve(result); }); }).catch(() => { @@ -932,7 +932,7 @@ class NodeManager extends AbstractManager { const bin = getQuicksyncPath(); const nodeDataDir = this.getNodeDataPath(); const paused = await this.checkForPausedQuicksync(nodeDataDir); - const partial = await this.checkForPartialQuicksync(nodeDataDir); + const incremental = await this.checkForIncrementalQuicksync(nodeDataDir); return new Promise((resolve, reject) => { const args = [ 'check', @@ -994,7 +994,7 @@ class NodeManager extends AbstractManager { current, available, paused, - partial, + incremental, }; this.quicksyncStatus = result; @@ -1028,15 +1028,15 @@ class NodeManager extends AbstractManager { const nodeDataPath = this.getNodeDataPath(); const stateFile = this.getNodeStateFile(nodeDataPath); - const isPartialAvailable = - this.quicksyncStatus?.partial && - this.quicksyncStatus.db >= this.quicksyncStatus.partial.from && - this.quicksyncStatus.db < this.quicksyncStatus.partial.to; + const isIncrementalAvailable = + this.quicksyncStatus?.incremental && + this.quicksyncStatus.db >= this.quicksyncStatus.incremental.from && + this.quicksyncStatus.db < this.quicksyncStatus.incremental.to; - const shouldQuicksyncPartially = !forceEntire && isPartialAvailable; + const shouldQuicksyncIncrementally = !forceEntire && isIncrementalAvailable; - const args = shouldQuicksyncPartially - ? ['partial', '--state-sql', stateFile] + const args = shouldQuicksyncIncrementally + ? ['incremental', '--state-sql', stateFile] : [ 'download', '--node-data', diff --git a/node/use-version-quicksync b/node/use-version-quicksync index cedb07d02..f5efc86be 100644 --- a/node/use-version-quicksync +++ b/node/use-version-quicksync @@ -1 +1 @@ -v0.2.0-alpha.1 +v0.2.0-alpha.3 diff --git a/shared/types/quicksync.ts b/shared/types/quicksync.ts index 58c0c5d3a..8ddbf0abb 100644 --- a/shared/types/quicksync.ts +++ b/shared/types/quicksync.ts @@ -9,7 +9,7 @@ export interface QuicksyncStatus { current: number; available: number; paused: PausedQuicksyncStatus | null; - partial: { + incremental: { from: number; to: number; } | null;