From d4f1e5e8446abc75403b62b8e37f69b459fb7afd Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 26 Feb 2024 20:38:43 +0000 Subject: [PATCH 01/20] current version, not working Signed-off-by: Jeromy Cannon --- src/commands/account.mjs | 3 -- src/commands/node.mjs | 3 +- src/core/account_manager.mjs | 90 ++++++++++++++++++++++-------------- src/core/constants.mjs | 2 +- 4 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/commands/account.mjs b/src/commands/account.mjs index bf18b4170..dad5e89fe 100644 --- a/src/commands/account.mjs +++ b/src/commands/account.mjs @@ -20,7 +20,6 @@ import { flags } from './index.mjs' import { Listr } from 'listr2' import * as prompts from './prompts.mjs' import { constants } from '../core/index.mjs' -import { sleep } from '../core/helpers.mjs' import { HbarUnit, PrivateKey } from '@hashgraph/sdk' export class AccountCommand extends BaseCommand { @@ -37,10 +36,8 @@ export class AccountCommand extends BaseCommand { async closeConnections () { if (this.nodeClient) { this.nodeClient.close() - await sleep(5) // sleep a couple of ticks for connections to close } await this.accountManager.stopPortForwards() - await sleep(5) // sleep a couple of ticks for connections to close } async buildAccountInfo (accountInfo, namespace, shouldRetrievePrivateKey) { diff --git a/src/commands/node.mjs b/src/commands/node.mjs index a8c7fca8a..981ed47d8 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -492,7 +492,7 @@ export class NodeCommand extends BaseCommand { // set up the sub-tasks return task.newListr(subTasks, { - concurrent: false, + concurrent: true, rendererOptions: { collapseSubtasks: false } @@ -586,7 +586,6 @@ export class NodeCommand extends BaseCommand { if (nodeClient) { nodeClient.close() } - await sleep(5) // sleep a few ticks to allow network connections to close } } diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 50fc93e24..da78e22ae 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -35,6 +35,7 @@ import { sleep } from './helpers.mjs' import net from 'net' import chalk from 'chalk' import { Templates } from './templates.mjs' +import * as util from 'util' const REASON_FAILED_TO_GET_KEYS = 'failed to get keys for accountId' const REASON_SKIPPED = 'skipped since it does not have a genesis key' @@ -140,11 +141,19 @@ export class AccountManager { */ async stopPortForwards () { if (this.portForwards) { - this.portForwards.forEach(server => { - server.close() - }) + for (const webSocketServer of this.portForwards) { + // const serverClose = util.promisify(webSocketServer.close) + // await serverClose() + await (() => { + return new Promise((resolve, reject) => { + webSocketServer.close((err, data) => { + if (err) return reject(err) + resolve(data) + }) + }) + })() + } this.portForwards = [] - await sleep(1) // give a tick for connections to close } } @@ -233,43 +242,57 @@ export class AccountManager { */ async updateSpecialAccountsKeys (namespace, nodeClient, accounts) { const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) - const accountUpdatePromiseArray = [] const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard + const batchSize = constants.ACCOUNT_CREATE_BATCH_SIZE + const batchSets = [] + let batchCounter = 0 + let currentBatch = [] for (const [start, end] of accounts) { for (let i = start; i <= end; i++) { - accountUpdatePromiseArray.push(this.updateAccountKeys( - namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${i}`), genesisKey)) - - await sleep(constants.ACCOUNT_KEYS_UPDATE_PAUSE) // sleep a little to prevent overwhelming the servers + if (++batchCounter >= batchSize) { + batchSets.push(currentBatch) + currentBatch = [] + } + currentBatch.push(i) } } - await Promise.allSettled(accountUpdatePromiseArray).then((results) => { - let rejectedCount = 0 - let fulfilledCount = 0 - let skippedCount = 0 - - for (const result of results) { - switch (result.status) { - case 'rejected': - if (result.reason === REASON_SKIPPED) { - skippedCount++ - } else { - this.logger.error(`REJECT: ${result.reason}: ${result.value}`) - rejectedCount++ - } - break - case 'fulfilled': - fulfilledCount++ - break - } + let rejectedCount = 0 + let fulfilledCount = 0 + let skippedCount = 0 + + for (const currentSet of batchSets) { + const accountUpdatePromiseArray = [] + + for (const accountNum of currentSet) { + accountUpdatePromiseArray.push(this.updateAccountKeys( + namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey)) } - this.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${fulfilledCount}`)) - if (skippedCount > 0) this.logger.showUser(chalk.cyan(`Account keys updates SKIPPED: ${skippedCount}`)) - if (rejectedCount > 0) this.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${rejectedCount}`)) - }) + + await Promise.allSettled(accountUpdatePromiseArray).then((results) => { + for (const result of results) { + switch (result.status) { + case 'rejected': + if (result.reason === REASON_SKIPPED) { + skippedCount++ + } else { + this.logger.error(`REJECT: ${result.reason}: ${result.value}`) + rejectedCount++ + } + break + case 'fulfilled': + fulfilledCount++ + break + } + } + }) + } + + this.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${fulfilledCount}`)) + if (skippedCount > 0) this.logger.showUser(chalk.cyan(`Account keys updates SKIPPED: ${skippedCount}`)) + if (rejectedCount > 0) this.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${rejectedCount}`)) } /** @@ -448,8 +471,7 @@ export class AccountManager { if (!socket) { throw new FullstackTestingError(`failed to connect to port '${port}' of pod ${podName} at IP address ${host}`) } - socket.destroy() - await sleep(1) // gives a few ticks for connections to close + await socket.destroy() } /** diff --git a/src/core/constants.mjs b/src/core/constants.mjs index 00de7f220..c6aadf1d0 100644 --- a/src/core/constants.mjs +++ b/src/core/constants.mjs @@ -81,7 +81,7 @@ export const GENESIS_KEY = process.env.GENESIS_KEY || '302e020100300506032b65700 export const SYSTEM_ACCOUNTS = [[3, 100], [200, 349], [400, 750], [900, 1000]] // do account 0.0.2 last and outside the loop export const TREASURY_ACCOUNTS = [[2, 2]] export const LOCAL_NODE_START_PORT = process.env.LOCAL_NODE_START_PORT || 30212 -export const ACCOUNT_KEYS_UPDATE_PAUSE = process.env.ACCOUNT_KEYS_UPDATE_PAUSE || 5 +export const ACCOUNT_CREATE_BATCH_SIZE = process.env.ACCOUNT_CREATE_BATCH_SIZE || 50 export const POD_STATUS_RUNNING = 'Running' export const POD_STATUS_READY = 'Ready' From d377407d958f9e3cca4f887c9f964bd4fe2892b2 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 28 Feb 2024 20:40:06 +0000 Subject: [PATCH 02/20] improved account update task output Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 4 +- src/core/account_manager.mjs | 108 +++++++++++++++++++++++++---------- 2 files changed, 80 insertions(+), 32 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 981ed47d8..f93e1475d 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -543,7 +543,7 @@ export class NodeCommand extends BaseCommand { title: 'Update special account keys', task: async (ctx, task) => { if (ctx.config.updateAccountKeys) { - await self.accountManager.prepareAccounts(ctx.config.namespace) + await self.accountManager.prepareAccounts(ctx.config.namespace, task) } else { this.logger.showUser(chalk.yellowBright('> WARNING:'), chalk.yellow( 'skipping special account keys update, special accounts will retain genesis private keys')) @@ -580,7 +580,7 @@ export class NodeCommand extends BaseCommand { // Retrieve the AddressBook as base64 return await this.accountManager.prepareAddressBookBase64(nodeClient) } catch (e) { - throw new FullstackTestingError('an error was encountered while trying to prepare the address book') + throw new FullstackTestingError(`an error was encountered while trying to prepare the address book: ${e.message}`, e) } finally { await this.accountManager.stopPortForwards() if (nodeClient) { diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index da78e22ae..0e49ca915 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -35,12 +35,13 @@ import { sleep } from './helpers.mjs' import net from 'net' import chalk from 'chalk' import { Templates } from './templates.mjs' -import * as util from 'util' const REASON_FAILED_TO_GET_KEYS = 'failed to get keys for accountId' const REASON_SKIPPED = 'skipped since it does not have a genesis key' const REASON_FAILED_TO_UPDATE_ACCOUNT = 'failed to update account keys' const REASON_FAILED_TO_CREATE_K8S_S_KEY = 'failed to create k8s scrt key' +const FULFILLED = 'fulfilled' +const REJECTED = 'rejected' /** * Copyright (C) 2024 Hedera Hashgraph, LLC @@ -121,15 +122,17 @@ export class AccountManager { * @param namespace the namespace to run the update of account keys for * @returns {Promise} */ - async prepareAccounts (namespace) { + async prepareAccounts (namespace, task) { // TODO update task into jsdoc (multiple places) const serviceMap = await this.getNodeServiceMap(namespace) const nodeClient = await this.getNodeClient( namespace, serviceMap, constants.OPERATOR_ID, constants.OPERATOR_KEY) - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.SYSTEM_ACCOUNTS) + // TODO check to see if any account key secrets exist in the cluster, if they don't we can skip the deleteSecret + + await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.SYSTEM_ACCOUNTS, task) // update the treasury account last - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.TREASURY_ACCOUNTS) + await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.TREASURY_ACCOUNTS, task) nodeClient.close() await this.stopPortForwards() @@ -140,19 +143,60 @@ export class AccountManager { * @returns {Promise} */ async stopPortForwards () { + // const closedCount = 0 if (this.portForwards) { - for (const webSocketServer of this.portForwards) { - // const serverClose = util.promisify(webSocketServer.close) - // await serverClose() - await (() => { - return new Promise((resolve, reject) => { - webSocketServer.close((err, data) => { - if (err) return reject(err) - resolve(data) - }) - }) - })() - } + // original + this.portForwards.forEach(server => { + server.close() + }) + this.portForwards = [] + await sleep(1) // give a tick for connections to close + // for (const webSocketServer of this.portForwards) { + // Option 1: + // const serverClose = util.promisify(webSocketServer.close) + // await serverClose() + // Option 2: + // await (() => { + // return new Promise((resolve, reject) => { + // webSocketServer.close((err, data) => { + // if (err) return reject(err) + // resolve(data) + // }) + // }) + // })() + // Option 3: + // const closeWebSocket = (callback) => { + // // don't close if it's already closed + // if (webSocketServer.readyState === 3) { + // callback() + // } else { + // // don't notify on user-initiated shutdown ('disconnect' event) + // webSocketServer.removeAllListeners('close') + // webSocketServer.once('close', () => { + // webSocketServer.removeAllListeners() + // callback() + // }) + // webSocketServer.close() + // } + // } + // await new Promise((resolve, reject) => { + // closeWebSocket(resolve) + // }) + // Option 4: + // webSocketServer.addEventListener('close', (event) => { + // console.log('The connection has been closed successfully.') + // }) + // webSocketServer.onclose = (event) => { + // closedCount++ + // console.log(`event: ${event}`) + // } + // webSocketServer.close() + // } + // let sleepCounter = 0 + // while (closedCount < this.portForwards.length) { + // await sleep(50) + // console.log(`sleeping.... ${++sleepCounter}`) + // } this.portForwards = [] } } @@ -240,7 +284,7 @@ export class AccountManager { * @param accounts the accounts to update * @returns {Promise} */ - async updateSpecialAccountsKeys (namespace, nodeClient, accounts) { + async updateSpecialAccountsKeys (namespace, nodeClient, accounts, task) { const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard @@ -251,13 +295,16 @@ export class AccountManager { let currentBatch = [] for (const [start, end] of accounts) { for (let i = start; i <= end; i++) { - if (++batchCounter >= batchSize) { + if (batchCounter >= batchSize) { batchSets.push(currentBatch) currentBatch = [] + batchCounter = 0 } + batchCounter++ currentBatch.push(i) } } + batchSets.push(currentBatch) let rejectedCount = 0 let fulfilledCount = 0 @@ -273,21 +320,22 @@ export class AccountManager { await Promise.allSettled(accountUpdatePromiseArray).then((results) => { for (const result of results) { - switch (result.status) { - case 'rejected': - if (result.reason === REASON_SKIPPED) { + switch (result.value.status) { + case REJECTED: + if (result.value.reason === REASON_SKIPPED) { skippedCount++ } else { - this.logger.error(`REJECT: ${result.reason}: ${result.value}`) + this.logger.error(`REJECT: ${result.value.reason}: ${result.value.value}`) rejectedCount++ } break - case 'fulfilled': + case FULFILLED: fulfilledCount++ break } } }) + task.output = `Current counts: [fulfilled: ${fulfilledCount}, skipped: ${skippedCount}, rejected: ${rejectedCount}` } this.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${fulfilledCount}`)) @@ -311,7 +359,7 @@ export class AccountManager { } catch (e) { this.logger.error(`failed to get keys for accountId ${accountId.toString()}, e: ${e.toString()}\n ${e.stack}`) return { - status: 'rejected', + status: REJECTED, reason: REASON_FAILED_TO_GET_KEYS, value: accountId.toString() } @@ -320,7 +368,7 @@ export class AccountManager { if (constants.OPERATOR_PUBLIC_KEY !== keys[0].toString()) { this.logger.debug(`account ${accountId.toString()} can be skipped since it does not have a genesis key`) return { - status: 'rejected', + status: REJECTED, reason: REASON_SKIPPED, value: accountId.toString() } @@ -340,7 +388,7 @@ export class AccountManager { ) { this.logger.error(`failed to create secret for accountId ${accountId.toString()}`) return { - status: 'rejected', + status: REJECTED, reason: REASON_FAILED_TO_CREATE_K8S_S_KEY, value: accountId.toString() } @@ -348,7 +396,7 @@ export class AccountManager { } catch (e) { this.logger.error(`failed to create secret for accountId ${accountId.toString()}, e: ${e.toString()}`) return { - status: 'rejected', + status: REJECTED, reason: REASON_FAILED_TO_CREATE_K8S_S_KEY, value: accountId.toString() } @@ -358,7 +406,7 @@ export class AccountManager { if (!(await this.sendAccountKeyUpdate(accountId, newPrivateKey, nodeClient, genesisKey))) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}`) return { - status: 'rejected', + status: REJECTED, reason: REASON_FAILED_TO_UPDATE_ACCOUNT, value: accountId.toString() } @@ -366,14 +414,14 @@ export class AccountManager { } catch (e) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}, e: ${e.toString()}`) return { - status: 'rejected', + status: REJECTED, reason: REASON_FAILED_TO_UPDATE_ACCOUNT, value: accountId.toString() } } return { - status: 'fulfilled', + status: FULFILLED, value: accountId.toString() } } From ea199a60c455e2cd437fe601e35d5170794bdbc9 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 28 Feb 2024 22:18:29 +0000 Subject: [PATCH 03/20] working version, needed the 20 second sleep Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 1 + src/core/account_manager.mjs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index f93e1475d..58cf91d17 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -506,6 +506,7 @@ export class NodeCommand extends BaseCommand { { title: 'Get the mirror node importer address book', task: async (ctx, _) => { + await sleep(20000) // TODO is this needed? previously there was an extra 20 seconds before I made the check nodes concurrent ctx.addressBook = await self.getAddressBook(ctx.config.namespace) ctx.config.valuesArg += ` --set "hedera-mirror-node.importer.addressBook=${ctx.addressBook}"` } diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 0e49ca915..35ce41b20 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -232,6 +232,15 @@ export class AccountManager { this.logger.debug(`creating client from network configuration: ${JSON.stringify(nodes)}`) const nodeClient = Client.fromConfig({ network: nodes }) + // nodeClient.setMinBackoff(100) overriden by nodeWaitTime + nodeClient.setNodeMinBackoff(100) + nodeClient.setMaxExecutionTime(10000 * 6) + nodeClient.setNodeWaitTime(10000 * 6) + nodeClient.setMaxBackoff(8000 * 6) + nodeClient.setNodeMaxBackoff(8000 * 6) + nodeClient.setMaxAttempts(50) + nodeClient.setMaxNodeAttempts(10) + nodeClient.setRequestTimeout(10000 * 6) nodeClient.setOperator(operatorId, operatorKey) return nodeClient From f82434ae9827558c59a9174c244c1b87ff8c9889 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 29 Feb 2024 15:52:13 +0000 Subject: [PATCH 04/20] current working version Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 10 ++--- src/core/account_manager.mjs | 85 ++++++++++-------------------------- 2 files changed, 26 insertions(+), 69 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 58cf91d17..2650b5a8d 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -60,15 +60,13 @@ export class NodeCommand extends BaseCommand { } } - async checkNetworkNodeStarted (nodeId, maxAttempt = 50, status = 'ACTIVE') { + async checkNetworkNodeStarted (nodeId, maxAttempt = 100, status = 'ACTIVE') { nodeId = nodeId.trim() const podName = Templates.renderNetworkPodName(nodeId) const logfilePath = `${constants.HEDERA_HAPI_PATH}/logs/hgcaa.log` let attempt = 0 let isActive = false - await sleep(10000) // sleep in case this the user ran the start command again at a later time - // check log file is accessible let logFileAccessible = false while (attempt++ < maxAttempt) { @@ -90,7 +88,7 @@ export class NodeCommand extends BaseCommand { while (attempt < maxAttempt) { try { const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-10', logfilePath]) - if (output.indexOf(`Terminating Netty = ${status}`) < 0 && // make sure we are not at the beginning of a restart + if (output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart output.indexOf(`Now current platform status = ${status}`) > 0) { this.logger.debug(`Node ${nodeId} is ${status} [ attempt: ${attempt}/${maxAttempt}]`) isActive = true @@ -492,7 +490,7 @@ export class NodeCommand extends BaseCommand { // set up the sub-tasks return task.newListr(subTasks, { - concurrent: true, + concurrent: false, rendererOptions: { collapseSubtasks: false } @@ -506,7 +504,7 @@ export class NodeCommand extends BaseCommand { { title: 'Get the mirror node importer address book', task: async (ctx, _) => { - await sleep(20000) // TODO is this needed? previously there was an extra 20 seconds before I made the check nodes concurrent + await sleep(15000) // give time for haproxy to detect that the node server is up on grpc port, queries every 10 seconds ctx.addressBook = await self.getAddressBook(ctx.config.namespace) ctx.config.valuesArg += ` --set "hedera-mirror-node.importer.addressBook=${ctx.addressBook}"` } diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 35ce41b20..f6b6ec233 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -42,6 +42,8 @@ const REASON_FAILED_TO_UPDATE_ACCOUNT = 'failed to update account keys' const REASON_FAILED_TO_CREATE_K8S_S_KEY = 'failed to create k8s scrt key' const FULFILLED = 'fulfilled' const REJECTED = 'rejected' +const MAX_PORT_FORWARD_SLEEP_ITERATIONS = 500 +const PORT_FORWARD_CLOSE_SLEEP = 500 /** * Copyright (C) 2024 Hedera Hashgraph, LLC @@ -143,60 +145,24 @@ export class AccountManager { * @returns {Promise} */ async stopPortForwards () { - // const closedCount = 0 + global.accountManagerPortForwardClosedCount = 0 // global variable to be used for WebSocketServer callback + if (this.portForwards) { - // original - this.portForwards.forEach(server => { - server.close() - }) - this.portForwards = [] - await sleep(1) // give a tick for connections to close - // for (const webSocketServer of this.portForwards) { - // Option 1: - // const serverClose = util.promisify(webSocketServer.close) - // await serverClose() - // Option 2: - // await (() => { - // return new Promise((resolve, reject) => { - // webSocketServer.close((err, data) => { - // if (err) return reject(err) - // resolve(data) - // }) - // }) - // })() - // Option 3: - // const closeWebSocket = (callback) => { - // // don't close if it's already closed - // if (webSocketServer.readyState === 3) { - // callback() - // } else { - // // don't notify on user-initiated shutdown ('disconnect' event) - // webSocketServer.removeAllListeners('close') - // webSocketServer.once('close', () => { - // webSocketServer.removeAllListeners() - // callback() - // }) - // webSocketServer.close() - // } - // } - // await new Promise((resolve, reject) => { - // closeWebSocket(resolve) - // }) - // Option 4: - // webSocketServer.addEventListener('close', (event) => { - // console.log('The connection has been closed successfully.') - // }) - // webSocketServer.onclose = (event) => { - // closedCount++ - // console.log(`event: ${event}`) - // } - // webSocketServer.close() - // } - // let sleepCounter = 0 - // while (closedCount < this.portForwards.length) { - // await sleep(50) - // console.log(`sleeping.... ${++sleepCounter}`) - // } + for (const webSocketServer of this.portForwards) { + webSocketServer.close(() => { + global.accountManagerPortForwardClosedCount++ + }) + } + + let sleepCounter = 0 + while (global.accountManagerPortForwardClosedCount < this.portForwards.length && sleepCounter < MAX_PORT_FORWARD_SLEEP_ITERATIONS) { + await sleep(PORT_FORWARD_CLOSE_SLEEP) + this.logger.debug(`waiting ${PORT_FORWARD_CLOSE_SLEEP}ms for port forward server to close .... ${++sleepCounter} of ${MAX_PORT_FORWARD_SLEEP_ITERATIONS}`) + } + if (sleepCounter >= MAX_PORT_FORWARD_SLEEP_ITERATIONS) { + this.logger.error(`failed to detect that all port forward servers closed correctly, only ${global.accountManagerPortForwardClosedCount} of ${this.portForwards.length} reported that they closed`) + } + this.portForwards = [] } } @@ -232,15 +198,6 @@ export class AccountManager { this.logger.debug(`creating client from network configuration: ${JSON.stringify(nodes)}`) const nodeClient = Client.fromConfig({ network: nodes }) - // nodeClient.setMinBackoff(100) overriden by nodeWaitTime - nodeClient.setNodeMinBackoff(100) - nodeClient.setMaxExecutionTime(10000 * 6) - nodeClient.setNodeWaitTime(10000 * 6) - nodeClient.setMaxBackoff(8000 * 6) - nodeClient.setNodeMaxBackoff(8000 * 6) - nodeClient.setMaxAttempts(50) - nodeClient.setMaxNodeAttempts(10) - nodeClient.setRequestTimeout(10000 * 6) nodeClient.setOperator(operatorId, operatorKey) return nodeClient @@ -344,7 +301,9 @@ export class AccountManager { } } }) - task.output = `Current counts: [fulfilled: ${fulfilledCount}, skipped: ${skippedCount}, rejected: ${rejectedCount}` + const message = `Current counts: [fulfilled: ${fulfilledCount}, skipped: ${skippedCount}, rejected: ${rejectedCount}` + task.output = message + this.logger.debug(message) } this.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${fulfilledCount}`)) From 69d91b69bc116f6ff281ca40aa08eb28639f65d4 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 29 Feb 2024 18:23:30 +0000 Subject: [PATCH 05/20] fix for treasury account when running node start, a second time Signed-off-by: Jeromy Cannon --- src/core/account_manager.mjs | 38 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index f6b6ec233..ae6820de1 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -122,22 +122,30 @@ export class AccountManager { /** * Prepares the accounts with updated keys so that they do not contain the default genesis keys * @param namespace the namespace to run the update of account keys for + * @param task the listr2 task so that we can send updates to the user * @returns {Promise} */ - async prepareAccounts (namespace, task) { // TODO update task into jsdoc (multiple places) + async prepareAccounts (namespace, task) { const serviceMap = await this.getNodeServiceMap(namespace) - const nodeClient = await this.getNodeClient( - namespace, serviceMap, constants.OPERATOR_ID, constants.OPERATOR_KEY) + const treasuryAccountInfo = await this.getTreasuryAccountKeys(namespace) - // TODO check to see if any account key secrets exist in the cluster, if they don't we can skip the deleteSecret + const nodeClient = await this.getNodeClient( + namespace, serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.SYSTEM_ACCOUNTS, task) - // update the treasury account last - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.TREASURY_ACCOUNTS, task) + const secrets = await this.k8.getSecretsByLabel(['fullstack.hedera.com/account-id']) + const updateSecrets = secrets.length > 0 - nodeClient.close() - await this.stopPortForwards() + try { + await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.SYSTEM_ACCOUNTS, task, updateSecrets) + // update the treasury account last + await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.TREASURY_ACCOUNTS, task, updateSecrets) + } catch (e) { + this.logger.showUser(e) + } finally { + nodeClient.close() + await this.stopPortForwards() + } } /** @@ -163,6 +171,7 @@ export class AccountManager { this.logger.error(`failed to detect that all port forward servers closed correctly, only ${global.accountManagerPortForwardClosedCount} of ${this.portForwards.length} reported that they closed`) } + delete global.accountManagerPortForwardClosedCount this.portForwards = [] } } @@ -248,9 +257,11 @@ export class AccountManager { * @param namespace the namespace of the nodes network * @param nodeClient the active node client configured to point at the network * @param accounts the accounts to update + * @param task the listr2 task so that we can send updates to the user + * @param updateSecrets whether to delete the secret prior to creating a new secret * @returns {Promise} */ - async updateSpecialAccountsKeys (namespace, nodeClient, accounts, task) { + async updateSpecialAccountsKeys (namespace, nodeClient, accounts, task, updateSecrets) { const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard @@ -281,7 +292,7 @@ export class AccountManager { for (const accountNum of currentSet) { accountUpdatePromiseArray.push(this.updateAccountKeys( - namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey)) + namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey, updateSecrets)) } await Promise.allSettled(accountUpdatePromiseArray).then((results) => { @@ -318,9 +329,10 @@ export class AccountManager { * @param nodeClient the active node client configured to point at the network * @param accountId the account that will get its keys updated * @param genesisKey the genesis key to compare against + * @param updateSecrets whether to delete the secret prior to creating a new secret * @returns {Promise<{value: string, status: string}|{reason: string, value: string, status: string}>} the result of the call */ - async updateAccountKeys (namespace, nodeClient, accountId, genesisKey) { + async updateAccountKeys (namespace, nodeClient, accountId, genesisKey, updateSecrets) { let keys try { keys = await this.getAccountKeys(accountId, nodeClient) @@ -352,7 +364,7 @@ export class AccountManager { if (!(await this.k8.createSecret( Templates.renderAccountKeySecretName(accountId), namespace, 'Opaque', data, - Templates.renderAccountKeySecretLabelObject(accountId), true)) + Templates.renderAccountKeySecretLabelObject(accountId), updateSecrets)) ) { this.logger.error(`failed to create secret for accountId ${accountId.toString()}`) return { From 4111d35e1bc9bb571314845a63de8b0ace578512 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Thu, 29 Feb 2024 18:29:33 +0000 Subject: [PATCH 06/20] #76 Signed-off-by: Jeromy Cannon --- package-lock.json | 1 - package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index c9251b10b..2295260b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,6 @@ }, "devDependencies": { "@jest/globals": "^29.7.0", - "@jest/test-sequencer": "^29.7.0", "eslint": "^8.57.0", "eslint-config-standard": "^17.1.0", "eslint-plugin-headers": "^1.1.2", diff --git a/package.json b/package.json index 8a01cbc01..35b655b22 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,6 @@ }, "devDependencies": { "@jest/globals": "^29.7.0", - "@jest/test-sequencer": "^29.7.0", "eslint": "^8.57.0", "eslint-config-standard": "^17.1.0", "eslint-plugin-headers": "^1.1.2", From e3c691c735611ef2fb8f42cc4da0029e97ed4501 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Fri, 1 Mar 2024 18:36:56 +0000 Subject: [PATCH 07/20] refactored mirror node update for listr2 Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 169 ++++++++++++++++++++++++----------- src/core/account_manager.mjs | 125 ++++++++++---------------- src/core/constants.mjs | 2 +- 3 files changed, 165 insertions(+), 131 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 2650b5a8d..ea45b5695 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -427,16 +427,22 @@ export class NodeCommand extends BaseCommand { cacheDir: self.configManager.getFlag(flags.cacheDir) } - ctx.config.chartPath = await this.prepareChartPath(ctx.config.chartDir, + ctx.config.chartPath = await self.prepareChartPath(ctx.config.chartDir, constants.FULLSTACK_TESTING_CHART, constants.FULLSTACK_DEPLOYMENT_CHART) ctx.config.stagingDir = Templates.renderStagingDir(self.configManager, flags) ctx.config.valuesArg = ` --set hedera-mirror-node.enabled=${ctx.config.deployMirrorNode} --set hedera-explorer.enabled=${ctx.config.deployHederaExplorer}` - if (!await this.k8.hasNamespace(ctx.config.namespace)) { + if (!await self.k8.hasNamespace(ctx.config.namespace)) { throw new FullstackTestingError(`namespace ${ctx.config.namespace} does not exist`) } + + ctx.treasuryAccountInfo = await self.accountManager.getTreasuryAccountKeys(ctx.config.namespace) + const serviceMap = await self.accountManager.getNodeServiceMap(ctx.config.namespace) + + ctx.nodeClient = await self.accountManager.getNodeClient(ctx.config.namespace, + serviceMap, ctx.treasuryAccountInfo.accountId, ctx.treasuryAccountInfo.privateKey) } }, { @@ -500,54 +506,125 @@ export class NodeCommand extends BaseCommand { { title: 'Enable mirror node', task: async (ctx, parentTask) => { - const subTasks = [ - { - title: 'Get the mirror node importer address book', - task: async (ctx, _) => { - await sleep(15000) // give time for haproxy to detect that the node server is up on grpc port, queries every 10 seconds - ctx.addressBook = await self.getAddressBook(ctx.config.namespace) - ctx.config.valuesArg += ` --set "hedera-mirror-node.importer.addressBook=${ctx.addressBook}"` - } - }, - { - title: `Upgrade chart '${constants.FULLSTACK_DEPLOYMENT_CHART}'`, - task: async (ctx, _) => { - await this.chartManager.upgrade( - ctx.config.namespace, - constants.FULLSTACK_DEPLOYMENT_CHART, - ctx.config.chartPath, - ctx.config.valuesArg - ) - } - }, - { - title: 'Waiting for explorer pod to be ready', - task: async (ctx, _) => { - if (ctx.config.deployHederaExplorer) { - await this.k8.waitForPod(constants.POD_STATUS_RUNNING, [ - 'app.kubernetes.io/component=hedera-explorer', 'app.kubernetes.io/name=hedera-explorer' - ], 1, 100) + if (ctx.config.deployMirrorNode) { + const subTasks = [ + { + title: 'Wait for proxies to verify node servers are running', + task: async (ctx, _) => { + await sleep(15000) // give time for haproxy to detect that the node server is up on grpc port, queries every 10 seconds + } + }, + { + title: 'Get the mirror node importer address book', + task: async (ctx, _) => { + ctx.addressBook = await self.getAddressBook(ctx.nodeClient) + ctx.config.valuesArg += ` --set "hedera-mirror-node.importer.addressBook=${ctx.addressBook}"` + } + }, + { + title: `Upgrade chart '${constants.FULLSTACK_DEPLOYMENT_CHART}'`, + task: async (ctx, _) => { + await self.chartManager.upgrade( + ctx.config.namespace, + constants.FULLSTACK_DEPLOYMENT_CHART, + ctx.config.chartPath, + ctx.config.valuesArg + ) + } + }, + { + title: 'Waiting for explorer pod to be ready', + task: async (ctx, _) => { + if (ctx.config.deployHederaExplorer) { + await self.k8.waitForPod(constants.POD_STATUS_RUNNING, [ + 'app.kubernetes.io/component=hedera-explorer', 'app.kubernetes.io/name=hedera-explorer' + ], 1, 100) + } } } - } - ] + ] - return parentTask.newListr(subTasks, { - concurrent: false, - rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION - }) + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: constants.LISTR_DEFAULT_RENDERER_OPTION + }) + } } }, { title: 'Update special account keys', task: async (ctx, task) => { if (ctx.config.updateAccountKeys) { - await self.accountManager.prepareAccounts(ctx.config.namespace, task) + return new Listr([ + { + title: 'Prepare for account key updates', + task: async (ctx) => { + const secrets = await self.k8.getSecretsByLabel(['fullstack.hedera.com/account-id']) + ctx.updateSecrets = secrets.length > 0 + + ctx.accountsBatchedSet = self.accountManager.batchAccounts() + + ctx.resultTracker = { + rejectedCount: 0, + fulfilledCount: 0, + skippedCount: 0 + } + } + }, + { + title: 'Update special account key sets', + task: async (ctx) => { + let setIndex = 1 + const subTasks = [] + for (const currentSet of ctx.accountsBatchedSet) { + subTasks.push({ + title: `Updating set ${chalk.yellow( + setIndex)} of ${chalk.yellow( + ctx.accountsBatchedSet.length)}`, + task: async (ctx) => { + ctx.resultTracker = await self.accountManager.updateSpecialAccountsKeys( + ctx.config.namespace, ctx.nodeClient, currentSet, + ctx.updateSecrets, ctx.resultTracker) + } + }) + setIndex++ + } + + // set up the sub-tasks + return task.newListr(subTasks, { + concurrent: false, + rendererOptions: { + collapseSubtasks: false + } + }) + } + }, + { + title: 'Display results', + task: async (ctx) => { + self.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${ctx.resultTracker.fulfilledCount}`)) + if (ctx.resultTracker.skippedCount > 0) self.logger.showUser(chalk.cyan(`Account keys updates SKIPPED: ${ctx.resultTracker.skippedCount}`)) + if (ctx.resultTracker.rejectedCount > 0) self.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${ctx.resultTracker.rejectedCount}`)) + } + } + ], { + concurrent: false, + rendererOptions: { + collapseSubtasks: false + } + }) } else { - this.logger.showUser(chalk.yellowBright('> WARNING:'), chalk.yellow( + self.logger.showUser(chalk.yellowBright('> WARNING:'), chalk.yellow( 'skipping special account keys update, special accounts will retain genesis private keys')) } } + }, + { + title: 'Close connections', + task: async (ctx, task) => { + ctx.nodeClient.close() + await self.accountManager.stopPortForwards() + } } ], { concurrent: false, @@ -556,6 +633,7 @@ export class NodeCommand extends BaseCommand { try { await tasks.run() + self.logger.debug('node start has completed') } catch (e) { throw new FullstackTestingError(`Error starting node: ${e.message}`, e) } @@ -565,26 +643,15 @@ export class NodeCommand extends BaseCommand { /** * Will get the address book from the network (base64 encoded) - * @param namespace + * @param nodeClient the configured and active NodeClient to use to retrieve the address book * @returns {Promise} the base64 encoded address book for the network */ - async getAddressBook (namespace) { - const treasuryAccountInfo = await this.accountManager.getTreasuryAccountKeys(namespace) - const serviceMap = await this.accountManager.getNodeServiceMap(namespace) - - const nodeClient = await this.accountManager.getNodeClient(namespace, - serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) - + async getAddressBook (nodeClient) { try { // Retrieve the AddressBook as base64 return await this.accountManager.prepareAddressBookBase64(nodeClient) } catch (e) { throw new FullstackTestingError(`an error was encountered while trying to prepare the address book: ${e.message}`, e) - } finally { - await this.accountManager.stopPortForwards() - if (nodeClient) { - nodeClient.close() - } } } @@ -606,7 +673,7 @@ export class NodeCommand extends BaseCommand { nodeIds: helpers.parseNodeIDs(self.configManager.getFlag(flags.nodeIDs)) } - if (!await this.k8.hasNamespace(ctx.config.namespace)) { + if (!await self.k8.hasNamespace(ctx.config.namespace)) { throw new FullstackTestingError(`namespace ${ctx.config.namespace} does not exist`) } } diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index ae6820de1..e2a706544 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -33,7 +33,6 @@ import { import { FullstackTestingError } from './errors.mjs' import { sleep } from './helpers.mjs' import net from 'net' -import chalk from 'chalk' import { Templates } from './templates.mjs' const REASON_FAILED_TO_GET_KEYS = 'failed to get keys for accountId' @@ -119,33 +118,27 @@ export class AccountManager { return accountInfo } - /** - * Prepares the accounts with updated keys so that they do not contain the default genesis keys - * @param namespace the namespace to run the update of account keys for - * @param task the listr2 task so that we can send updates to the user - * @returns {Promise} - */ - async prepareAccounts (namespace, task) { - const serviceMap = await this.getNodeServiceMap(namespace) - - const treasuryAccountInfo = await this.getTreasuryAccountKeys(namespace) - - const nodeClient = await this.getNodeClient( - namespace, serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) - - const secrets = await this.k8.getSecretsByLabel(['fullstack.hedera.com/account-id']) - const updateSecrets = secrets.length > 0 + batchAccounts () { + const batchSize = constants.ACCOUNT_CREATE_BATCH_SIZE + const batchSets = [] - try { - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.SYSTEM_ACCOUNTS, task, updateSecrets) - // update the treasury account last - await this.updateSpecialAccountsKeys(namespace, nodeClient, constants.TREASURY_ACCOUNTS, task, updateSecrets) - } catch (e) { - this.logger.showUser(e) - } finally { - nodeClient.close() - await this.stopPortForwards() + let batchCounter = 0 + let currentBatch = [] + for (const [start, end] of constants.SYSTEM_ACCOUNTS) { + for (let i = start; i <= end; i++) { + if (batchCounter >= batchSize) { + batchSets.push(currentBatch) + currentBatch = [] + batchCounter = 0 + } + batchCounter++ + currentBatch.push(i) + } } + batchSets.push(currentBatch) + batchSets.push([constants.TREASURY_ACCOUNT]) + + return batchSets } /** @@ -256,70 +249,44 @@ export class AccountManager { * Kubernetes secret * @param namespace the namespace of the nodes network * @param nodeClient the active node client configured to point at the network - * @param accounts the accounts to update - * @param task the listr2 task so that we can send updates to the user + * @param currentSet the accounts to update * @param updateSecrets whether to delete the secret prior to creating a new secret - * @returns {Promise} + * @param resultTracker an object to keep track of the results from the accounts that are being updated + * @returns {Promise<*>} the updated resultTracker object */ - async updateSpecialAccountsKeys (namespace, nodeClient, accounts, task, updateSecrets) { + async updateSpecialAccountsKeys (namespace, nodeClient, currentSet, updateSecrets, resultTracker) { const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard - const batchSize = constants.ACCOUNT_CREATE_BATCH_SIZE - const batchSets = [] - - let batchCounter = 0 - let currentBatch = [] - for (const [start, end] of accounts) { - for (let i = start; i <= end; i++) { - if (batchCounter >= batchSize) { - batchSets.push(currentBatch) - currentBatch = [] - batchCounter = 0 - } - batchCounter++ - currentBatch.push(i) - } - } - batchSets.push(currentBatch) - let rejectedCount = 0 - let fulfilledCount = 0 - let skippedCount = 0 + const accountUpdatePromiseArray = [] - for (const currentSet of batchSets) { - const accountUpdatePromiseArray = [] + for (const accountNum of currentSet) { + accountUpdatePromiseArray.push(this.updateAccountKeys( + namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey, updateSecrets)) + } - for (const accountNum of currentSet) { - accountUpdatePromiseArray.push(this.updateAccountKeys( - namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey, updateSecrets)) + await Promise.allSettled(accountUpdatePromiseArray).then((results) => { + for (const result of results) { + switch (result.value.status) { + case REJECTED: + if (result.value.reason === REASON_SKIPPED) { + resultTracker.skippedCount++ + } else { + this.logger.error(`REJECT: ${result.value.reason}: ${result.value.value}`) + resultTracker.rejectedCount++ + } + break + case FULFILLED: + resultTracker.fulfilledCount++ + break + } } + }) - await Promise.allSettled(accountUpdatePromiseArray).then((results) => { - for (const result of results) { - switch (result.value.status) { - case REJECTED: - if (result.value.reason === REASON_SKIPPED) { - skippedCount++ - } else { - this.logger.error(`REJECT: ${result.value.reason}: ${result.value.value}`) - rejectedCount++ - } - break - case FULFILLED: - fulfilledCount++ - break - } - } - }) - const message = `Current counts: [fulfilled: ${fulfilledCount}, skipped: ${skippedCount}, rejected: ${rejectedCount}` - task.output = message - this.logger.debug(message) - } + this.logger.debug(`Current counts: [fulfilled: ${resultTracker.fulfilledCount}, skipped: ${resultTracker.skippedCount}, rejected: ${resultTracker.rejectedCount}`) - this.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${fulfilledCount}`)) - if (skippedCount > 0) this.logger.showUser(chalk.cyan(`Account keys updates SKIPPED: ${skippedCount}`)) - if (rejectedCount > 0) this.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${rejectedCount}`)) + return resultTracker } /** diff --git a/src/core/constants.mjs b/src/core/constants.mjs index c6aadf1d0..69929c39e 100644 --- a/src/core/constants.mjs +++ b/src/core/constants.mjs @@ -79,7 +79,7 @@ export const OPERATOR_PUBLIC_KEY = process.env.SOLO_OPERATOR_PUBLIC_KEY || '302a export const TREASURY_ACCOUNT_ID = `${HEDERA_NODE_ACCOUNT_ID_START.realm}.${HEDERA_NODE_ACCOUNT_ID_START.shard}.2` export const GENESIS_KEY = process.env.GENESIS_KEY || '302e020100300506032b65700422042091132178e72057a1d7528025956fe39b0b847f200ab59b2fdd367017f3087137' export const SYSTEM_ACCOUNTS = [[3, 100], [200, 349], [400, 750], [900, 1000]] // do account 0.0.2 last and outside the loop -export const TREASURY_ACCOUNTS = [[2, 2]] +export const TREASURY_ACCOUNT = 2 export const LOCAL_NODE_START_PORT = process.env.LOCAL_NODE_START_PORT || 30212 export const ACCOUNT_CREATE_BATCH_SIZE = process.env.ACCOUNT_CREATE_BATCH_SIZE || 50 From d142d8f15a3a51c1f2e55a330d0f900c3cb1f319 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 12:48:25 +0000 Subject: [PATCH 08/20] updates based on review comments Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 9 +++++---- src/core/constants.mjs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index ea45b5695..5a953735f 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -511,18 +511,19 @@ export class NodeCommand extends BaseCommand { { title: 'Wait for proxies to verify node servers are running', task: async (ctx, _) => { + // TODO change this into a wait for haproxy logs to show server is up await sleep(15000) // give time for haproxy to detect that the node server is up on grpc port, queries every 10 seconds } }, { - title: 'Get the mirror node importer address book', + title: 'Prepare address book', task: async (ctx, _) => { ctx.addressBook = await self.getAddressBook(ctx.nodeClient) ctx.config.valuesArg += ` --set "hedera-mirror-node.importer.addressBook=${ctx.addressBook}"` } }, { - title: `Upgrade chart '${constants.FULLSTACK_DEPLOYMENT_CHART}'`, + title: 'Deploy mirror node', task: async (ctx, _) => { await self.chartManager.upgrade( ctx.config.namespace, @@ -533,12 +534,12 @@ export class NodeCommand extends BaseCommand { } }, { - title: 'Waiting for explorer pod to be ready', + title: 'Waiting for Hedera Explorer to be ready', task: async (ctx, _) => { if (ctx.config.deployHederaExplorer) { await self.k8.waitForPod(constants.POD_STATUS_RUNNING, [ 'app.kubernetes.io/component=hedera-explorer', 'app.kubernetes.io/name=hedera-explorer' - ], 1, 100) + ], 1, 200) } } } diff --git a/src/core/constants.mjs b/src/core/constants.mjs index 69929c39e..ca2d3452e 100644 --- a/src/core/constants.mjs +++ b/src/core/constants.mjs @@ -81,7 +81,7 @@ export const GENESIS_KEY = process.env.GENESIS_KEY || '302e020100300506032b65700 export const SYSTEM_ACCOUNTS = [[3, 100], [200, 349], [400, 750], [900, 1000]] // do account 0.0.2 last and outside the loop export const TREASURY_ACCOUNT = 2 export const LOCAL_NODE_START_PORT = process.env.LOCAL_NODE_START_PORT || 30212 -export const ACCOUNT_CREATE_BATCH_SIZE = process.env.ACCOUNT_CREATE_BATCH_SIZE || 50 +export const ACCOUNT_CREATE_BATCH_SIZE = process.env.ACCOUNT_CREATE_BATCH_SIZE || 25 export const POD_STATUS_RUNNING = 'Running' export const POD_STATUS_READY = 'Ready' From bcef0c7a7ddc1b630081b0e52a4b420da2250c6d Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 21:00:17 +0000 Subject: [PATCH 09/20] fixed issues Signed-off-by: Jeromy Cannon --- package-lock.json | 6 ++ package.json | 1 + src/commands/account.mjs | 37 +++------ src/commands/node.mjs | 71 ++++++++++++---- src/core/account_manager.mjs | 128 ++++++++++++++++++----------- src/core/k8.mjs | 2 + test/e2e/commands/01_node.test.mjs | 17 ++-- 7 files changed, 165 insertions(+), 97 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2295260b8..30fffaa70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "got": "^14.2.0", "inquirer": "^9.2.15", "js-base64": "^3.7.7", + "killable": "^1.0.1", "listr2": "^8.0.2", "tar": "^6.2.0", "uuid": "^9.0.1", @@ -7249,6 +7250,11 @@ "json-buffer": "3.0.1" } }, + "node_modules/killable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/killable/-/killable-1.0.1.tgz", + "integrity": "sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==" + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", diff --git a/package.json b/package.json index 35b655b22..f9ca7f284 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "got": "^14.2.0", "inquirer": "^9.2.15", "js-base64": "^3.7.7", + "killable": "^1.0.1", "listr2": "^8.0.2", "tar": "^6.2.0", "uuid": "^9.0.1", diff --git a/src/commands/account.mjs b/src/commands/account.mjs index dad5e89fe..ab8544f51 100644 --- a/src/commands/account.mjs +++ b/src/commands/account.mjs @@ -29,15 +29,11 @@ export class AccountCommand extends BaseCommand { if (!opts || !opts.accountManager) throw new IllegalArgumentError('An instance of core/AccountManager is required', opts.accountManager) this.accountManager = opts.accountManager - this.nodeClient = null this.accountInfo = null } async closeConnections () { - if (this.nodeClient) { - this.nodeClient.close() - } - await this.accountManager.stopPortForwards() + await this.accountManager.close() } async buildAccountInfo (accountInfo, namespace, shouldRetrievePrivateKey) { @@ -63,25 +59,17 @@ export class AccountCommand extends BaseCommand { } return await this.accountManager.createNewAccount(ctx.config.namespace, - ctx.nodeClient, ctx.privateKey, ctx.config.amount) - } - - async loadNodeClient (ctx) { - const serviceMap = await this.accountManager.getNodeServiceMap(ctx.config.namespace) - - ctx.nodeClient = await this.accountManager.getNodeClient(ctx.config.namespace, - serviceMap, ctx.treasuryAccountInfo.accountId, ctx.treasuryAccountInfo.privateKey) - this.nodeClient = ctx.nodeClient // store in class so that we can make sure connections are closed + ctx.privateKey, ctx.config.amount) } async getAccountInfo (ctx) { - return this.accountManager.accountInfoQuery(ctx.config.accountId, ctx.nodeClient) + return this.accountManager.accountInfoQuery(ctx.config.accountId) } async updateAccountInfo (ctx) { let amount = ctx.config.amount if (ctx.config.privateKey) { - if (!(await this.accountManager.sendAccountKeyUpdate(ctx.accountInfo.accountId, ctx.config.privateKey, ctx.nodeClient, ctx.accountInfo.privateKey))) { + if (!(await this.accountManager.sendAccountKeyUpdate(ctx.accountInfo.accountId, ctx.config.privateKey, ctx.accountInfo.privateKey))) { this.logger.error(`failed to update account keys for accountId ${ctx.accountInfo.accountId}`) return false } @@ -95,7 +83,7 @@ export class AccountCommand extends BaseCommand { } if (hbarAmount > 0) { - if (!(await this.transferAmountFromOperator(ctx.nodeClient, ctx.accountInfo.accountId, hbarAmount))) { + if (!(await this.transferAmountFromOperator(ctx.accountInfo.accountId, hbarAmount))) { this.logger.error(`failed to transfer amount for accountId ${ctx.accountInfo.accountId}`) return false } @@ -104,8 +92,8 @@ export class AccountCommand extends BaseCommand { return true } - async transferAmountFromOperator (nodeClient, toAccountId, amount) { - return await this.accountManager.transferAmount(nodeClient, constants.TREASURY_ACCOUNT_ID, toAccountId, amount) + async transferAmountFromOperator (toAccountId, amount) { + return await this.accountManager.transferAmount(constants.TREASURY_ACCOUNT_ID, toAccountId, amount) } async create (argv) { @@ -139,8 +127,7 @@ export class AccountCommand extends BaseCommand { self.logger.debug('Initialized config', { config }) - ctx.treasuryAccountInfo = await self.accountManager.getTreasuryAccountKeys(ctx.config.namespace) - await self.loadNodeClient(ctx) + await self.accountManager.loadNodeClient(ctx.config.namespace) } }, { @@ -196,14 +183,14 @@ export class AccountCommand extends BaseCommand { // set config in the context for later tasks to use ctx.config = config + await self.accountManager.loadNodeClient(config.namespace) + self.logger.debug('Initialized config', { config }) } }, { title: 'get the account info', task: async (ctx, task) => { - ctx.treasuryAccountInfo = await self.accountManager.getTreasuryAccountKeys(ctx.config.namespace) - await self.loadNodeClient(ctx) ctx.accountInfo = await self.buildAccountInfo(await self.getAccountInfo(ctx), ctx.config.namespace, ctx.config.privateKey) } }, @@ -263,14 +250,14 @@ export class AccountCommand extends BaseCommand { // set config in the context for later tasks to use ctx.config = config + await self.accountManager.loadNodeClient(config.namespace) + self.logger.debug('Initialized config', { config }) } }, { title: 'get the account info', task: async (ctx, task) => { - ctx.treasuryAccountInfo = await self.accountManager.getTreasuryAccountKeys(ctx.config.namespace) - await self.loadNodeClient(ctx) self.accountInfo = await self.buildAccountInfo(await self.getAccountInfo(ctx), ctx.config.namespace, false) this.logger.showJSON('account info', self.accountInfo) } diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 5a953735f..942a185eb 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -88,7 +88,7 @@ export class NodeCommand extends BaseCommand { while (attempt < maxAttempt) { try { const output = await this.k8.execContainer(podName, constants.ROOT_CONTAINER, ['tail', '-10', logfilePath]) - if (output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart + if (output && output.indexOf('Terminating Netty') < 0 && // make sure we are not at the beginning of a restart output.indexOf(`Now current platform status = ${status}`) > 0) { this.logger.debug(`Node ${nodeId} is ${status} [ attempt: ${attempt}/${maxAttempt}]`) isActive = true @@ -438,11 +438,7 @@ export class NodeCommand extends BaseCommand { throw new FullstackTestingError(`namespace ${ctx.config.namespace} does not exist`) } - ctx.treasuryAccountInfo = await self.accountManager.getTreasuryAccountKeys(ctx.config.namespace) - const serviceMap = await self.accountManager.getNodeServiceMap(ctx.config.namespace) - - ctx.nodeClient = await self.accountManager.getNodeClient(ctx.config.namespace, - serviceMap, ctx.treasuryAccountInfo.accountId, ctx.treasuryAccountInfo.privateKey) + await self.accountManager.loadNodeClient(ctx.config.namespace) } }, { @@ -511,8 +507,21 @@ export class NodeCommand extends BaseCommand { { title: 'Wait for proxies to verify node servers are running', task: async (ctx, _) => { - // TODO change this into a wait for haproxy logs to show server is up - await sleep(15000) // give time for haproxy to detect that the node server is up on grpc port, queries every 10 seconds + const subTasks = [] + for (const nodeId of ctx.config.nodeIds) { + subTasks.push({ + title: `Check node proxy: ${chalk.yellow(nodeId)}`, + task: async () => await self.checkNetworkNodeProxyUp(ctx.config.namespace, nodeId) + }) + } + + // set up the sub-tasks + return parentTask.newListr(subTasks, { + concurrent: false, + rendererOptions: { + collapseSubtasks: false + } + }) } }, { @@ -584,7 +593,7 @@ export class NodeCommand extends BaseCommand { ctx.accountsBatchedSet.length)}`, task: async (ctx) => { ctx.resultTracker = await self.accountManager.updateSpecialAccountsKeys( - ctx.config.namespace, ctx.nodeClient, currentSet, + ctx.config.namespace, currentSet, ctx.updateSecrets, ctx.resultTracker) } }) @@ -605,7 +614,10 @@ export class NodeCommand extends BaseCommand { task: async (ctx) => { self.logger.showUser(chalk.green(`Account keys updated SUCCESSFULLY: ${ctx.resultTracker.fulfilledCount}`)) if (ctx.resultTracker.skippedCount > 0) self.logger.showUser(chalk.cyan(`Account keys updates SKIPPED: ${ctx.resultTracker.skippedCount}`)) - if (ctx.resultTracker.rejectedCount > 0) self.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${ctx.resultTracker.rejectedCount}`)) + if (ctx.resultTracker.rejectedCount > 0) { + self.logger.showUser(chalk.yellowBright(`Account keys updates with ERROR: ${ctx.resultTracker.rejectedCount}`)) + throw new FullstackTestingError(`Account keys updates failed for ${ctx.resultTracker.rejectedCount} accounts, exiting`) + } } } ], { @@ -619,13 +631,6 @@ export class NodeCommand extends BaseCommand { 'skipping special account keys update, special accounts will retain genesis private keys')) } } - }, - { - title: 'Close connections', - task: async (ctx, task) => { - ctx.nodeClient.close() - await self.accountManager.stopPortForwards() - } } ], { concurrent: false, @@ -637,11 +642,43 @@ export class NodeCommand extends BaseCommand { self.logger.debug('node start has completed') } catch (e) { throw new FullstackTestingError(`Error starting node: ${e.message}`, e) + } finally { + await self.accountManager.close() } return true } + async checkNetworkNodeProxyUp (namespace, nodeId, maxAttempts = 100) { + const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) + + if (podArray.length > 0) { + const podName = podArray[0].metadata.name + + let attempts = 0 + while (attempts < maxAttempts) { + const logResponse = await this.k8.kubeClient.readNamespacedPodLog( + podName, namespace) + + if (logResponse.response.statusCode !== 200) { + throw new FullstackTestingError(`Expected pod ${podName} log query to execute successful, but instead got a status of ${logResponse.response.statusCode}`) + } + + if (logResponse.body.includes('Server be_servers/server1 is UP')) { + return true + } + + attempts++ + this.logger.debug(`Checking for pod ${podName} to realize network node is UP [attempt: ${attempts}/${maxAttempts}]`) + await sleep(1000) + } + } else { + throw new FullstackTestingError('TBD') + } + + return false + } + /** * Will get the address book from the network (base64 encoded) * @param nodeClient the configured and active NodeClient to use to retrieve the address book diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index e2a706544..61f18e67e 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -61,6 +61,8 @@ const PORT_FORWARD_CLOSE_SLEEP = 500 * */ export class AccountManager { + static _openPortForwardConnections = 0 + /** * creates a new AccountManager instance * @param logger the logger to use @@ -72,7 +74,8 @@ export class AccountManager { this.logger = logger this.k8 = k8 - this.portForwards = [] + this._portForwards = [] + this._nodeClient = null } /** @@ -126,49 +129,78 @@ export class AccountManager { let currentBatch = [] for (const [start, end] of constants.SYSTEM_ACCOUNTS) { for (let i = start; i <= end; i++) { - if (batchCounter >= batchSize) { + currentBatch.push(i) + batchCounter++ + + // push only a single account for the first to give time for the + // network to create all the system accounts and become stable + if (batchCounter >= batchSize || batchSets.length < 1) { batchSets.push(currentBatch) currentBatch = [] batchCounter = 0 } - batchCounter++ - currentBatch.push(i) } } - batchSets.push(currentBatch) + + if (currentBatch.length > 0) { + batchSets.push(currentBatch) + } + batchSets.push([constants.TREASURY_ACCOUNT]) return batchSets } /** - * stops and closes all of the port forwards that are running + * stops and closes the port forwards and the _nodeClient * @returns {Promise} */ - async stopPortForwards () { - global.accountManagerPortForwardClosedCount = 0 // global variable to be used for WebSocketServer callback + async close () { + if (this._nodeClient) { + this._nodeClient.close() + this._nodeClient = null + } + await this._stopPortForwards() + } - if (this.portForwards) { - for (const webSocketServer of this.portForwards) { - webSocketServer.close(() => { - global.accountManagerPortForwardClosedCount++ + /** + * stops and closes all of the port forwards that are running + * @returns {Promise} + */ + async _stopPortForwards () { + if (this._portForwards) { + for (const webSocketServer of this._portForwards) { + webSocketServer.kill(() => { + AccountManager._openPortForwardConnections-- }) } let sleepCounter = 0 - while (global.accountManagerPortForwardClosedCount < this.portForwards.length && sleepCounter < MAX_PORT_FORWARD_SLEEP_ITERATIONS) { + while (AccountManager._openPortForwardConnections > 0 && sleepCounter < MAX_PORT_FORWARD_SLEEP_ITERATIONS) { await sleep(PORT_FORWARD_CLOSE_SLEEP) this.logger.debug(`waiting ${PORT_FORWARD_CLOSE_SLEEP}ms for port forward server to close .... ${++sleepCounter} of ${MAX_PORT_FORWARD_SLEEP_ITERATIONS}`) } if (sleepCounter >= MAX_PORT_FORWARD_SLEEP_ITERATIONS) { - this.logger.error(`failed to detect that all port forward servers closed correctly, only ${global.accountManagerPortForwardClosedCount} of ${this.portForwards.length} reported that they closed`) + this.logger.error(`failed to detect that all port forward servers closed correctly, ${AccountManager._openPortForwardConnections} of ${this._portForwards.length} remain open`) } - delete global.accountManagerPortForwardClosedCount - this.portForwards = [] + this._portForwards = [] } } + /** + * loads and initializes the Node Client + * @param namespace the namespace of the network + * @returns {Promise} + */ + async loadNodeClient (namespace) { + const treasuryAccountInfo = await this.getTreasuryAccountKeys(namespace) + const serviceMap = await this.getNodeServiceMap(namespace) + + this._nodeClient = await this._getNodeClient(namespace, + serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) + } + /** * Returns a node client that can be used to make calls against * @param namespace the namespace for which the node client resides @@ -177,7 +209,7 @@ export class AccountManager { * @param operatorKey the private key of the operator of the transactions * @returns {Promise} a node client that can be used to call transactions */ - async getNodeClient (namespace, serviceMap, operatorId, operatorKey) { + async _getNodeClient (namespace, serviceMap, operatorId, operatorKey) { const nodes = {} try { let localPort = constants.LOCAL_NODE_START_PORT @@ -189,7 +221,8 @@ export class AccountManager { const targetPort = usePortForward ? localPort : port if (usePortForward) { - this.portForwards.push(await this.k8.portForward(serviceObject.podName, localPort, port)) + this._portForwards.push(await this.k8.portForward(serviceObject.podName, localPort, port)) + AccountManager._openPortForwardConnections++ } nodes[`${host}:${targetPort}`] = AccountId.fromString(serviceObject.accountId) @@ -199,10 +232,10 @@ export class AccountManager { } this.logger.debug(`creating client from network configuration: ${JSON.stringify(nodes)}`) - const nodeClient = Client.fromConfig({ network: nodes }) - nodeClient.setOperator(operatorId, operatorKey) + this._nodeClient = Client.fromConfig({ network: nodes }) + this._nodeClient.setOperator(operatorId, operatorKey) - return nodeClient + return this._nodeClient } catch (e) { throw new FullstackTestingError(`failed to setup node client: ${e.message}`, e) } @@ -248,13 +281,12 @@ export class AccountManager { * updates a set of special accounts keys with a newly generated key and stores them in a * Kubernetes secret * @param namespace the namespace of the nodes network - * @param nodeClient the active node client configured to point at the network * @param currentSet the accounts to update * @param updateSecrets whether to delete the secret prior to creating a new secret * @param resultTracker an object to keep track of the results from the accounts that are being updated * @returns {Promise<*>} the updated resultTracker object */ - async updateSpecialAccountsKeys (namespace, nodeClient, currentSet, updateSecrets, resultTracker) { + async updateSpecialAccountsKeys (namespace, currentSet, updateSecrets, resultTracker) { const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard @@ -263,7 +295,7 @@ export class AccountManager { for (const accountNum of currentSet) { accountUpdatePromiseArray.push(this.updateAccountKeys( - namespace, nodeClient, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey, updateSecrets)) + namespace, AccountId.fromString(`${realm}.${shard}.${accountNum}`), genesisKey, updateSecrets)) } await Promise.allSettled(accountUpdatePromiseArray).then((results) => { @@ -293,16 +325,16 @@ export class AccountManager { * update the account keys for a given account and store its new key in a Kubernetes * secret * @param namespace the namespace of the nodes network - * @param nodeClient the active node client configured to point at the network * @param accountId the account that will get its keys updated * @param genesisKey the genesis key to compare against * @param updateSecrets whether to delete the secret prior to creating a new secret * @returns {Promise<{value: string, status: string}|{reason: string, value: string, status: string}>} the result of the call */ - async updateAccountKeys (namespace, nodeClient, accountId, genesisKey, updateSecrets) { + async updateAccountKeys (namespace, accountId, genesisKey, updateSecrets) { let keys try { - keys = await this.getAccountKeys(accountId, nodeClient) + this.logger.debug(`getAccountKeys: account ${accountId.toString()} begin`) + keys = await this.getAccountKeys(accountId) } catch (e) { this.logger.error(`failed to get keys for accountId ${accountId.toString()}, e: ${e.toString()}\n ${e.stack}`) return { @@ -350,7 +382,8 @@ export class AccountManager { } try { - if (!(await this.sendAccountKeyUpdate(accountId, newPrivateKey, nodeClient, genesisKey))) { + this.logger.debug(`sendAccountKeyUpdate: account ${accountId.toString()} begin...`) + if (!(await this.sendAccountKeyUpdate(accountId, newPrivateKey, genesisKey))) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}`) return { status: REJECTED, @@ -358,6 +391,7 @@ export class AccountManager { value: accountId.toString() } } + this.logger.debug(`sendAccountKeyUpdate: account ${accountId.toString()} ...end`) } catch (e) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}, e: ${e.toString()}`) return { @@ -376,23 +410,21 @@ export class AccountManager { /** * gets the account info from Hedera network * @param accountId the account - * @param nodeClient the active and configured node client * @returns {AccountInfo} the private key of the account */ - async accountInfoQuery (accountId, nodeClient) { + async accountInfoQuery (accountId) { return await new AccountInfoQuery() .setAccountId(accountId) - .execute(nodeClient) + .execute(this._nodeClient) } /** * gets the account private and public key from the Kubernetes secret from which it is stored * @param accountId the account - * @param nodeClient the active and configured node client * @returns {Promise} the private key of the account */ - async getAccountKeys (accountId, nodeClient) { - const accountInfo = await this.accountInfoQuery(accountId, nodeClient) + async getAccountKeys (accountId) { + const accountInfo = await this.accountInfoQuery(accountId) let keys if (accountInfo.key instanceof KeyList) { @@ -409,11 +441,10 @@ export class AccountManager { * send an account key update transaction to the network of nodes * @param accountId the account that will get it's keys updated * @param newPrivateKey the new private key - * @param nodeClient the active and configured node client * @param oldPrivateKey the genesis key that is the current key * @returns {Promise} whether the update was successful */ - async sendAccountKeyUpdate (accountId, newPrivateKey, nodeClient, oldPrivateKey) { + async sendAccountKeyUpdate (accountId, newPrivateKey, oldPrivateKey) { if (typeof newPrivateKey === 'string') { newPrivateKey = PrivateKey.fromStringED25519(newPrivateKey) } @@ -426,17 +457,17 @@ export class AccountManager { const transaction = await new AccountUpdateTransaction() .setAccountId(accountId) .setKey(newPrivateKey.publicKey) - .freezeWith(nodeClient) + .freezeWith(this._nodeClient) // Sign the transaction with the old key and new key const signTx = await (await transaction.sign(oldPrivateKey)).sign( newPrivateKey) // SIgn the transaction with the client operator private key and submit to a Hedera network - const txResponse = await signTx.execute(nodeClient) + const txResponse = await signTx.execute(this._nodeClient) // Request the receipt of the transaction - const receipt = await txResponse.getReceipt(nodeClient) + const receipt = await txResponse.getReceipt(this._nodeClient) return receipt.status === Status.Success } @@ -472,20 +503,19 @@ export class AccountManager { /** * creates a new Hedera account * @param namespace the namespace to store the Kubernetes key secret into - * @param nodeClient the active and network configured node client * @param privateKey the private key of type PrivateKey * @param amount the amount of HBAR to add to the account * @returns {{accountId: AccountId, privateKey: string, publicKey: string, balance: number}} a * custom object with the account information in it */ - async createNewAccount (namespace, nodeClient, privateKey, amount) { + async createNewAccount (namespace, privateKey, amount) { const newAccount = await new AccountCreateTransaction() .setKey(privateKey) .setInitialBalance(Hbar.from(amount, HbarUnit.Hbar)) - .execute(nodeClient) + .execute(this._nodeClient) // Get the new account ID - const getReceipt = await newAccount.getReceipt(nodeClient) + const getReceipt = await newAccount.getReceipt(this._nodeClient) const accountInfo = { accountId: getReceipt.accountId.toString(), privateKey: privateKey.toString(), @@ -511,21 +541,20 @@ export class AccountManager { /** * transfer the specified amount of HBAR from one account to another - * @param nodeClient the configured and active network node client * @param fromAccountId the account to pull the HBAR from * @param toAccountId the account to put the HBAR * @param hbarAmount the amount of HBAR * @returns {Promise} if the transaction was successfully posted */ - async transferAmount (nodeClient, fromAccountId, toAccountId, hbarAmount) { + async transferAmount (fromAccountId, toAccountId, hbarAmount) { try { const transaction = new TransferTransaction() .addHbarTransfer(fromAccountId, new Hbar(-1 * hbarAmount)) .addHbarTransfer(toAccountId, new Hbar(hbarAmount)) - const txResponse = await transaction.execute(nodeClient) + const txResponse = await transaction.execute(this._nodeClient) - const receipt = await txResponse.getReceipt(nodeClient) + const receipt = await txResponse.getReceipt(this._nodeClient) this.logger.debug(`The transfer from account ${fromAccountId} to account ${toAccountId} for amount ${hbarAmount} was ${receipt.status.toString()} `) @@ -539,13 +568,12 @@ export class AccountManager { /** * Fetch and prepare address book as a base64 string - * @param nodeClient node client * @return {Promise} */ - async prepareAddressBookBase64 (nodeClient) { + async prepareAddressBookBase64 () { // fetch AddressBook const fileQuery = new FileContentsQuery().setFileId(FileId.ADDRESS_BOOK) - let addressBookBytes = await fileQuery.execute(nodeClient) + let addressBookBytes = await fileQuery.execute(this._nodeClient) // ensure serviceEndpoint.ipAddressV4 value for all nodes in the addressBook is a 4 bytes array instead of string // See: https://github.com/hashgraph/hedera-protobufs/blob/main/services/basic_types.proto#L1309 diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 04c832be1..082a9c047 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -25,6 +25,7 @@ import * as sb from 'stream-buffers' import * as tar from 'tar' import { v4 as uuid4 } from 'uuid' import { V1ObjectMeta, V1Secret } from '@kubernetes/client-node' +import killable from 'killable' /** * A kubernetes API wrapper class providing custom functionalities required by solo @@ -669,6 +670,7 @@ export class K8 { forwarder.portForward(ns, podName, [podPort], socket, null, socket) }) + killable(server) return server.listen(localPort, '127.0.0.1') } diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index 9f00290ad..e850e36f5 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -54,7 +54,7 @@ class TestHelper { 'this implies that node start did not finish the accountManager.prepareAccounts successfully') } const serviceMap = await accountManager.getNodeServiceMap(argv[flags.namespace.name]) - return await accountManager.getNodeClient(argv[flags.namespace.name], serviceMap, operator.accountId, operator.privateKey) + return await accountManager._getNodeClient(argv[flags.namespace.name], serviceMap, operator.accountId, operator.privateKey) } } @@ -155,10 +155,7 @@ describe.each([ }) afterAll(() => { - if (client) { - client.close() - } - accountManager.stopPortForwards().then().catch() + accountManager.close().then().catch() sleep(100).then().catch() }) @@ -227,5 +224,15 @@ describe.each([ await accountManager.stopPortForwards() } }, 20000) + + it('checkNetworkNodeProxyUp should succeed', async () => { + expect.assertions(1) + try { + await expect(nodeCmd.checkNetworkNodeProxyUp('solo-e2e', 'node0')).resolves.toBeTruthy() + } catch (e) { + nodeCmd.logger.showUserError(e) + expect(e).toBeNull() + } + }, 20000) }) }) From c31c71d42f2f1ad07f41bd4630b8c6a32597e2ab Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 21:33:15 +0000 Subject: [PATCH 10/20] cleanup and prep for review Signed-off-by: Jeromy Cannon --- src/core/account_manager.mjs | 13 ++++++------- test/e2e/commands/01_node.test.mjs | 12 +++--------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 61f18e67e..18d95c2be 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -121,6 +121,10 @@ export class AccountManager { return accountInfo } + /** + * batch up the accounts into sets to be processed + * @returns {*[]} an array of arrays of numbers representing the accounts to update + */ batchAccounts () { const batchSize = constants.ACCOUNT_CREATE_BATCH_SIZE const batchSets = [] @@ -156,10 +160,8 @@ export class AccountManager { * @returns {Promise} */ async close () { - if (this._nodeClient) { - this._nodeClient.close() - this._nodeClient = null - } + this._nodeClient?.close() + this._nodeClient = null await this._stopPortForwards() } @@ -333,7 +335,6 @@ export class AccountManager { async updateAccountKeys (namespace, accountId, genesisKey, updateSecrets) { let keys try { - this.logger.debug(`getAccountKeys: account ${accountId.toString()} begin`) keys = await this.getAccountKeys(accountId) } catch (e) { this.logger.error(`failed to get keys for accountId ${accountId.toString()}, e: ${e.toString()}\n ${e.stack}`) @@ -382,7 +383,6 @@ export class AccountManager { } try { - this.logger.debug(`sendAccountKeyUpdate: account ${accountId.toString()} begin...`) if (!(await this.sendAccountKeyUpdate(accountId, newPrivateKey, genesisKey))) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}`) return { @@ -391,7 +391,6 @@ export class AccountManager { value: accountId.toString() } } - this.logger.debug(`sendAccountKeyUpdate: account ${accountId.toString()} ...end`) } catch (e) { this.logger.error(`failed to update account keys for accountId ${accountId.toString()}, e: ${e.toString()}`) return { diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index e850e36f5..cc3144ee5 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -164,7 +164,7 @@ describe.each([ it(`special account ${i} should not have genesis key`, async () => { const accountId = `${realm}.${shard}.${i}` nodeCmd.logger.info(`getAccountKeys: accountId ${accountId}`) - const keys = await accountManager.getAccountKeys(accountId, client) + const keys = await accountManager.getAccountKeys(accountId) expect(keys[0].toString()).not.toEqual(genesisKey.toString()) }, 60000) } @@ -187,10 +187,7 @@ describe.each([ nodeCmd.logger.showUserError(e) expect(e).toBeNull() } finally { - if (client) { - client.close() - } - await accountManager.stopPortForwards() + await accountManager.close() await sleep(100) } }, 20000) @@ -218,10 +215,7 @@ describe.each([ nodeCmd.logger.showUserError(e) expect(e).toBeNull() } finally { - if (client) { - client.close() - } - await accountManager.stopPortForwards() + await accountManager.close() } }, 20000) From 1f5fa29c5886ceebed14b166298ff061b6acee41 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 21:35:16 +0000 Subject: [PATCH 11/20] fixed debug log message Signed-off-by: Jeromy Cannon --- src/core/account_manager.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index 18d95c2be..be6e7fefe 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -318,7 +318,7 @@ export class AccountManager { } }) - this.logger.debug(`Current counts: [fulfilled: ${resultTracker.fulfilledCount}, skipped: ${resultTracker.skippedCount}, rejected: ${resultTracker.rejectedCount}`) + this.logger.debug(`Current counts: [fulfilled: ${resultTracker.fulfilledCount}, skipped: ${resultTracker.skippedCount}, rejected: ${resultTracker.rejectedCount}]`) return resultTracker } From 043778ac708a0768dce2288d9e1a7c69c26d6efa Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 21:44:18 +0000 Subject: [PATCH 12/20] fix lint issue Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index cc3144ee5..e984d5901 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -145,15 +145,10 @@ describe.each([ }, 600000) describe('only genesis account should have genesis key for all special accounts', () => { - let client = null const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard - beforeAll(async () => { - client = await TestHelper.getNodeClient(accountManager, argv) - }) - afterAll(() => { accountManager.close().then().catch() sleep(100).then().catch() @@ -216,6 +211,7 @@ describe.each([ expect(e).toBeNull() } finally { await accountManager.close() + await sleep(100) } }, 20000) From e50b36ebc6f0f0367544ffcaf269742a60707a5b Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 21:50:56 +0000 Subject: [PATCH 13/20] fix lint issue Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index e984d5901..77ff1e40f 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -22,7 +22,7 @@ import { } from '@hashgraph/sdk' import { afterAll, - afterEach, beforeAll, + afterEach, describe, expect, it From 71e50f48fd30f6e1ee74d70df02decd6a9fecadd Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Mon, 4 Mar 2024 22:17:50 +0000 Subject: [PATCH 14/20] fixed test case Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 40 ++++++++---------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index 77ff1e40f..d309b37d6 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -22,7 +22,7 @@ import { } from '@hashgraph/sdk' import { afterAll, - afterEach, + afterEach, beforeAll, describe, expect, it @@ -46,18 +46,6 @@ import { getTestCacheDir, testLogger } from '../../test_util.js' import { AccountManager } from '../../../src/core/account_manager.mjs' import { sleep } from '../../../src/core/helpers.mjs' -class TestHelper { - static async getNodeClient (accountManager, argv) { - const operator = await accountManager.getAccountKeysFromSecret(constants.OPERATOR_ID, argv[flags.namespace.name]) - if (!operator) { - throw new Error(`account key not found for operator ${constants.OPERATOR_ID} during getNodeClient()\n` + - 'this implies that node start did not finish the accountManager.prepareAccounts successfully') - } - const serviceMap = await accountManager.getNodeServiceMap(argv[flags.namespace.name]) - return await accountManager._getNodeClient(argv[flags.namespace.name], serviceMap, operator.accountId, operator.privateKey) - } -} - describe.each([ ['v0.42.5', constants.KEY_FORMAT_PFX] // ['v0.47.0-alpha.0', constants.KEY_FORMAT_PFX], @@ -145,10 +133,13 @@ describe.each([ }, 600000) describe('only genesis account should have genesis key for all special accounts', () => { - const genesisKey = PrivateKey.fromStringED25519(constants.OPERATOR_KEY) + const genesisKey = PrivateKey.fromStringED25519(constants.GENESIS_KEY) const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard + beforeAll(() => { + accountManager.loadNodeClient(argv[flags.namespace.name]).then().catch() + }) afterAll(() => { accountManager.close().then().catch() sleep(100).then().catch() @@ -168,50 +159,39 @@ describe.each([ it('balance query should succeed', async () => { expect.assertions(1) - let client = null try { - client = await TestHelper.getNodeClient(accountManager, argv) - const balance = await new AccountBalanceQuery() - .setAccountId(client.getOperator().accountId) - .execute(client) + .setAccountId(accountManager._nodeClient.getOperator().accountId) + .execute(accountManager._nodeClient) expect(balance.hbars).not.toBeNull() } catch (e) { nodeCmd.logger.showUserError(e) expect(e).toBeNull() - } finally { - await accountManager.close() - await sleep(100) } }, 20000) it('account creation should succeed', async () => { expect.assertions(1) - let client = null try { - client = await TestHelper.getNodeClient(accountManager, argv) const accountKey = PrivateKey.generate() let transaction = await new AccountCreateTransaction() .setNodeAccountIds([constants.HEDERA_NODE_ACCOUNT_ID_START]) .setInitialBalance(new Hbar(0)) .setKey(accountKey.publicKey) - .freezeWith(client) + .freezeWith(accountManager._nodeClient) transaction = await transaction.sign(accountKey) - const response = await transaction.execute(client) - const receipt = await response.getReceipt(client) + const response = await transaction.execute(accountManager._nodeClient) + const receipt = await response.getReceipt(accountManager._nodeClient) expect(receipt.accountId).not.toBeNull() } catch (e) { nodeCmd.logger.showUserError(e) expect(e).toBeNull() - } finally { - await accountManager.close() - await sleep(100) } }, 20000) From cd81ae669b97d2a2076a622cfa16338b608b9d53 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Tue, 5 Mar 2024 22:27:35 +0000 Subject: [PATCH 15/20] saving current version Signed-off-by: Jeromy Cannon --- src/core/account_manager.mjs | 23 +++++++++++----- src/core/k8.mjs | 43 ++++++++++++++++++++++++++---- test/e2e/commands/01_node.test.mjs | 1 + 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/core/account_manager.mjs b/src/core/account_manager.mjs index be6e7fefe..dd8bf5370 100644 --- a/src/core/account_manager.mjs +++ b/src/core/account_manager.mjs @@ -34,6 +34,7 @@ import { FullstackTestingError } from './errors.mjs' import { sleep } from './helpers.mjs' import net from 'net' import { Templates } from './templates.mjs' +import { K8 } from './k8.mjs' const REASON_FAILED_TO_GET_KEYS = 'failed to get keys for accountId' const REASON_SKIPPED = 'skipped since it does not have a genesis key' @@ -162,6 +163,7 @@ export class AccountManager { async close () { this._nodeClient?.close() this._nodeClient = null + K8.close() await this._stopPortForwards() } @@ -170,17 +172,24 @@ export class AccountManager { * @returns {Promise} */ async _stopPortForwards () { + let sleepCounter = 0 if (this._portForwards) { for (const webSocketServer of this._portForwards) { webSocketServer.kill(() => { AccountManager._openPortForwardConnections-- }) + + while (webSocketServer.sockets.length > 0 && sleepCounter < MAX_PORT_FORWARD_SLEEP_ITERATIONS) { + ++sleepCounter + this.logger.debug(`waiting ${PORT_FORWARD_CLOSE_SLEEP}ms for port forward server to close .... ${sleepCounter} of ${MAX_PORT_FORWARD_SLEEP_ITERATIONS}`) + await sleep(PORT_FORWARD_CLOSE_SLEEP) + } } - let sleepCounter = 0 while (AccountManager._openPortForwardConnections > 0 && sleepCounter < MAX_PORT_FORWARD_SLEEP_ITERATIONS) { + ++sleepCounter + this.logger.debug(`waiting ${PORT_FORWARD_CLOSE_SLEEP}ms for port forward server to close .... ${sleepCounter} of ${MAX_PORT_FORWARD_SLEEP_ITERATIONS}`) await sleep(PORT_FORWARD_CLOSE_SLEEP) - this.logger.debug(`waiting ${PORT_FORWARD_CLOSE_SLEEP}ms for port forward server to close .... ${++sleepCounter} of ${MAX_PORT_FORWARD_SLEEP_ITERATIONS}`) } if (sleepCounter >= MAX_PORT_FORWARD_SLEEP_ITERATIONS) { this.logger.error(`failed to detect that all port forward servers closed correctly, ${AccountManager._openPortForwardConnections} of ${this._portForwards.length} remain open`) @@ -196,11 +205,13 @@ export class AccountManager { * @returns {Promise} */ async loadNodeClient (namespace) { - const treasuryAccountInfo = await this.getTreasuryAccountKeys(namespace) - const serviceMap = await this.getNodeServiceMap(namespace) + if (!this._nodeClient || this._nodeClient.isClientShutDown) { + const treasuryAccountInfo = await this.getTreasuryAccountKeys(namespace) + const serviceMap = await this.getNodeServiceMap(namespace) - this._nodeClient = await this._getNodeClient(namespace, - serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) + this._nodeClient = await this._getNodeClient(namespace, + serviceMap, treasuryAccountInfo.accountId, treasuryAccountInfo.privateKey) + } } /** diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 082a9c047..2fb6f59e8 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -25,7 +25,6 @@ import * as sb from 'stream-buffers' import * as tar from 'tar' import { v4 as uuid4 } from 'uuid' import { V1ObjectMeta, V1Secret } from '@kubernetes/client-node' -import killable from 'killable' /** * A kubernetes API wrapper class providing custom functionalities required by solo @@ -34,6 +33,8 @@ import killable from 'killable' * For parallel execution, create separate instances by invoking clone() */ export class K8 { + static _webSocketGetters = [] + constructor (configManager, logger) { if (!configManager) throw new MissingArgumentError('An instance of core/ConfigManager is required') if (!logger) throw new MissingArgumentError('An instance of core/Logger is required') @@ -44,6 +45,14 @@ export class K8 { this.init() } + static close () { + for (const webSocketGetter of K8._webSocketGetters) { + if (webSocketGetter) { + webSocketGetter()._sender?._socket?.destroy() + } + } + } + /** * Clone a new instance with the same config manager and logger * Internally it instantiates a new kube API client @@ -653,6 +662,29 @@ export class K8 { }) } + static makeKillable (server, webSocket) { + server.sockets = [] + server.webSocket = webSocket + + server.on('connection', function (socket) { + server.sockets.push(socket) + }) + + server.kill = function (cb) { + server.sockets.forEach(function (socket) { + socket.destroy(null, (exception) => { + server.sockets.splice(server.sockets.indexOf(socket), 1) + }) + }) + + server.close(cb) + + server.sockets = [] + } + + return server + } + /** * Port forward a port from a pod to localhost * @@ -665,12 +697,13 @@ export class K8 { */ async portForward (podName, localPort, podPort) { const ns = this._getNamespace() - const forwarder = new k8s.PortForward(this.kubeConfig, true) - const server = net.createServer((socket) => { - forwarder.portForward(ns, podName, [podPort], socket, null, socket) + const forwarder = new k8s.PortForward(this.kubeConfig, false) + const server = await net.createServer(async (socket) => { + K8._webSocketGetters.push(await forwarder.portForward(ns, podName, [podPort], socket, null, socket, 3)) }) - killable(server) + K8.makeKillable(server) + return server.listen(localPort, '127.0.0.1') } diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index d309b37d6..22617a3d0 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -140,6 +140,7 @@ describe.each([ beforeAll(() => { accountManager.loadNodeClient(argv[flags.namespace.name]).then().catch() }) + afterAll(() => { accountManager.close().then().catch() sleep(100).then().catch() From b6469eb93c08ea719c029cf0b725939a5e324ff0 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 6 Mar 2024 10:11:16 +0000 Subject: [PATCH 16/20] updated test cases Signed-off-by: Jeromy Cannon --- package-lock.json | 6 ------ package.json | 1 - test/e2e/commands/01_node.test.mjs | 14 +++++++------- test/e2e/commands/02_account.test.mjs | 16 ++-------------- 4 files changed, 9 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 30fffaa70..2295260b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,6 @@ "got": "^14.2.0", "inquirer": "^9.2.15", "js-base64": "^3.7.7", - "killable": "^1.0.1", "listr2": "^8.0.2", "tar": "^6.2.0", "uuid": "^9.0.1", @@ -7250,11 +7249,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/killable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/killable/-/killable-1.0.1.tgz", - "integrity": "sha512-LzqtLKlUwirEUyl/nicirVmNiPvYs7l5n8wOPP7fyJVpUPkvCnW/vuiXGpylGUlnPDnB7311rARzAt3Mhswpjg==" - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", diff --git a/package.json b/package.json index f9ca7f284..35b655b22 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "got": "^14.2.0", "inquirer": "^9.2.15", "js-base64": "^3.7.7", - "killable": "^1.0.1", "listr2": "^8.0.2", "tar": "^6.2.0", "uuid": "^9.0.1", diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index 22617a3d0..df1dd0800 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -101,8 +101,8 @@ describe.each([ configManager.update(argv) const nodeIds = argv[flags.nodeIDs.name].split(',') - afterEach(() => { - sleep(5).then().catch() // give a few ticks so that connections can close + afterEach(async () => { + await sleep(5) // give a few ticks so that connections can close }) it('should pre-generate keys', async () => { @@ -137,13 +137,13 @@ describe.each([ const realm = constants.HEDERA_NODE_ACCOUNT_ID_START.realm const shard = constants.HEDERA_NODE_ACCOUNT_ID_START.shard - beforeAll(() => { - accountManager.loadNodeClient(argv[flags.namespace.name]).then().catch() + beforeAll(async () => { + await accountManager.loadNodeClient(argv[flags.namespace.name]) }) - afterAll(() => { - accountManager.close().then().catch() - sleep(100).then().catch() + afterAll(async () => { + await accountManager.close() + await sleep(5000) // sometimes takes a while to close all sockets }) for (const [start, end] of constants.SYSTEM_ACCOUNTS) { diff --git a/test/e2e/commands/02_account.test.mjs b/test/e2e/commands/02_account.test.mjs index 916d89a7c..fd9f68bc0 100644 --- a/test/e2e/commands/02_account.test.mjs +++ b/test/e2e/commands/02_account.test.mjs @@ -78,8 +78,8 @@ describe('account commands should work correctly', () => { configManager.update(argv, true) }) - afterEach(() => { - sleep(5).then().catch() // give a few ticks so that connections can close + afterEach(async () => { + await sleep(5) // give a few ticks so that connections can close }) it('account create with no options', async () => { @@ -96,8 +96,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) @@ -119,8 +117,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) @@ -140,8 +136,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) @@ -163,8 +157,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) @@ -183,8 +175,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) @@ -203,8 +193,6 @@ describe('account commands should work correctly', () => { } catch (e) { testLogger.showUserError(e) expect(e).toBeNull() - } finally { - await accountCmd.closeConnections() } }, defaultTimeout) }) From 919978a18975c901de5e7bb433e7f83afd986ed8 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 6 Mar 2024 10:29:56 +0000 Subject: [PATCH 17/20] updated test cases Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index df1dd0800..c4840eb2f 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -144,14 +144,17 @@ describe.each([ afterAll(async () => { await accountManager.close() await sleep(5000) // sometimes takes a while to close all sockets - }) + }, 10000) for (const [start, end] of constants.SYSTEM_ACCOUNTS) { for (let i = start; i <= end; i++) { it(`special account ${i} should not have genesis key`, async () => { + expect(accountManager._nodeClient).not.toBeNull() + const accountId = `${realm}.${shard}.${i}` nodeCmd.logger.info(`getAccountKeys: accountId ${accountId}`) const keys = await accountManager.getAccountKeys(accountId) + expect(keys[0].toString()).not.toEqual(genesisKey.toString()) }, 60000) } @@ -162,6 +165,8 @@ describe.each([ expect.assertions(1) try { + expect(accountManager._nodeClient).not.toBeNull() + const balance = await new AccountBalanceQuery() .setAccountId(accountManager._nodeClient.getOperator().accountId) .execute(accountManager._nodeClient) @@ -177,6 +182,7 @@ describe.each([ expect.assertions(1) try { + expect(accountManager._nodeClient).not.toBeNull() const accountKey = PrivateKey.generate() let transaction = await new AccountCreateTransaction() From 3c9aa6a7723798f2077f44d1ba396a1f4d8de279 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 6 Mar 2024 10:41:50 +0000 Subject: [PATCH 18/20] fixed test cases by adding new describe block Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 75 +++++++++++++++++------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index c4840eb2f..bf2cbd2ee 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -161,46 +161,57 @@ describe.each([ } }) - it('balance query should succeed', async () => { - expect.assertions(1) + describe('use the node client to interact with the Hedera network', () => { + beforeAll(async () => { + await accountManager.loadNodeClient(argv[flags.namespace.name]) + }) - try { - expect(accountManager._nodeClient).not.toBeNull() + afterAll(async () => { + await accountManager.close() + await sleep(5000) // sometimes takes a while to close all sockets + }, 10000) - const balance = await new AccountBalanceQuery() - .setAccountId(accountManager._nodeClient.getOperator().accountId) - .execute(accountManager._nodeClient) + it('balance query should succeed', async () => { + expect.assertions(2) - expect(balance.hbars).not.toBeNull() - } catch (e) { - nodeCmd.logger.showUserError(e) - expect(e).toBeNull() - } - }, 20000) + try { + expect(accountManager._nodeClient).not.toBeNull() - it('account creation should succeed', async () => { - expect.assertions(1) + const balance = await new AccountBalanceQuery() + .setAccountId(accountManager._nodeClient.getOperator().accountId) + .execute(accountManager._nodeClient) - try { - expect(accountManager._nodeClient).not.toBeNull() - const accountKey = PrivateKey.generate() + expect(balance.hbars).not.toBeNull() + } catch (e) { + nodeCmd.logger.showUserError(e) + expect(e).toBeNull() + } + }, 20000) - let transaction = await new AccountCreateTransaction() - .setNodeAccountIds([constants.HEDERA_NODE_ACCOUNT_ID_START]) - .setInitialBalance(new Hbar(0)) - .setKey(accountKey.publicKey) - .freezeWith(accountManager._nodeClient) + it('account creation should succeed', async () => { + expect.assertions(1) - transaction = await transaction.sign(accountKey) - const response = await transaction.execute(accountManager._nodeClient) - const receipt = await response.getReceipt(accountManager._nodeClient) + try { + expect(accountManager._nodeClient).not.toBeNull() + const accountKey = PrivateKey.generate() - expect(receipt.accountId).not.toBeNull() - } catch (e) { - nodeCmd.logger.showUserError(e) - expect(e).toBeNull() - } - }, 20000) + let transaction = await new AccountCreateTransaction() + .setNodeAccountIds([constants.HEDERA_NODE_ACCOUNT_ID_START]) + .setInitialBalance(new Hbar(0)) + .setKey(accountKey.publicKey) + .freezeWith(accountManager._nodeClient) + + transaction = await transaction.sign(accountKey) + const response = await transaction.execute(accountManager._nodeClient) + const receipt = await response.getReceipt(accountManager._nodeClient) + + expect(receipt.accountId).not.toBeNull() + } catch (e) { + nodeCmd.logger.showUserError(e) + expect(e).toBeNull() + } + }, 20000) + }) it('checkNetworkNodeProxyUp should succeed', async () => { expect.assertions(1) From 0ac9da6144a63824fdfa64d6c88ee52fb40cc91d Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 6 Mar 2024 10:54:54 +0000 Subject: [PATCH 19/20] fixed test cases by updating expected assertions Signed-off-by: Jeromy Cannon --- test/e2e/commands/01_node.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/commands/01_node.test.mjs b/test/e2e/commands/01_node.test.mjs index bf2cbd2ee..957ffb41e 100644 --- a/test/e2e/commands/01_node.test.mjs +++ b/test/e2e/commands/01_node.test.mjs @@ -189,7 +189,7 @@ describe.each([ }, 20000) it('account creation should succeed', async () => { - expect.assertions(1) + expect.assertions(2) try { expect(accountManager._nodeClient).not.toBeNull() From 50965e0654f030c9486188dfe37dcf89c7576690 Mon Sep 17 00:00:00 2001 From: Jeromy Cannon Date: Wed, 6 Mar 2024 11:22:57 +0000 Subject: [PATCH 20/20] updates based on pr feedback Signed-off-by: Jeromy Cannon --- src/commands/node.mjs | 8 ++++---- src/core/k8.mjs | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 942a185eb..ec69addeb 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -505,12 +505,12 @@ export class NodeCommand extends BaseCommand { if (ctx.config.deployMirrorNode) { const subTasks = [ { - title: 'Wait for proxies to verify node servers are running', + title: 'Check node proxies are ACTIVE', task: async (ctx, _) => { const subTasks = [] for (const nodeId of ctx.config.nodeIds) { subTasks.push({ - title: `Check node proxy: ${chalk.yellow(nodeId)}`, + title: `Check proxy for node: ${chalk.yellow(nodeId)}`, task: async () => await self.checkNetworkNodeProxyUp(ctx.config.namespace, nodeId) }) } @@ -652,10 +652,10 @@ export class NodeCommand extends BaseCommand { async checkNetworkNodeProxyUp (namespace, nodeId, maxAttempts = 100) { const podArray = await this.k8.getPodsByLabel([`app=haproxy-${nodeId}`, 'fullstack.hedera.com/type=haproxy']) + let attempts = 0 if (podArray.length > 0) { const podName = podArray[0].metadata.name - let attempts = 0 while (attempts < maxAttempts) { const logResponse = await this.k8.kubeClient.readNamespacedPodLog( podName, namespace) @@ -673,7 +673,7 @@ export class NodeCommand extends BaseCommand { await sleep(1000) } } else { - throw new FullstackTestingError('TBD') + throw new FullstackTestingError(`proxy for '${nodeId}' is not ACTIVE [ attempt = ${attempts}/${maxAttempts}`) } return false diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 2fb6f59e8..2c978f98b 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -33,6 +33,8 @@ import { V1ObjectMeta, V1Secret } from '@kubernetes/client-node' * For parallel execution, create separate instances by invoking clone() */ export class K8 { + // track the webSocket generated when creating the Kubernetes port forward + // so that we can call destroy and speed up shutting down the server static _webSocketGetters = [] constructor (configManager, logger) {