From 54ecde22d04f95cba8e2f67d13f2aba20ec71c77 Mon Sep 17 00:00:00 2001 From: instamenta Date: Tue, 11 Feb 2025 14:01:59 +0200 Subject: [PATCH 1/2] added the new flags as well as the changes to the taskfile and the migration script Signed-off-by: instamenta --- Taskfile.helper.yml | 2 +- examples/external-database-test/Taskfile.yml | 19 ++++- .../external-database-test/scripts/init.sh | 12 ++- src/commands/flags.ts | 47 +++++++++++ src/commands/mirror_node.ts | 84 ++++++++++++------- 5 files changed, 126 insertions(+), 38 deletions(-) diff --git a/Taskfile.helper.yml b/Taskfile.helper.yml index f0f3dbc5f..c57cf0460 100644 --- a/Taskfile.helper.yml +++ b/Taskfile.helper.yml @@ -506,4 +506,4 @@ tasks: cmd: | kubectl exec -it {{.postgres_container_name}} \ -n {{.postgres_database_namespace}} \ - -- /bin/bash /tmp/init.sh "{{.postgres_username}}" "{{.postgres_password}}" + -- /bin/bash /tmp/init.sh "{{.postgres_username}}" "{{.postgres_readonly_username}}" "{{.postgres_readonly_password}}" diff --git a/examples/external-database-test/Taskfile.yml b/examples/external-database-test/Taskfile.yml index 75f80ed3f..c6ac0a127 100644 --- a/examples/external-database-test/Taskfile.yml +++ b/examples/external-database-test/Taskfile.yml @@ -6,19 +6,30 @@ includes: vars: use_port_forwards: "true" postgres_username: "postgres" - postgres_password: "XXXXXXXXX" + postgres_password: "XXXXXXXX" + + postgres_readonly_username: "readonlyuser" + postgres_readonly_password: "XXXXXXXX" + + postgres_mirror_node_database_name: "mirror_node" + postgres_name: "my-postgresql" + postgres_database_namespace: "database" postgres_container_name: "{{.postgres_name}}-0" postgres_host_fqdn: "{{.postgres_name}}.database.svc.cluster.local" postgres_container_fdqn: "{{.postgres_container_name}}.database.svc.cluster.local" - postgres_mirror_node_database_name: "mirror_node" - postgres_database_namespace: "database" env: SOLO_NETWORK_SIZE: "1" SOLO_DEPLOYMENT: "solo-e2e" SOLO_NAMESPACE: "solo-e2e" SOLO_CLUSTER_NAME: "solo-e2e" - MIRROR_NODE_DEPLOY_EXTRA_FLAGS: "--use-external-database --external-database-host {{.postgres_host_fqdn}} --external-database-owner-username {{.postgres_username}} --external-database-owner-password {{.postgres_password}}" + MIRROR_NODE_DEPLOY_EXTRA_FLAGS: | + --use-external-database + --external-database-host {{.postgres_host_fqdn}} + --external-database-owner-username {{.postgres_username}} + --external-database-owner-password {{.postgres_password}} + --external-database-read-username {{.postgres_readonly_username}} + --external-database-read-password {{.postgres_readonly_password}} tasks: default: silent: true diff --git a/examples/external-database-test/scripts/init.sh b/examples/external-database-test/scripts/init.sh index ff345b0c5..7db61b2c8 100644 --- a/examples/external-database-test/scripts/init.sh +++ b/examples/external-database-test/scripts/init.sh @@ -3,7 +3,9 @@ set -e export HEDERA_MIRROR_DATABASE_NAME="mirror_node" HEDERA_MIRROR_OWNER="$1" -HEDERA_MIRROR_OWNER_PASSWORD="$2" +HEDERA_MIRROR_READ="$2" +HEDERA_MIRROR_READ_PASSWORD="$3" + export HEDERA_MIRROR_GRPC_DB_HOST="localhost" @@ -26,7 +28,9 @@ psql -d "user=postgres connect_timeout=3" \ --set "dbName=${HEDERA_MIRROR_IMPORTER_DB_NAME}" \ --set "dbSchema=${HEDERA_MIRROR_IMPORTER_DB_SCHEMA}" \ --set "ownerUsername=${HEDERA_MIRROR_IMPORTER_DB_OWNER}" \ - --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" <<__SQL__ + --set "tempSchema=${HEDERA_MIRROR_IMPORTER_DB_TEMPSCHEMA}" \ + --set "readUsername=${HEDERA_MIRROR_READ}" \ + --set "readPassword=${HEDERA_MIRROR_READ_PASSWORD}" <<__SQL__ -- Create database & owner create database :dbName with owner :ownerUsername; @@ -59,6 +63,10 @@ create schema if not exists :tempSchema authorization temporary_admin; grant usage on schema :tempSchema to public; revoke create on schema :tempSchema from public; +-- Create readonly user with password and grant privileges +create user :readUsername with password :'readPassword'; +grant readonly to :readUsername; + -- Grant readonly privileges grant connect on database :dbName to readonly; grant select on all tables in schema :dbSchema, :tempSchema to readonly; diff --git a/src/commands/flags.ts b/src/commands/flags.ts index 3a4bf26a4..2e4294b89 100644 --- a/src/commands/flags.ts +++ b/src/commands/flags.ts @@ -1567,6 +1567,8 @@ export class Flags { prompt: undefined, }; + //* ----------------- External Mirror Node PostgreSQL Database Related Flags ------------------ *// + static readonly externalDatabaseHost: CommandFlag = { constName: 'externalDatabaseHost', name: 'external-database-host', @@ -1628,6 +1630,49 @@ export class Flags { }, }; + static readonly externalDatabaseReadonlyUsername: CommandFlag = { + constName: 'externalDatabaseReadonlyUsername', + name: 'external-database-read-username', + definition: { + describe: `Use to provide the external database readonly user's username if the '--${Flags.useExternalDatabase.name}' is passed`, + defaultValue: '', + type: 'string', + }, + prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper, input: any) { + return await Flags.promptText( + task, + input, + Flags.externalDatabaseReadonlyUsername.definition.defaultValue, + 'Enter username of the external database readonly user', + null, + Flags.externalDatabaseReadonlyUsername.name, + ); + }, + }; + + static readonly externalDatabaseReadonlyPassword: CommandFlag = { + constName: 'externalDatabaseReadonlyPassword', + name: 'external-database-read-password', + definition: { + describe: `Use to provide the external database readonly user's password if the '--${Flags.useExternalDatabase.name}' is passed`, + defaultValue: '', + type: 'string', + dataMask: constants.STANDARD_DATAMASK, + }, + prompt: async function promptGrpcWebTlsKeyPath(task: ListrTaskWrapper, input: any) { + return await Flags.promptText( + task, + input, + Flags.externalDatabaseReadonlyPassword.definition.defaultValue, + 'Enter password of the external database readonly user', + null, + Flags.externalDatabaseReadonlyPassword.name, + ); + }, + }; + + //* ------------------------------------------------------------------------------------------- *// + static readonly grpcTlsKeyPath: CommandFlag = { constName: 'grpcTlsKeyPath', name: 'grpc-tls-key', @@ -1918,6 +1963,8 @@ export class Flags { Flags.externalDatabaseHost, Flags.externalDatabaseOwnerUsername, Flags.externalDatabaseOwnerPassword, + Flags.externalDatabaseReadonlyUsername, + Flags.externalDatabaseReadonlyPassword, ]; /** Resets the definition.disablePrompt for all flags */ diff --git a/src/commands/mirror_node.ts b/src/commands/mirror_node.ts index 246204c6f..ee0293faa 100644 --- a/src/commands/mirror_node.ts +++ b/src/commands/mirror_node.ts @@ -54,6 +54,8 @@ interface MirrorNodeDeployConfigClass { externalDatabaseHost: Optional; externalDatabaseOwnerUsername: Optional; externalDatabaseOwnerPassword: Optional; + externalDatabaseReadonlyUsername: Optional; + externalDatabaseReadonlyPassword: Optional; } interface Context { @@ -102,6 +104,8 @@ export class MirrorNodeCommand extends BaseCommand { flags.externalDatabaseHost, flags.externalDatabaseOwnerUsername, flags.externalDatabaseOwnerPassword, + flags.externalDatabaseReadonlyUsername, + flags.externalDatabaseReadonlyPassword, ]; } @@ -149,8 +153,10 @@ export class MirrorNodeCommand extends BaseCommand { if (config.useExternalDatabase) { const { externalDatabaseHost: host, - externalDatabaseOwnerUsername: username, - externalDatabaseOwnerPassword: password, + externalDatabaseOwnerUsername: ownerUsername, + externalDatabaseOwnerPassword: ownerPassword, + externalDatabaseReadonlyUsername: readonlyUsername, + externalDatabaseReadonlyPassword: readonlyPassword, } = config; valuesArg += helpers.populateHelmArgs({ @@ -163,21 +169,24 @@ export class MirrorNodeCommand extends BaseCommand { 'db.name': 'mirror_node', // set the usernames - 'db.owner.username': username, - 'importer.db.username': username, - 'grpc.db.username': username, - 'restjava.db.username': username, - 'web3.db.username': username, - // Fixes problem where importer's V1.0__Init.sql migration fails - // 'rest.db.username': username, + 'db.owner.username': ownerUsername, + 'importer.db.username': ownerUsername, + + 'grpc.db.username': readonlyUsername, + 'restjava.db.username': readonlyUsername, + 'web3.db.username': readonlyUsername, + + // TODO: Fixes a problem where importer's V1.0__Init.sql migration fails + // 'rest.db.username': readonlyUsername, // set the passwords - 'db.owner.password': password, - 'importer.db.password': password, - 'grpc.db.password': password, - 'rest.db.password': password, - 'restjava.db.password': password, - 'web3.db.password': password, + 'db.owner.password': ownerPassword, + 'importer.db.password': ownerPassword, + + 'grpc.db.password': readonlyPassword, + 'restjava.db.password': readonlyPassword, + 'web3.db.password': readonlyPassword, + 'rest.db.password': readonlyPassword, }); } @@ -206,6 +215,8 @@ export class MirrorNodeCommand extends BaseCommand { flags.externalDatabaseHost, flags.externalDatabaseOwnerUsername, flags.externalDatabaseOwnerPassword, + flags.externalDatabaseReadonlyUsername, + flags.externalDatabaseReadonlyPassword, ]); await self.configManager.executePrompt(task, MirrorNodeCommand.DEPLOY_FLAGS_LIST); @@ -276,24 +287,35 @@ export class MirrorNodeCommand extends BaseCommand { flags.externalDatabaseHost, flags.externalDatabaseOwnerUsername, flags.externalDatabaseOwnerPassword, + flags.externalDatabaseReadonlyUsername, + flags.externalDatabaseReadonlyPassword, ]); - } else if (ctx.config.useExternalDatabase) { - if ( - !ctx.config.externalDatabaseHost || + } else if ( + ctx.config.useExternalDatabase && + (!ctx.config.externalDatabaseHost || !ctx.config.externalDatabaseOwnerUsername || - !ctx.config.externalDatabaseOwnerPassword - ) { - const missingFlags: CommandFlag[] = []; - if (!ctx.config.externalDatabaseHost) missingFlags.push(flags.externalDatabaseHost); - if (!ctx.config.externalDatabaseOwnerUsername) missingFlags.push(flags.externalDatabaseOwnerUsername); - if (!ctx.config.externalDatabaseOwnerPassword) missingFlags.push(flags.externalDatabaseOwnerPassword); - if (missingFlags.length) { - const errorMessage = - 'There are missing values that need to be provided when' + - `${chalk.cyan(`--${flags.useExternalDatabase.name}`)} is provided: `; - - throw new SoloError(`${errorMessage} ${missingFlags.map(flag => `--${flag.name}`).join(', ')}`); - } + !ctx.config.externalDatabaseOwnerPassword || + !ctx.config.externalDatabaseReadonlyUsername || + !ctx.config.externalDatabaseReadonlyPassword) + ) { + const missingFlags: CommandFlag[] = []; + if (!ctx.config.externalDatabaseHost) missingFlags.push(flags.externalDatabaseHost); + if (!ctx.config.externalDatabaseOwnerUsername) missingFlags.push(flags.externalDatabaseOwnerUsername); + if (!ctx.config.externalDatabaseOwnerPassword) missingFlags.push(flags.externalDatabaseOwnerPassword); + + if (!ctx.config.externalDatabaseReadonlyUsername) { + missingFlags.push(flags.externalDatabaseReadonlyUsername); + } + if (!ctx.config.externalDatabaseReadonlyPassword) { + missingFlags.push(flags.externalDatabaseReadonlyPassword); + } + + if (missingFlags.length) { + const errorMessage = + 'There are missing values that need to be provided when' + + `${chalk.cyan(`--${flags.useExternalDatabase.name}`)} is provided: `; + + throw new SoloError(`${errorMessage} ${missingFlags.map(flag => `--${flag.name}`).join(', ')}`); } } From f1fb2d6af6b5651fa2cf8f34cadf2ce2dbef9ee9 Mon Sep 17 00:00:00 2001 From: instamenta Date: Tue, 11 Feb 2025 14:27:35 +0200 Subject: [PATCH 2/2] fix failing test Signed-off-by: instamenta --- test/e2e/commands/mirror_node.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/commands/mirror_node.test.ts b/test/e2e/commands/mirror_node.test.ts index 6151f1d82..ed5278a12 100644 --- a/test/e2e/commands/mirror_node.test.ts +++ b/test/e2e/commands/mirror_node.test.ts @@ -98,6 +98,8 @@ e2eTestSuite(testName, argv, undefined, undefined, undefined, undefined, undefin flags.externalDatabaseHost.constName, flags.externalDatabaseOwnerUsername.constName, flags.externalDatabaseOwnerPassword.constName, + flags.externalDatabaseReadonlyUsername.constName, + flags.externalDatabaseReadonlyPassword.constName, ]); expect(explorerCommand.getUnusedConfigs(MirrorNodeCommand.DEPLOY_CONFIGS_NAME)).to.deep.equal([ flags.hederaExplorerTlsHostName.constName,