diff --git a/packages/amplify-category-function/src/__tests__/provider-utils/awscloudformation/secrets/functionSecretsStateManager.test.ts b/packages/amplify-category-function/src/__tests__/provider-utils/awscloudformation/secrets/functionSecretsStateManager.test.ts index 01b14ace852..2c2bf39a9b6 100644 --- a/packages/amplify-category-function/src/__tests__/provider-utils/awscloudformation/secrets/functionSecretsStateManager.test.ts +++ b/packages/amplify-category-function/src/__tests__/provider-utils/awscloudformation/secrets/functionSecretsStateManager.test.ts @@ -27,10 +27,10 @@ pathManagerMock.getBackendDirPath.mockReturnValue(path.join('test', 'path')); getAppIdMock.mockReturnValue('testAppId'); -SSMClientWrapperMock.getInstance.mockResolvedValue(({ +SSMClientWrapperMock.getInstance.mockResolvedValue({ deleteSecret: jest.fn(), setSecret: jest.fn(), -} as unknown) as SSMClientWrapper); +} as unknown as SSMClientWrapper); describe('syncSecretDeltas', () => { const contextStub = ({ diff --git a/packages/amplify-category-function/src/provider-utils/awscloudformation/secrets/ssmClientWrapper.ts b/packages/amplify-category-function/src/provider-utils/awscloudformation/secrets/ssmClientWrapper.ts index 70ba4710ece..a9c198708cc 100644 --- a/packages/amplify-category-function/src/provider-utils/awscloudformation/secrets/ssmClientWrapper.ts +++ b/packages/amplify-category-function/src/provider-utils/awscloudformation/secrets/ssmClientWrapper.ts @@ -1,10 +1,9 @@ -import { $TSAny, $TSContext, spinner } from 'amplify-cli-core'; -import aws from 'aws-sdk'; +import { $TSContext } from 'amplify-cli-core'; +import type { SSM } from 'aws-sdk'; /** * Wrapper around SSM SDK calls */ -// eslint-disable-next-line export class SSMClientWrapper { private static instance: SSMClientWrapper; @@ -15,12 +14,12 @@ export class SSMClientWrapper { return SSMClientWrapper.instance; }; - private constructor(private readonly ssmClient: aws.SSM) {} + private constructor(private readonly ssmClient: SSM) {} /** * Returns a list of secret name value pairs */ - getSecrets = async (secretNames: string[]): Promise<$TSAny> => { + getSecrets = async (secretNames: string[]): Promise<{ secretName?: string, secretValue?: string }[] | undefined> => { if (!secretNames || secretNames.length === 0) { return []; } @@ -31,7 +30,7 @@ export class SSMClientWrapper { }) .promise(); - return result.Parameters.map(({ Name, Value }) => ({ secretName: Name, secretValue: Value })); + return result?.Parameters?.map(({ Name, Value }) => ({ secretName: Name, secretValue: Value })); }; /** @@ -41,8 +40,7 @@ export class SSMClientWrapper { let NextToken; const accumulator: string[] = []; do { - // eslint-disable-next-line - const result = await this.ssmClient + const result: SSM.GetParametersByPathResult = await this.ssmClient .getParametersByPath({ Path: secretPath, MaxResults: 10, @@ -56,7 +54,11 @@ export class SSMClientWrapper { NextToken, }) .promise(); - accumulator.push(...result.Parameters.map(param => param.Name)); + + if (Array.isArray(result?.Parameters)) { + accumulator.push(...result.Parameters.filter(param => param?.Name !== undefined).map(param => param.Name)); + } + NextToken = result.NextToken; } while (NextToken); return accumulator; @@ -108,17 +110,14 @@ export class SSMClientWrapper { }; } -const getSSMClient = async (context: $TSContext): Promise => { - try { - spinner.start(); - spinner.text = 'Building and packaging resources'; - - const { client } = await context.amplify.invokePluginMethod<{ client: aws.SSM }>(context, 'awscloudformation', undefined, 'getConfiguredSSMClient', [ - context, - ]); +const getSSMClient = async (context: $TSContext): Promise => { + const { client } = await context.amplify.invokePluginMethod<{ client: SSM }>( + context, + 'awscloudformation', + undefined, + 'getConfiguredSSMClient', + [context], + ); - return client; - } finally { - spinner.stop(); - } + return client; }; diff --git a/packages/amplify-cli-core/API.md b/packages/amplify-cli-core/API.md index 4864c2f8a2f..b848665bb9b 100644 --- a/packages/amplify-cli-core/API.md +++ b/packages/amplify-cli-core/API.md @@ -122,7 +122,7 @@ export class AmplifyError extends AmplifyException { } // @public (undocumented) -export type AmplifyErrorType = 'AmplifyStudioError' | 'AmplifyStudioLoginError' | 'AmplifyStudioNotEnabledError' | 'ApiCategorySchemaNotFoundError' | 'AuthImportError' | 'BackendConfigValidationError' | 'BucketAlreadyExistsError' | 'BucketNotFoundError' | 'CategoryNotEnabledError' | 'CloudFormationTemplateError' | 'CommandNotSupportedError' | 'ConfigurationError' | 'DeploymentError' | 'DeploymentInProgressError' | 'DirectoryError' | 'DirectoryAlreadyExistsError' | 'DuplicateLogicalIdError' | 'EnvironmentConfigurationError' | 'EnvironmentNameError' | 'EnvironmentNotInitializedError' | 'FeatureFlagsValidationError' | 'FrameworkNotSupportedError' | 'FunctionTooLargeError' | 'InputValidationError' | 'InvalidAmplifyAppIdError' | 'InvalidCustomResourceError' | 'InvalidOverrideError' | 'InvalidStackError' | 'IterativeRollbackError' | 'LambdaLayerDeleteError' | 'MigrationError' | 'MissingAmplifyMetaFileError' | 'MissingOverridesInstallationRequirementsError' | 'ModelgenError' | 'NestedProjectInitError' | 'NoUpdateBackendError' | 'NotImplementedError' | 'OpenSslCertificateError' | 'ParameterNotFoundError' | 'PermissionsError' | 'PluginMethodNotFoundError' | 'PluginNotFoundError' | 'PluginPolicyAddError' | 'ProfileConfigurationError' | 'ProjectAppIdResolveError' | 'ProjectInitError' | 'ProjectNotFoundError' | 'ProjectNotInitializedError' | 'PushResourcesError' | 'RegionNotAvailableError' | 'RemoveNotificationAppError' | 'ResourceAlreadyExistsError' | 'ResourceInUseError' | 'ResourceNotReadyError' | 'StackNotFoundError' | 'StackStateError' | 'UnsupportedLockFileTypeError' | 'UserInputError' | 'MockProcessError' | 'SearchableMockUnsupportedPlatformError' | 'SearchableMockUnavailablePortError' | 'SearchableMockProcessError'; +export type AmplifyErrorType = 'AmplifyStudioError' | 'AmplifyStudioLoginError' | 'AmplifyStudioNotEnabledError' | 'ApiCategorySchemaNotFoundError' | 'AuthImportError' | 'BackendConfigValidationError' | 'BucketAlreadyExistsError' | 'BucketNotFoundError' | 'CategoryNotEnabledError' | 'CloudFormationTemplateError' | 'CommandNotSupportedError' | 'ConfigurationError' | 'DeploymentError' | 'DeploymentInProgressError' | 'DirectoryError' | 'DirectoryAlreadyExistsError' | 'DuplicateLogicalIdError' | 'EnvironmentConfigurationError' | 'EnvironmentNameError' | 'EnvironmentNotInitializedError' | 'FeatureFlagsValidationError' | 'FrameworkNotSupportedError' | 'FunctionTooLargeError' | 'InputValidationError' | 'InvalidAmplifyAppIdError' | 'InvalidCustomResourceError' | 'InvalidOverrideError' | 'InvalidStackError' | 'IterativeRollbackError' | 'LambdaLayerDeleteError' | 'MigrationError' | 'MissingAmplifyMetaFileError' | 'MissingExpectedParameterError' | 'MissingOverridesInstallationRequirementsError' | 'ModelgenError' | 'NestedProjectInitError' | 'NoUpdateBackendError' | 'NotImplementedError' | 'OpenSslCertificateError' | 'ParameterNotFoundError' | 'PermissionsError' | 'PluginMethodNotFoundError' | 'PluginNotFoundError' | 'PluginPolicyAddError' | 'ProfileConfigurationError' | 'ProjectAppIdResolveError' | 'ProjectInitError' | 'ProjectNotFoundError' | 'ProjectNotInitializedError' | 'PushResourcesError' | 'RegionNotAvailableError' | 'RemoveNotificationAppError' | 'ResourceAlreadyExistsError' | 'ResourceInUseError' | 'ResourceNotReadyError' | 'StackNotFoundError' | 'StackStateError' | 'UnsupportedLockFileTypeError' | 'UserInputError' | 'MockProcessError' | 'SearchableMockUnsupportedPlatformError' | 'SearchableMockUnavailablePortError' | 'SearchableMockProcessError'; // @public (undocumented) export enum AmplifyEvent { @@ -230,7 +230,7 @@ export class AmplifyFault extends AmplifyException { } // @public (undocumented) -export type AmplifyFaultType = 'AnalyticsCategoryFault' | 'AmplifyBackupFault' | 'BackendPullFault' | 'ConfigurationFault' | 'BackendDeleteFault' | 'ConfigurationFault' | 'DeploymentFault' | 'LockFileNotFoundFault' | 'LockFileParsingFault' | 'NotificationsChannelAPNSFault' | 'NotificationsChannelEmailFault' | 'NotificationsChannelFCMFault' | 'NotificationsChannelSmsFault' | 'NotificationsChannelInAppMessagingFault' | 'NotImplementedFault' | 'ProjectDeleteFault' | 'ProjectInitFault' | 'PluginNotLoadedFault' | 'PushResourcesFault' | 'PullBackendFault' | 'ResourceExportFault' | 'ResourceNotFoundFault' | 'ResourceNotReadyFault' | 'ResourceRemoveFault' | 'RootStackNotFoundFault' | 'ServiceCallFault' | 'SnsSandboxModeCheckFault' | 'TimeoutFault' | 'TriggerUploadFault' | 'UnexpectedS3Fault' | 'UnknownFault' | 'UnknownNodeJSFault' | 'MockProcessFault' | 'AuthCategoryFault' | 'ZipExtractFault'; +export type AmplifyFaultType = 'AnalyticsCategoryFault' | 'AmplifyBackupFault' | 'BackendPullFault' | 'ConfigurationFault' | 'BackendDeleteFault' | 'ConfigurationFault' | 'DeploymentFault' | 'LockFileNotFoundFault' | 'LockFileParsingFault' | 'NotificationsChannelAPNSFault' | 'NotificationsChannelEmailFault' | 'NotificationsChannelFCMFault' | 'NotificationsChannelSmsFault' | 'NotificationsChannelInAppMessagingFault' | 'NotImplementedFault' | 'ParameterDownloadFault' | 'ParameterUploadFault' | 'ProjectDeleteFault' | 'ParametersDeleteFault' | 'ProjectInitFault' | 'PluginNotLoadedFault' | 'PushResourcesFault' | 'PullBackendFault' | 'ResourceExportFault' | 'ResourceNotFoundFault' | 'ResourceNotReadyFault' | 'ResourceRemoveFault' | 'RootStackNotFoundFault' | 'ServiceCallFault' | 'SnsSandboxModeCheckFault' | 'TimeoutFault' | 'TriggerUploadFault' | 'UnexpectedS3Fault' | 'UnknownFault' | 'UnknownNodeJSFault' | 'MockProcessFault' | 'AuthCategoryFault' | 'ZipExtractFault'; // @public (undocumented) export enum AmplifyFrontend { @@ -559,6 +559,7 @@ export const constants: { CODEGEN: string; AMPLIFY: string; DOT_AMPLIFY_DIR_NAME: string; + DEFAULT_PROVIDER: string; AMPLIFY_PREFIX: string; LOCAL_NODE_MODULES: string; PARENT_DIRECTORY: string; diff --git a/packages/amplify-cli-core/src/constants.ts b/packages/amplify-cli-core/src/constants.ts index 3bc6906b9c5..cd3ed23411d 100644 --- a/packages/amplify-cli-core/src/constants.ts +++ b/packages/amplify-cli-core/src/constants.ts @@ -14,6 +14,7 @@ export const constants = { CODEGEN: 'codegen', AMPLIFY: 'amplify', DOT_AMPLIFY_DIR_NAME: '.amplify', + DEFAULT_PROVIDER: 'awscloudformation', AMPLIFY_PREFIX: 'amplify-', LOCAL_NODE_MODULES: 'cli-local-node-modules', PARENT_DIRECTORY: 'cli-parent-directory', diff --git a/packages/amplify-cli-core/src/errors/amplify-exception.ts b/packages/amplify-cli-core/src/errors/amplify-exception.ts index 9814875dc39..3e1b6b1aea7 100644 --- a/packages/amplify-cli-core/src/errors/amplify-exception.ts +++ b/packages/amplify-cli-core/src/errors/amplify-exception.ts @@ -47,14 +47,17 @@ export abstract class AmplifyException extends Error { } toObject = (): object => { - const { - name: errorName, message: errorMessage, details: errorDetails, resolution, link, stack, - } = this; + const { name: errorName, message: errorMessage, details: errorDetails, resolution, link, stack } = this; return { - errorName, errorMessage, errorDetails, resolution, link, ...(process.argv.includes('--debug') ? { stack } : {}), + errorName, + errorMessage, + errorDetails, + resolution, + link, + ...(process.argv.includes('--debug') ? { stack } : {}), }; - } + }; } /** @@ -66,13 +69,13 @@ export type AmplifyExceptionClassification = 'FAULT' | 'ERROR'; * Amplify Error options object */ export type AmplifyExceptionOptions = { - message: string, - details?: string, - resolution?: string, - link?: string, + message: string; + details?: string; + resolution?: string; + link?: string; // CloudFormation or NodeJS error codes - code?: string, + code?: string; }; /** @@ -123,6 +126,7 @@ export type AmplifyErrorType = | 'LambdaLayerDeleteError' | 'MigrationError' | 'MissingAmplifyMetaFileError' + | 'MissingExpectedParameterError' | 'MissingOverridesInstallationRequirementsError' | 'ModelgenError' | 'NestedProjectInitError' @@ -173,7 +177,10 @@ export type AmplifyFaultType = | 'NotificationsChannelSmsFault' | 'NotificationsChannelInAppMessagingFault' | 'NotImplementedFault' + | 'ParameterDownloadFault' + | 'ParameterUploadFault' | 'ProjectDeleteFault' + | 'ParametersDeleteFault' | 'ProjectInitFault' | 'PluginNotLoadedFault' | 'PushResourcesFault' diff --git a/packages/amplify-cli-core/src/types.ts b/packages/amplify-cli-core/src/types.ts index 7010bc55ba4..829beb78301 100644 --- a/packages/amplify-cli-core/src/types.ts +++ b/packages/amplify-cli-core/src/types.ts @@ -369,8 +369,21 @@ interface AmplifyToolkit { updateBackendConfigAfterResourceAdd: (category: string, resourceName: string, resourceData: $TSObject) => void; updateBackendConfigAfterResourceUpdate: (category: string, resourceName: string, attribute: string, value: $TSAny) => void; updateBackendConfigAfterResourceRemove: (category: string, resourceName: string) => void; + /** + * use EnvironmentParameterManager from the amplify-environment-parameters package + * @deprecated + */ loadEnvResourceParameters: (context: $TSContext, category: string, resourceName: string) => $TSAny; - saveEnvResourceParameters: (context: $TSContext, category: string, resourceName: string, envSpecificParams?: $TSObject) => void; + /** + * use EnvironmentParameterManager from the amplify-environment-parameters package + * @deprecated + */ + saveEnvResourceParameters: ( + context: $TSContext | undefined, + category: string, + resourceName: string, + envSpecificParams?: $TSObject, + ) => void; removeResourceParameters: (context: $TSContext, category: string, resource: string) => void; triggerFlow: (...args: unknown[]) => $TSAny; addTrigger: () => $TSAny; diff --git a/packages/amplify-cli/src/__tests__/extensions/amplify-helpers/delete-project.test.ts b/packages/amplify-cli/src/__tests__/extensions/amplify-helpers/delete-project.test.ts index 8b979ad0657..40891016c9d 100644 --- a/packages/amplify-cli/src/__tests__/extensions/amplify-helpers/delete-project.test.ts +++ b/packages/amplify-cli/src/__tests__/extensions/amplify-helpers/delete-project.test.ts @@ -96,6 +96,7 @@ describe('deleteProject', () => { amplify: { getEnvDetails: () => [], getProjectConfig: () => ({ frontend: 'test' }), + invokePluginMethod: async () => {}, }, filesystem: { remove: jest.fn(), diff --git a/packages/amplify-cli/src/commands/env/remove.ts b/packages/amplify-cli/src/commands/env/remove.ts index 0e08229c477..b422b881d57 100644 --- a/packages/amplify-cli/src/commands/env/remove.ts +++ b/packages/amplify-cli/src/commands/env/remove.ts @@ -1,10 +1,9 @@ import ora from 'ora'; -import { - FeatureFlags, stateManager, $TSContext, AmplifyError, -} from 'amplify-cli-core'; +import { FeatureFlags, stateManager, $TSContext, AmplifyError } from 'amplify-cli-core'; import { printer } from 'amplify-prompts'; import { getConfirmation } from '../../extensions/amplify-helpers/delete-project'; import { removeEnvFromCloud } from '../../extensions/amplify-helpers/remove-env-from-cloud'; +import { invokeDeleteEnvParamsFromService } from '../../extensions/amplify-helpers/invoke-delete-env-params'; /** * Entry point for env subcommand @@ -41,6 +40,7 @@ export const run = async (context: $TSContext): Promise => { spinner.start(); try { await removeEnvFromCloud(context, envName, confirmation.deleteS3); + await invokeDeleteEnvParamsFromService(context, envName); } catch (ex) { // safely exit spinner, then allow the exception to propagate up spinner.fail(`remove env failed: ${ex.message}`); diff --git a/packages/amplify-cli/src/commands/init.ts b/packages/amplify-cli/src/commands/init.ts index 2d22a869b54..b62ddc2f42d 100644 --- a/packages/amplify-cli/src/commands/init.ts +++ b/packages/amplify-cli/src/commands/init.ts @@ -18,9 +18,10 @@ const constructExeInfo = (context: $TSContext): void => { }; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type -const runStrategy = (quickstart: boolean) => (quickstart - ? [preInitSetup, analyzeProjectHeadless, scaffoldProjectHeadless, onHeadlessSuccess] - : [preInitSetup, analyzeProject, initFrontend, initProviders, onSuccess, postInitSetup]); +const runStrategy = (quickstart: boolean) => + quickstart + ? [preInitSetup, analyzeProjectHeadless, scaffoldProjectHeadless, onHeadlessSuccess] + : [preInitSetup, analyzeProject, initFrontend, initProviders, onSuccess, postInitSetup]; /** * entry point for the init command @@ -35,6 +36,6 @@ export const run = async (context: $TSContext): Promise => { } if (context.exeInfo.sourceEnvName && context.exeInfo.localEnvInfo.envName) { - await raisePostEnvAddEvent(context as unknown as Context, context.exeInfo.sourceEnvName, context.exeInfo.localEnvInfo.envName); + await raisePostEnvAddEvent((context as unknown) as Context, context.exeInfo.sourceEnvName, context.exeInfo.localEnvInfo.envName); } }; diff --git a/packages/amplify-cli/src/execution-manager.ts b/packages/amplify-cli/src/execution-manager.ts index f8936ce9f94..cd194355ddf 100644 --- a/packages/amplify-cli/src/execution-manager.ts +++ b/packages/amplify-cli/src/execution-manager.ts @@ -1,6 +1,5 @@ import * as fs from 'fs-extra'; import * as path from 'path'; -import sequential from 'promise-sequential'; import { prompter } from 'amplify-prompts'; import { twoStringSetsAreEqual, twoStringSetsAreDisjoint } from './utils/set-ops'; import { Context } from './domain/context'; @@ -335,7 +334,9 @@ export const raiseEvent = async (context: Context, args: }; return eventHandler; }); - await sequential(eventHandlers); + for (const eventHandler of eventHandlers) { + await eventHandler(); + } } }; diff --git a/packages/amplify-cli/src/extensions/amplify-helpers/delete-project.ts b/packages/amplify-cli/src/extensions/amplify-helpers/delete-project.ts index 2cb59e1ea80..dbd4c2c8491 100644 --- a/packages/amplify-cli/src/extensions/amplify-helpers/delete-project.ts +++ b/packages/amplify-cli/src/extensions/amplify-helpers/delete-project.ts @@ -10,6 +10,7 @@ import { getFrontendPlugins } from './get-frontend-plugins'; import { getPluginInstance } from './get-plugin-instance'; import { getAmplifyAppId } from './get-amplify-appId'; import { getAmplifyDirPath } from './path-manager'; +import { invokeDeleteEnvParamsFromService } from '../../extensions/amplify-helpers/invoke-delete-env-params'; /** * Deletes the amplify project from the cloud and local machine @@ -41,6 +42,10 @@ export const deleteProject = async (context: $TSContext): Promise => { printer.warn('Amplify App cannot be deleted, other environments still linked to Application'); } } + + // delete env parameters from service for each env + await Promise.all(envNames.map(envName => invokeDeleteEnvParamsFromService(context, envName))); + spinner.succeed('Project deleted in the cloud.'); } catch (ex) { if ('name' in ex && ex.name === 'BucketNotFoundError') { diff --git a/packages/amplify-cli/src/extensions/amplify-helpers/invoke-delete-env-params.ts b/packages/amplify-cli/src/extensions/amplify-helpers/invoke-delete-env-params.ts new file mode 100644 index 00000000000..111eb830965 --- /dev/null +++ b/packages/amplify-cli/src/extensions/amplify-helpers/invoke-delete-env-params.ts @@ -0,0 +1,9 @@ +import { $TSContext, constants } from 'amplify-cli-core'; + +export const invokeDeleteEnvParamsFromService = async (context: $TSContext, envName: string): Promise => { + const CloudFormationProviderName = constants.DEFAULT_PROVIDER; + await context.amplify.invokePluginMethod(context, CloudFormationProviderName, undefined, 'deleteEnvironmentParametersFromService', [ + context, + envName, + ]); +}; diff --git a/packages/amplify-cli/src/extensions/amplify-helpers/push-resources.ts b/packages/amplify-cli/src/extensions/amplify-helpers/push-resources.ts index 29cf26d3c36..a868914891a 100644 --- a/packages/amplify-cli/src/extensions/amplify-helpers/push-resources.ts +++ b/packages/amplify-cli/src/extensions/amplify-helpers/push-resources.ts @@ -8,6 +8,7 @@ import { stateManager, } from 'amplify-cli-core'; import { generateDependentResourcesType } from '@aws-amplify/amplify-category-custom'; +import { ensureEnvParamManager, IEnvironmentParameterManager } from '@aws-amplify/amplify-environment-parameters'; import { printer, prompter } from 'amplify-prompts'; import { getResources } from '../../commands/build'; import { initializeEnv } from '../../initialize-env'; @@ -90,6 +91,50 @@ export const pushResources = async ( return false; } + // Verify any environment parameters before push operation + const envParamManager = (await ensureEnvParamManager()).instance; + + const promptMissingParameter = async ( + categoryName: string, + resourceName: string, + parameterName: string, + envParamManager: IEnvironmentParameterManager, + ): Promise => { + printer.warn(`Could not find value for parameter ${parameterName}`); + const value = await prompter.input(`Enter a value for ${parameterName} for the ${categoryName} resource: ${resourceName}`); + const resourceParamManager = envParamManager.getResourceParamManager(categoryName, resourceName); + resourceParamManager.setParam(parameterName, value); + }; + + const parametersToCheck = resourcesToBuild + .filter(({ category: c, resourceName: r }) => { + // Filter based on optional parameters + if (category) { + if (c !== category) { + return false; + } + } + if (resourceName) { + if (r !== resourceName) { + return false; + } + } + return true; + }); + + + if (context?.exeInfo?.inputParams?.yes || context?.exeInfo?.inputParams?.headless) { + await envParamManager.verifyExpectedEnvParameters(parametersToCheck); + } else { + const missingParameters = await envParamManager.getMissingParameters(parametersToCheck); + if (missingParameters.length > 0) { + for (const { categoryName, resourceName, parameterName } of missingParameters) { + await promptMissingParameter(categoryName, resourceName, parameterName , envParamManager); + } + await envParamManager.save(); // Values must be in TPI for CFN deployment to work + } + } + // rebuild has an upstream confirmation prompt so no need to prompt again here let continueToPush = !!context?.exeInfo?.inputParams?.yes || rebuild; diff --git a/packages/amplify-cli/src/index.ts b/packages/amplify-cli/src/index.ts index d6f3ff3201d..fbdaa74221d 100644 --- a/packages/amplify-cli/src/index.ts +++ b/packages/amplify-cli/src/index.ts @@ -18,7 +18,7 @@ import { EventEmitter } from 'events'; import * as fs from 'fs-extra'; import * as path from 'path'; import { printer, prompter } from 'amplify-prompts'; -import { saveAll as saveAllEnvParams } from '@aws-amplify/amplify-environment-parameters'; +import { saveAll as saveAllEnvParams, ServiceUploadHandler } from '@aws-amplify/amplify-environment-parameters'; import { logInput } from './conditional-local-logging-init'; import { attachUsageData, constructContext } from './context-manager'; import { displayBannerMessages } from './display-banner-messages'; @@ -151,18 +151,35 @@ export const run = async (startTime: number): Promise => { await displayBannerMessages(input); await executeCommand(context); - const exitCode = process.exitCode || 0; - if (exitCode === 0) { - await context.usageData.emitSuccess(); - } - // no command supplied defaults to help, give update notification at end of execution if (input.command === 'help') { // Checks for available update, defaults to a 1 day interval for notification notify({ defer: true, isGlobal: true }); } - await saveAllEnvParams(); + if (context.input.command === 'push') { + const { providers } = stateManager.getProjectConfig(undefined, { throwIfNotExist: false, default: {} }); + const CloudFormationProviderName = 'awscloudformation'; + let uploadHandler: ServiceUploadHandler | undefined; + if (Array.isArray(providers) && providers.find((value) => value === CloudFormationProviderName)) { + uploadHandler = await context.amplify.invokePluginMethod( + context, + CloudFormationProviderName, + undefined, + 'getEnvParametersUploadHandler', + [context], + ); + } + await saveAllEnvParams(uploadHandler); + } + else { + await saveAllEnvParams(); + } + + const exitCode = process.exitCode || 0; + if (exitCode === 0) { + await context.usageData.emitSuccess(); + } }; const ensureFilePermissions = (filePath: string): void => { diff --git a/packages/amplify-cli/src/init-steps/s0-analyzeProject.ts b/packages/amplify-cli/src/init-steps/s0-analyzeProject.ts index c6a80b2fb31..9d66018e67b 100644 --- a/packages/amplify-cli/src/init-steps/s0-analyzeProject.ts +++ b/packages/amplify-cli/src/init-steps/s0-analyzeProject.ts @@ -1,6 +1,4 @@ -import { - $TSContext, AmplifyError, stateManager, -} from 'amplify-cli-core'; +import { $TSContext, AmplifyError, stateManager } from 'amplify-cli-core'; import * as fs from 'fs-extra'; import * as inquirer from 'inquirer'; import * as path from 'path'; diff --git a/packages/amplify-cli/src/initialize-env.ts b/packages/amplify-cli/src/initialize-env.ts index 9ddc7ac67ea..d111fb988f1 100644 --- a/packages/amplify-cli/src/initialize-env.ts +++ b/packages/amplify-cli/src/initialize-env.ts @@ -1,7 +1,8 @@ import sequential from 'promise-sequential'; -import { stateManager, $TSAny, $TSMeta, $TSContext, AmplifyFault, spinner } from 'amplify-cli-core'; +import { stateManager, $TSAny, $TSMeta, $TSContext, AmplifyFault, constants, spinner } from 'amplify-cli-core'; import { printer } from 'amplify-prompts'; -import { ensureEnvParamManager, IEnvironmentParameterManager } from '@aws-amplify/amplify-environment-parameters'; +import { ensureEnvParamManager, IEnvironmentParameterManager, ServiceDownloadHandler } from '@aws-amplify/amplify-environment-parameters'; + import { getProviderPlugins } from './extensions/amplify-helpers/get-provider-plugins'; import { ManuallyTimedCodePath } from './domain/amplify-usageData/UsageDataTypes'; @@ -24,6 +25,18 @@ export const initializeEnv = async ( amplifyMeta.providers.awscloudformation = teamProviderInfo?.[currentEnv]?.awscloudformation; const envParamManager = (await ensureEnvParamManager(currentEnv)).instance; + const { providers } = stateManager.getProjectConfig(undefined, { throwIfNotExist: false, default: {} }); + const CloudFormationProviderName = constants.DEFAULT_PROVIDER; + if (Array.isArray(providers) && providers.find((value) => value === CloudFormationProviderName)) { + const downloadHandler: ServiceDownloadHandler = await context.amplify.invokePluginMethod( + context, + CloudFormationProviderName, + undefined, + 'getEnvParametersDownloadHandler', + [context], + ); + await envParamManager.downloadParameters(downloadHandler); + } if (!context.exeInfo.restoreBackend) { mergeBackendConfigIntoAmplifyMeta(projectPath, amplifyMeta); diff --git a/packages/amplify-e2e-core/src/utils/sdk-calls.ts b/packages/amplify-e2e-core/src/utils/sdk-calls.ts index 3fa1a21f5d2..493c5b3b78d 100644 --- a/packages/amplify-e2e-core/src/utils/sdk-calls.ts +++ b/packages/amplify-e2e-core/src/utils/sdk-calls.ts @@ -82,10 +82,12 @@ export const getDeploymentBucketObject = async (projectRoot: string, objectKey: const meta = getProjectMeta(projectRoot); const deploymentBucket = meta.providers.awscloudformation.DeploymentBucketName; const s3 = new S3(); - const result = await s3.getObject({ - Bucket: deploymentBucket, - Key: objectKey, - }).promise(); + const result = await s3 + .getObject({ + Bucket: deploymentBucket, + Key: objectKey, + }) + .promise(); return result.Body.toLocaleString(); }; @@ -198,8 +200,8 @@ export const addUserToUserPool = async (userPoolId: string, region: string) => { export const listUserPoolGroupsForUser = async (userPoolId: string, userName: string, region: string): Promise => { const provider = new CognitoIdentityServiceProvider({ region }); const params = { - UserPoolId: userPoolId, /* required */ - Username: userName, /* required */ + UserPoolId: userPoolId /* required */, + Username: userName /* required */, }; const res = await provider.adminListGroupsForUser(params).promise(); const groups = res.Groups.map(group => group.GroupName); @@ -401,42 +403,128 @@ export const getSSMParameters = async (region: string, appId: string, envName: s .promise(); }; +export const getSSMParametersCategoryPrefix = async ( + region: string, + appId: string, + envName: string, + category: string, + resourceName: string, + parameterNames: string[], +) => { + const ssmClient = new SSM({ region }); + if (!parameterNames || parameterNames.length === 0) { + throw new Error('no parameterNames specified'); + } + return ssmClient + .getParameters({ + Names: parameterNames.map(name => `/amplify/${appId}/${envName}/AMPLIFY_${category}_${resourceName}_${name}`), + }) + .promise(); +}; + +export const getAllSSMParamatersForAppId = async (appId: string, region: string): Promise> => { + const ssmClient = new SSM({ region }); + const retrievedParameters: Array = []; + let receivedNextToken = ''; + do { + const ssmArgument = getSsmSdkParametersByPath(appId, receivedNextToken); + const data = await ssmClient.getParametersByPath(ssmArgument).promise(); + retrievedParameters.push(...data.Parameters.map(returnedParameter => returnedParameter.Name)); + receivedNextToken = data.NextToken; + } while (receivedNextToken); + return retrievedParameters; +}; + +export const expectParametersOptionalValue = async ( + expectToExist: NameOptionalValuePair[], + expectNotExist: string[], + region: string, + appId: string, + envName: string, + category: string, + resourceName: string, +): Promise => { + const parametersToRequest = expectToExist.map(exist => exist.name).concat(expectNotExist); + const result = await getSSMParametersCategoryPrefix(region, appId, envName, category, resourceName, parametersToRequest); + const mapName = (name: string) => `/amplify/${appId}/${envName}/AMPLIFY_${category}_${resourceName}_${name}`; + expect(result.InvalidParameters.length).toBe(expectNotExist.length); + expect(result.InvalidParameters.sort()).toEqual(expectNotExist.map(mapName).sort()); + expect(result.Parameters.length).toBe(expectToExist.length); + const mappedResult = result.Parameters.map(param => ({ name: param.Name, value: JSON.parse(param.Value) })).sort(sortByName); + const mappedExpect = expectToExist.map(exist => ({ name: mapName(exist.name), value: exist.value ? exist.value : '' })).sort(sortByName); + + const mappedResultKeys = mappedResult.map(parameter => parameter.name); + for (const expectedParam of mappedExpect) { + if (expectedParam.value) { + expect(mappedResult).toContainEqual(expectedParam); + } else { + expect(mappedResultKeys).toContainEqual(expectedParam.name); + } + } +}; + +const sortByName = (a: NameOptionalValuePair, b: NameOptionalValuePair) => a.name.localeCompare(b.name); +type NameOptionalValuePair = { name: string; value?: string }; + +const getSsmSdkParametersByPath = (appId: string, nextToken?: string): SsmGetParametersByPathArgument => { + const sdkParameters: SsmGetParametersByPathArgument = { Path: `/amplify/${appId}/` }; + if (nextToken) { + sdkParameters.NextToken = nextToken; + } + return sdkParameters; +}; + +type SsmGetParametersByPathArgument = { + Path: string; + NextToken?: string; +}; + // Amazon location service calls export const getMap = async (mapName: string, region: string) => { const service = new Location({ region }); - return await service.describeMap({ - MapName: mapName, - }).promise(); + return await service + .describeMap({ + MapName: mapName, + }) + .promise(); }; export const getPlaceIndex = async (placeIndexName: string, region: string) => { const service = new Location({ region }); - return await service.describePlaceIndex({ - IndexName: placeIndexName, - }).promise(); + return await service + .describePlaceIndex({ + IndexName: placeIndexName, + }) + .promise(); }; export const getGeofenceCollection = async (geofenceCollectionName: string, region: string) => { const service = new Location({ region }); - return await service.describeGeofenceCollection({ - CollectionName: geofenceCollectionName, - }).promise(); + return await service + .describeGeofenceCollection({ + CollectionName: geofenceCollectionName, + }) + .promise(); }; export const getGeofence = async (geofenceCollectionName: string, geofenceId: string, region: string) => { const service = new Location({ region }); - return (await service.getGeofence({ - CollectionName: geofenceCollectionName, - GeofenceId: geofenceId, - })).promise(); + return ( + await service.getGeofence({ + CollectionName: geofenceCollectionName, + GeofenceId: geofenceId, + }) + ).promise(); }; // eslint-disable-next-line spellcheck/spell-checker export const listGeofences = async (geofenceCollectionName: string, region: string, nextToken: string = null) => { const service = new Location({ region }); // eslint-disable-next-line spellcheck/spell-checker - return (await service.listGeofences({ - CollectionName: geofenceCollectionName, - NextToken: nextToken, - })).promise(); + return ( + await service.listGeofences({ + CollectionName: geofenceCollectionName, + NextToken: nextToken, + }) + ).promise(); }; diff --git a/packages/amplify-e2e-tests/src/__tests__/parameter-store.test.ts b/packages/amplify-e2e-tests/src/__tests__/parameter-store.test.ts new file mode 100644 index 00000000000..d0720e03166 --- /dev/null +++ b/packages/amplify-e2e-tests/src/__tests__/parameter-store.test.ts @@ -0,0 +1,193 @@ +import { + addAuthWithEmailVerificationAndUserPoolGroupTriggers, + addFunction, + amplifyPull, + amplifyPushAuth, + createNewProjectDir, + deleteProject, + deleteProjectDir, + expectParametersOptionalValue, + generateRandomShortId, + getAppId, + getProjectMeta, + getTeamProviderInfo, + gitCleanFdx, + gitCommitAll, + gitInit, + initJSProjectWithProfile, +} from '@aws-amplify/amplify-e2e-core'; +import { + addEnvironmentCarryOverEnvVars, + checkoutEnvironment, + removeEnvironment, +} from '../environment/env'; + +describe('upload and delete parameters', () => { + let projRoot: string; + beforeEach(async () => { + projRoot = await createNewProjectDir('upload-delete-parameters-test'); + }); + + afterEach(async () => { + deleteProjectDir(projRoot); + }); + + it('adding function should upload to service, removing environment and deleting project should delete parameters', async () => { + const firstEnvName = 'enva'; + const secondEnvName = 'envb'; + const envVariableName = 'envVariableName'; + const envVariableValue = 'envVariableValue'; + await initJSProjectWithProfile(projRoot, { disableAmplifyAppCreation: false, envName: firstEnvName }); + + const meta = getProjectMeta(projRoot); + expect(meta).toBeDefined(); + const appId = getAppId(projRoot); + expect(appId).toBeDefined(); + const region = meta.providers.awscloudformation.Region; + expect(region).toBeDefined(); + + const fnName = `parameterstestfn${generateRandomShortId()}`; + await addFunction( + projRoot, + { + name: fnName, + functionTemplate: 'Hello World', + environmentVariables: { + key: envVariableName, + value: envVariableValue, + }, + }, + 'nodejs', + ); + await amplifyPushAuth(projRoot); + const expectedParamsAfterAddFunc = [ + { name: 'deploymentBucketName' }, + { name: envVariableName, value: envVariableValue }, + { name: 's3Key' }, + ]; + await expectParametersOptionalValue(expectedParamsAfterAddFunc, [], region, appId, firstEnvName, 'function', fnName); + + await addEnvironmentCarryOverEnvVars(projRoot, { envName: secondEnvName }); + await amplifyPushAuth(projRoot); + const expectedParamsAfterAddEnv = [ + { name: 'deploymentBucketName' }, + { name: envVariableName, value: envVariableValue }, + { name: 's3Key' }, + ]; + await expectParametersOptionalValue(expectedParamsAfterAddFunc, [], region, appId, firstEnvName, 'function', fnName); + await expectParametersOptionalValue(expectedParamsAfterAddEnv, [], region, appId, secondEnvName, 'function', fnName); + + await checkoutEnvironment(projRoot, { envName: firstEnvName }); + await removeEnvironment(projRoot, { envName: secondEnvName }); + await amplifyPushAuth(projRoot); + await expectParametersOptionalValue(expectedParamsAfterAddFunc, [], region, appId, firstEnvName, 'function', fnName); + await expectParametersOptionalValue( + [], + expectedParamsAfterAddEnv.map(pair => pair.name), + region, + appId, + secondEnvName, + 'function', + fnName, + ); + + await deleteProject(projRoot); + await expectParametersOptionalValue( + [], + expectedParamsAfterAddFunc.map(pair => pair.name), + region, + appId, + firstEnvName, + 'function', + fnName, + ); + await expectParametersOptionalValue( + [], + expectedParamsAfterAddEnv.map(pair => pair.name), + region, + appId, + secondEnvName, + 'function', + fnName, + ); + }); +}); + +describe('parameters in Parameter Store', () => { + let projRoot: string; + const envName = 'enva'; + + beforeAll(async () => { + projRoot = await createNewProjectDir('multi-env-parameters-test'); + }); + + afterAll(async () => { + await deleteProject(projRoot); + deleteProjectDir(projRoot); + }); + + it('hydrates missing parameters into TPI on pull', async () => { + await initJSProjectWithProfile(projRoot, { disableAmplifyAppCreation: false, envName }); + const meta = getProjectMeta(projRoot); + expect(meta).toBeDefined(); + const appId = getAppId(projRoot); + expect(appId).toBeDefined(); + const region = meta.providers.awscloudformation.Region; + expect(region).toBeDefined(); + await gitInit(projRoot); + await gitCommitAll(projRoot); // commit all just after init, so no categories block exists in TPI yet + + const envVariableName = 'envVariableName'; + const envVariableValue = 'envVariableValue'; + + const fnName = `parameterstestfn${generateRandomShortId()}`; + await addFunction( + projRoot, + { + name: fnName, + functionTemplate: 'Hello World', + environmentVariables: { + key: envVariableName, + value: envVariableValue, + }, + }, + 'nodejs', + ); + await addAuthWithEmailVerificationAndUserPoolGroupTriggers(projRoot); + await amplifyPushAuth(projRoot); + const expectedParamsAfterPush = [ + { name: 'deploymentBucketName' }, + { name: envVariableName, value: envVariableValue }, + { name: 's3Key' }, + ]; + await expectParametersOptionalValue(expectedParamsAfterPush, [], region, appId, envName, 'function', fnName); + + const preCleanTpi = getTeamProviderInfo(projRoot); + + // test pull --restore same dir + await gitCleanFdx(projRoot); // clear TPI + await amplifyPull(projRoot, { appId, envName, withRestore: true, emptyDir: true }); + const postPullWithRestoreTpi = getTeamProviderInfo(projRoot); + expect(postPullWithRestoreTpi).toEqual(preCleanTpi); + + // test pull same dir + await gitCleanFdx(projRoot); // clear TPI + await amplifyPull(projRoot, { appId, envName, withRestore: false, emptyDir: true }); + const postPullWithoutRestoreTpi = getTeamProviderInfo(projRoot); + expect(postPullWithoutRestoreTpi).toEqual(preCleanTpi); + + expect(await getTpiAfterPullInEmptyDir(appId, envName, true)).toEqual(preCleanTpi); + expect(await getTpiAfterPullInEmptyDir(appId, envName, false)).toEqual(preCleanTpi); + }); + + const getTpiAfterPullInEmptyDir = async (appId: string, envName: string, withRestore: boolean): Promise> => { + let emptyDir: string; + try { + emptyDir = await createNewProjectDir('empty-dir-parameters-test'); + await amplifyPull(emptyDir, { appId, envName, withRestore, emptyDir: true }); + return getTeamProviderInfo(emptyDir); + } finally { + deleteProjectDir(emptyDir); + } + }; +}); diff --git a/packages/amplify-e2e-tests/src/environment/env.ts b/packages/amplify-e2e-tests/src/environment/env.ts index 2be040357d9..2f1be951197 100644 --- a/packages/amplify-e2e-tests/src/environment/env.ts +++ b/packages/amplify-e2e-tests/src/environment/env.ts @@ -1,6 +1,6 @@ import { nspawn as spawn, getCLIPath, getSocialProviders, isCI } from '@aws-amplify/amplify-e2e-core'; -export function addEnvironment(cwd: string, settings: { envName: string; numLayers?: number }): Promise { +export function addEnvironment(cwd: string, settings: { envName: string; numLayers?: number; cloneParams?: boolean }): Promise { return new Promise((resolve, reject) => { const chain = spawn(getCLIPath(), ['env', 'add'], { cwd, stripColors: true }) .wait('Enter a name for the environment') @@ -20,6 +20,20 @@ export function addEnvironment(cwd: string, settings: { envName: string; numLaye }); } +export async function addEnvironmentCarryOverEnvVars(cwd: string, settings: { envName: string }): Promise { + return spawn(getCLIPath(), ['env', 'add'], { cwd, stripColors: true }) + .wait('Enter a name for the environment') + .sendLine(settings.envName) + .wait('Select the authentication method you want to use:') + .sendCarriageReturn() + .wait('Please choose the profile you want to use') + .sendCarriageReturn() + .wait('You have configured environment variables for functions. How do you want to proceed?') + .sendCarriageReturn() + .wait('Initialized your environment successfully.') + .runAsync(); +} + export function updateEnvironment(cwd: string, settings: { permissionsBoundaryArn: string }) { return new Promise((resolve, reject) => { spawn(getCLIPath(), ['env', 'update'], { cwd, stripColors: true }) diff --git a/packages/amplify-environment-parameters/API.md b/packages/amplify-environment-parameters/API.md index 8f45078f79b..7acac9904f0 100644 --- a/packages/amplify-environment-parameters/API.md +++ b/packages/amplify-environment-parameters/API.md @@ -4,6 +4,11 @@ ```ts +import { IAmplifyResource } from 'amplify-cli-core'; + +// @public (undocumented) +export const cloneEnvParamManager: (srcEnvParamManager: IEnvironmentParameterManager, destEnvName: string) => Promise; + // @public (undocumented) export const ensureEnvParamManager: (envName?: string) => Promise<{ instance: IEnvironmentParameterManager; @@ -14,11 +19,19 @@ export const getEnvParamManager: (envName?: string) => IEnvironmentParameterMana // @public (undocumented) export type IEnvironmentParameterManager = { + cloneEnvParamsToNewEnvParamManager: (destManager: IEnvironmentParameterManager) => Promise; + downloadParameters: (downloadHandler: ServiceDownloadHandler) => Promise; + getMissingParameters: (resourceFilterList?: IAmplifyResource[]) => Promise<{ + categoryName: string; + resourceName: string; + parameterName: string; + }[]>; + getResourceParamManager: (category: string, resource: string) => ResourceParameterManager; + hasResourceParamManager: (category: string, resource: string) => boolean; init: () => Promise; removeResourceParamManager: (category: string, resource: string) => void; - hasResourceParamManager: (category: string, resource: string) => boolean; - getResourceParamManager: (category: string, resource: string) => ResourceParameterManager; - save: () => Promise; + save: (serviceUploadHandler?: ServiceUploadHandler) => Promise; + verifyExpectedEnvParameters: (resourceFilterList?: IAmplifyResource[]) => Promise; }; // @public (undocumented) @@ -42,7 +55,13 @@ export class ResourceParameterManager { } // @public (undocumented) -export const saveAll: () => Promise; +export const saveAll: (serviceUploadHandler?: ServiceUploadHandler) => Promise; + +// @public (undocumented) +export type ServiceDownloadHandler = (parameters: string[]) => Promise>; + +// @public (undocumented) +export type ServiceUploadHandler = (key: string, value: string | number | boolean) => Promise; // (No @packageDocumentation comment for this package) diff --git a/packages/amplify-environment-parameters/src/__tests__/clone-env-param-manager.test.ts b/packages/amplify-environment-parameters/src/__tests__/clone-env-param-manager.test.ts new file mode 100644 index 00000000000..2d90f53f800 --- /dev/null +++ b/packages/amplify-environment-parameters/src/__tests__/clone-env-param-manager.test.ts @@ -0,0 +1,29 @@ +import { ensureEnvParamManager, IEnvironmentParameterManager } from '../environment-parameter-manager'; +import * as environmentParameterManager from '../environment-parameter-manager'; +import { cloneEnvParamManager } from '../clone-env-param-manager'; + +describe('clone env params test', () => { + const mockEnvParamManagerCloneFn = jest.fn().mockReturnValue(Promise.resolve()); + const envParamManager = { + instance: { + cloneEnvParamsToNewEnvParamManager: mockEnvParamManagerCloneFn, + downloadParameters: jest.fn(), + getMissingParameters: jest.fn(), + getResourceParamManager: jest.fn(), + hasResourceParamManager: jest.fn(), + init: jest.fn(), + removeResourceParamManager: jest.fn(), + save: jest.fn(), + verifyExpectedEnvParameters: jest.fn(), + } as IEnvironmentParameterManager, + }; + + jest.spyOn(environmentParameterManager, 'ensureEnvParamManager').mockReturnValue(Promise.resolve(envParamManager)); + + it('check if func is called', async () => { + const envParamManagerA: IEnvironmentParameterManager = (await ensureEnvParamManager('enva')).instance; + await cloneEnvParamManager(envParamManagerA, 'envB'); + expect(ensureEnvParamManager).toBeCalledTimes(2); + expect(mockEnvParamManagerCloneFn).toBeCalledTimes(1); + }); +}); diff --git a/packages/amplify-environment-parameters/src/__tests__/environment-parameter-manager.test.ts b/packages/amplify-environment-parameters/src/__tests__/environment-parameter-manager.test.ts index e82961158f1..72b8d06c786 100644 --- a/packages/amplify-environment-parameters/src/__tests__/environment-parameter-manager.test.ts +++ b/packages/amplify-environment-parameters/src/__tests__/environment-parameter-manager.test.ts @@ -100,3 +100,79 @@ describe('save', () => { `); }); }); + +describe('verifyExpectedEnvParameters', () => { + it('does not throw when nothing is missing', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + envParamManager.save(); + await envParamManager.verifyExpectedEnvParameters(); + }); + + it('throws when a parameter is missing', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + const funcParamManager = envParamManager.getResourceParamManager('function', 'funcName'); + + funcParamManager.setParam('missingParam', 'missingValue'); + envParamManager.save(); + funcParamManager.deleteParam('missingParam'); + + let error = undefined; + try { + await envParamManager.verifyExpectedEnvParameters(); + } catch (e) { + error = e; + } + expect(error).toBeDefined(); + }); + + it('does not throw when a parameter is missing on an ignored resource', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + const funcParamManager = envParamManager.getResourceParamManager('function', 'funcName'); + + funcParamManager.setParam('missingParam', 'missingValue'); + envParamManager.save(); + funcParamManager.deleteParam('missingParam'); + + let error = undefined; + try { + await envParamManager.verifyExpectedEnvParameters([{ category: 'auth', resourceName: 'mockAuth', service: 'Cognito' }]); + } catch (e) { + error = e; + } + expect(error).not.toBeDefined(); + }); +}); + +describe('getMissingParameters', () => { + it('returns an empty array when nothing is missing', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + envParamManager.save(); + expect(await envParamManager.getMissingParameters()).toEqual([]); + }); + + it('returns array of missing parameters', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + const funcParamManager = envParamManager.getResourceParamManager('function', 'funcName'); + + funcParamManager.setParam('missingParam', 'missingValue'); + envParamManager.save(); + funcParamManager.deleteParam('missingParam'); + + expect(await envParamManager.getMissingParameters()) + .toEqual([{ categoryName: 'function', resourceName: 'funcName', parameterName: 'missingParam' }]); + }); + + it('returns an empty array when a parameter is missing on an ignored resource', async () => { + const envParamManager = (await ensureEnvParamManager()).instance; + const funcParamManager = envParamManager.getResourceParamManager('function', 'funcName'); + + funcParamManager.setParam('missingParam', 'missingValue'); + envParamManager.save(); + funcParamManager.deleteParam('missingParam'); + + expect( + await envParamManager.getMissingParameters([{ category: 'auth', resourceName: 'mockAuth', service: 'Cognito' }]) + ).toEqual([]); + + }); +}); diff --git a/packages/amplify-environment-parameters/src/backend-config-parameters-controller.ts b/packages/amplify-environment-parameters/src/backend-config-parameters-controller.ts index e709d76ba0b..f7f1952c13a 100644 --- a/packages/amplify-environment-parameters/src/backend-config-parameters-controller.ts +++ b/packages/amplify-environment-parameters/src/backend-config-parameters-controller.ts @@ -1,8 +1,4 @@ -import { - ResourceTuple, - stateManager, - AmplifyError, -} from 'amplify-cli-core'; +import { ResourceTuple, stateManager, AmplifyError } from 'amplify-cli-core'; import Ajv from 'ajv'; import { BackendParameters } from './backend-parameters'; import parameterMapSchema from './schemas/BackendParameters.schema.json'; @@ -11,13 +7,13 @@ import parameterMapSchema from './schemas/BackendParameters.schema.json'; * Interface for controller that maps parameters to resources that depend on those parameters */ export type IBackendParametersController = { - save: () => Promise, - addParameter: (name: string, usedBy: ResourceTuple[]) => IBackendParametersController, - addAllParameters: (parameterMap: BackendParameters) => IBackendParametersController, - removeParameter: (name: string) => IBackendParametersController, - removeAllParameters: () => IBackendParametersController, - getParameters: () => Readonly, -} + save: () => Promise; + addParameter: (name: string, usedBy: ResourceTuple[]) => IBackendParametersController; + addAllParameters: (parameterMap: BackendParameters) => IBackendParametersController; + removeParameter: (name: string) => IBackendParametersController; + removeAllParameters: () => IBackendParametersController; + getParameters: () => Readonly; +}; let localBackendParametersController: IBackendParametersController; diff --git a/packages/amplify-environment-parameters/src/clone-env-param-manager.ts b/packages/amplify-environment-parameters/src/clone-env-param-manager.ts new file mode 100644 index 00000000000..bb8c5a4caed --- /dev/null +++ b/packages/amplify-environment-parameters/src/clone-env-param-manager.ts @@ -0,0 +1,6 @@ +import { IEnvironmentParameterManager, ensureEnvParamManager } from './environment-parameter-manager'; + +export const cloneEnvParamManager = async (srcEnvParamManager: IEnvironmentParameterManager, destEnvName: string): Promise => { + const destManager = (await ensureEnvParamManager(destEnvName)).instance; + await srcEnvParamManager.cloneEnvParamsToNewEnvParamManager(destManager); +}; diff --git a/packages/amplify-environment-parameters/src/environment-parameter-manager.ts b/packages/amplify-environment-parameters/src/environment-parameter-manager.ts index eadfe1f1d2c..088ab940820 100644 --- a/packages/amplify-environment-parameters/src/environment-parameter-manager.ts +++ b/packages/amplify-environment-parameters/src/environment-parameter-manager.ts @@ -1,4 +1,4 @@ -import { AmplifyFault, pathManager, stateManager } from 'amplify-cli-core'; +import { AmplifyError, AmplifyFault, IAmplifyResource, pathManager, stateManager } from 'amplify-cli-core'; import _ from 'lodash'; import { getParametersControllerInstance, IBackendParametersController } from './backend-config-parameters-controller'; import { ResourceParameterManager } from './resource-parameter-manager'; @@ -37,10 +37,10 @@ export const getEnvParamManager = (envName: string = stateManager.getLocalEnvInf /** * Execute the save method of all currently initialized IEnvironmentParameterManager instances */ -export const saveAll = async (): Promise => { +export const saveAll = async (serviceUploadHandler?: ServiceUploadHandler): Promise => { for (const envParamManager of Object.values(envParamManagerMap)) { // save methods must be executed in sequence to avoid race conditions writing to the tpi file - await envParamManager.save(); + await envParamManager.save(serviceUploadHandler); } }; @@ -85,7 +85,19 @@ class EnvironmentParameterManager implements IEnvironmentParameterManager { return !!this.resourceParamManagers[getResourceKey(category, resource)]; } - async save(): Promise { + async cloneEnvParamsToNewEnvParamManager(destManager: IEnvironmentParameterManager): Promise { + const resourceKeys = Object.keys(this.resourceParamManagers); + const categoryResourceNamePairs: string[][] = resourceKeys.map(key => key.split('_')); + categoryResourceNamePairs.forEach(([category, resourceName]) => { + const srcResourceParamManager: ResourceParameterManager = this.getResourceParamManager(category, resourceName); + const allSrcParams: Record = srcResourceParamManager.getAllParams(); + const destResourceParamManager: ResourceParameterManager = destManager.getResourceParamManager(category, resourceName); + destResourceParamManager.setAllParams(allSrcParams); + }); + await destManager.save(); + } + + async save(serviceUploadHandler?: ServiceUploadHandler): Promise { if (!pathManager.findProjectRoot()) { // assume that the project is deleted if we cannot find a project root return; @@ -109,18 +121,72 @@ class EnvironmentParameterManager implements IEnvironmentParameterManager { // update param mapping this.parameterMapController.removeAllParameters(); - Object.entries(this.resourceParamManagers).forEach(([resourceKey, paramManager]) => { + for (const [resourceKey, paramManager] of Object.entries(this.resourceParamManagers)) { const [category, resourceName] = splitResourceKey(resourceKey); const resourceParams = paramManager.getAllParams(); - Object.entries(resourceParams).forEach(([paramName]) => { + for (const [paramName, paramValue] of Object.entries(resourceParams)) { const ssmParamName = getParameterStoreKey(category, resourceName, paramName); this.parameterMapController.addParameter(ssmParamName, [{ category, resourceName }]); - }); - }); - // uploading values to PS will go here + if (serviceUploadHandler) { + await serviceUploadHandler(ssmParamName, paramValue); + } + } + } + await this.parameterMapController.save(); } + async downloadParameters(downloadHandler: ServiceDownloadHandler): Promise { + const missingParameters = (await this.getMissingParameters()) + .map(({ categoryName, resourceName, parameterName }) => getParameterStoreKey(categoryName, resourceName, parameterName)); + const params = await downloadHandler(missingParameters); + Object.entries(params).forEach(([key, value]) => { + const [categoryName, resourceName, parameterName] = getNamesFromParameterStoreKey(key); + const resourceParamManager = this.getResourceParamManager(categoryName, resourceName); + resourceParamManager.setParam(parameterName, value as string); // TODO remove need for type assertion + }); + } + + async getMissingParameters(resourceFilterList?: IAmplifyResource[]): Promise { + const expectedParameters = this.parameterMapController.getParameters(); + const allEnvParams = new Set(); + const missingResourceParameters: ResourceParameter[] = []; + + for (const [resourceKey, paramManager] of Object.entries(this.resourceParamManagers)) { + const resourceParams = paramManager.getAllParams(); + for (const paramName of Object.keys(resourceParams)) { + allEnvParams.add(`${resourceKey}_${paramName}`); + } + } + + Object.keys(expectedParameters).forEach(expectedParameter => { + const [categoryName, resourceName, parameterName] = getNamesFromParameterStoreKey(expectedParameter); + if (resourceFilterList && !resourceFilterList.some( + ({ category, resourceName: resource }) => categoryName === category && resource === resourceName + )) { + return; + } + if (!allEnvParams.has(`${categoryName}_${resourceName}_${parameterName}`)) { + missingResourceParameters.push({ categoryName, resourceName, parameterName }); + } + }); + + return missingResourceParameters; + } + + /** + * Throw an error if expected parameters are missing + */ + async verifyExpectedEnvParameters(resourceFilterList?: IAmplifyResource[]): Promise { + const missingParameterNames = await this.getMissingParameters(resourceFilterList); + + if (missingParameterNames.length > 0) { + throw new AmplifyError('MissingExpectedParameterError', { + message: `Expected parameter${missingParameterNames.length === 1 ? '' : 's'} ${missingParameterNames.join(', ')}`, + }); + } + } + private serializeTPICategories(): Record { return Object.entries(this.resourceParamManagers).reduce((acc, [resourceKey, resourceParams]) => { _.setWith(acc, splitResourceKey(resourceKey), resourceParams.getAllParams(), Object); @@ -141,12 +207,35 @@ const splitResourceKey = (key: string): readonly [string, string] => { * Interface for environment parameter managers */ export type IEnvironmentParameterManager = { + cloneEnvParamsToNewEnvParamManager: (destManager: IEnvironmentParameterManager) => Promise; + downloadParameters: (downloadHandler: ServiceDownloadHandler) => Promise; + getMissingParameters: (resourceFilterList?: IAmplifyResource[]) => + Promise<{ categoryName: string; resourceName: string; parameterName: string }[]>; + getResourceParamManager: (category: string, resource: string) => ResourceParameterManager; + hasResourceParamManager: (category: string, resource: string) => boolean; init: () => Promise; removeResourceParamManager: (category: string, resource: string) => void; - hasResourceParamManager: (category: string, resource: string) => boolean; - getResourceParamManager: (category: string, resource: string) => ResourceParameterManager; - save: () => Promise; -}; + save: (serviceUploadHandler?: ServiceUploadHandler) => Promise; + verifyExpectedEnvParameters: (resourceFilterList?: IAmplifyResource[]) => Promise; +} -const getParameterStoreKey = (categoryName: string, resourceName: string, paramName: string): string => - `AMPLIFY_${categoryName}_${resourceName}_${paramName}`; +export type ServiceUploadHandler = (key: string, value: string | number | boolean) => Promise; +export type ServiceDownloadHandler = (parameters: string[]) => Promise>; + +const getParameterStoreKey = ( + categoryName: string, + resourceName: string, + paramName: string, +): string => `AMPLIFY_${categoryName}_${resourceName}_${paramName}`; + +const getNamesFromParameterStoreKey = (fullParameter: string) => { + const [, categoryName, resourceName] = fullParameter.split('_'); // Ignores the AMPLIFY prefix + const parameterName = fullParameter.split('_').slice(3).join('_'); // In case parameterName contains underscores + return [categoryName, resourceName, parameterName]; +} + +type ResourceParameter = { + categoryName: string; + resourceName: string; + parameterName: string; +} diff --git a/packages/amplify-environment-parameters/src/index.ts b/packages/amplify-environment-parameters/src/index.ts index e6f2d6698e6..dece98dc0e5 100644 --- a/packages/amplify-environment-parameters/src/index.ts +++ b/packages/amplify-environment-parameters/src/index.ts @@ -1,2 +1,3 @@ export * from './environment-parameter-manager'; export { ResourceParameterManager } from './resource-parameter-manager'; +export * from './clone-env-param-manager'; diff --git a/packages/amplify-provider-awscloudformation/API.md b/packages/amplify-provider-awscloudformation/API.md index 8bc59bd1991..e105aedc004 100644 --- a/packages/amplify-provider-awscloudformation/API.md +++ b/packages/amplify-provider-awscloudformation/API.md @@ -14,6 +14,9 @@ import { Template } from 'amplify-cli-core'; // @public (undocumented) export const cfnRootStackFileName = "root-cloudformation-stack.json"; +// @public (undocumented) +export const deleteEnvironmentParametersFromService: (context: $TSContext, envName: string) => Promise; + // Warning: (ae-forgotten-export) The symbol "LocationService" needs to be exported by the entry point index.d.ts // // @public (undocumented) @@ -24,6 +27,12 @@ export function getConfiguredLocationServiceClient(context: $TSContext, options? // @public (undocumented) export function getConfiguredSSMClient(context: any): Promise; +// @public (undocumented) +export const getEnvParametersDownloadHandler: (context: $TSContext) => Promise<(keys: string[]) => Promise>>; + +// @public (undocumented) +export const getEnvParametersUploadHandler: (context: $TSContext) => Promise<(key: string, value: string | boolean | number) => Promise>; + // @public (undocumented) export const getLocationRegionMapping: () => $TSObject; diff --git a/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/delete-ssm-parameters.test.ts b/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/delete-ssm-parameters.test.ts new file mode 100644 index 00000000000..7df8ac1fef1 --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/delete-ssm-parameters.test.ts @@ -0,0 +1,61 @@ +import { $TSContext } from 'amplify-cli-core'; +import type { SSM as SSMType } from 'aws-sdk'; +import { SSM } from '../../../aws-utils/aws-ssm'; +import { deleteEnvironmentParametersFromService } from '../../../utils/ssm-utils/delete-ssm-parameters'; +import { + getSsmSdkParametersDeleteParameters, + getSsmSdkParametersGetParametersByPath, +} from '../../../utils/ssm-utils/get-ssm-sdk-parameters'; + +jest.mock('../../../aws-utils/aws-ssm'); + +const fakeAppId = 'fakeAppId'; +const keyPrefix = '/amplify/id/dev/'; +let keys: Array = ['AMPLIFY_one', 'AMPLIFY_two', 'toBeIgnored']; +let expectedKeys: Array = ['AMPLIFY_one', 'AMPLIFY_two']; +keys = keys.map(key => keyPrefix + key); +expectedKeys = expectedKeys.map(key => keyPrefix + key); +const envName: string = 'dev'; +const contextStub = { + exeInfo: { + inputParams: { + amplify: { + appId: fakeAppId, + }, + }, + }, +}; + +describe('parameters-delete-handler', () => { + it('check if returned function is called once with correct paramater', async () => { + const deleteParametersPromiseMock = jest.fn().mockImplementation(() => Promise.resolve()); + const deleteParametersMock = jest.fn().mockImplementation(() => ({ promise: deleteParametersPromiseMock })); + + const mockGetParamatersReturnedObject: SSMType.GetParametersByPathResult = { + Parameters: keys.map(key => { + return { Name: key }; + }), + }; + const getParametersByPathPromiseMock = jest.fn().mockImplementation(() => Promise.resolve(mockGetParamatersReturnedObject)); + const getParametersByPathMock = jest.fn().mockImplementation(() => ({ promise: getParametersByPathPromiseMock })); + + const mockSSM = SSM as jest.Mocked; + mockSSM.getInstance = jest.fn().mockResolvedValue({ + client: { + deleteParameters: deleteParametersMock, + getParametersByPath: getParametersByPathMock, + }, + }); + + await deleteEnvironmentParametersFromService((contextStub as unknown) as $TSContext, envName); + expect(deleteParametersPromiseMock).toBeCalledTimes(1); + expect(deleteParametersMock).toBeCalledTimes(1); + const expectedDeleteParamater = getSsmSdkParametersDeleteParameters(expectedKeys); + expect(deleteParametersMock).toBeCalledWith(expectedDeleteParamater); + + expect(getParametersByPathPromiseMock).toBeCalledTimes(1); + expect(getParametersByPathMock).toBeCalledTimes(1); + const expectedGetParamater = getSsmSdkParametersGetParametersByPath(fakeAppId, envName); + expect(getParametersByPathMock).toBeCalledWith(expectedGetParamater); + }); +}); diff --git a/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/env-parameter-ssm-helpers.test.ts b/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/env-parameter-ssm-helpers.test.ts new file mode 100644 index 00000000000..fc149c5957a --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/__tests__/utils/ssm-utils/env-parameter-ssm-helpers.test.ts @@ -0,0 +1,122 @@ +import { $TSContext, stateManager } from 'amplify-cli-core'; +import { SSM } from '../../../aws-utils/aws-ssm'; +import type { SSM as SSMType } from 'aws-sdk'; +import { getEnvParametersDownloadHandler, getEnvParametersUploadHandler } from '../../../utils/ssm-utils/env-parameter-ssm-helpers'; + +jest.mock('amplify-cli-core'); +jest.mock('../../../aws-utils/aws-ssm'); + +const stateManagerMock = stateManager as jest.Mocked; +const mockSSM = SSM as jest.Mocked; + +stateManagerMock.metaFileExists = jest.fn().mockReturnValue(true); +stateManagerMock.getMeta = jest.fn(); +stateManagerMock.getCurrentEnvName = jest.fn().mockReturnValue('mockEnv'); + +const putParameterPromiseMock = jest.fn().mockImplementation(() => Promise.resolve()); +const getParametersPromiseMock = jest.fn().mockImplementation(() => Promise.resolve()); + +const getParametersMock = jest.fn().mockImplementation(() => ({ + promise: getParametersPromiseMock, +})); + +mockSSM.getInstance = jest.fn().mockResolvedValue({ + client: { + putParameter: jest.fn().mockImplementation(() => ({ + promise: putParameterPromiseMock, + })), + getParameters: getParametersMock, + }, +}); + +const contextMock = {} as unknown as $TSContext; + +jest.useFakeTimers(); + +describe('uploading environment parameters', () => { + it('returns an async function which can invoke the SSM client', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': { 'AmplifyAppId': 'mockedAppId' } } }); + const returnedFn = await getEnvParametersUploadHandler(contextMock); + expect(returnedFn).toBeDefined(); + await returnedFn('key', 'value'); + expect(putParameterPromiseMock).toBeCalledTimes(1); + }); + + it('returns no-op when AmplifyAppId is undefined', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': {} } }); + const returnedFn = await getEnvParametersUploadHandler(contextMock); + expect(returnedFn).toBeDefined(); + await returnedFn('key1', 'value'); + await returnedFn('key2', 'value'); + expect(getParametersPromiseMock).not.toBeCalled(); + }); +}); + +describe('downloading environment parameters', () => { + afterEach(() => { + jest.clearAllMocks(); + jest.clearAllTimers(); + }); + + it('returns no-op when AmplifyAppId is undefined', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': {} } }); + const returnedFn = await getEnvParametersDownloadHandler(contextMock); + expect(returnedFn).toBeDefined(); + const mockParams = await returnedFn(['mockMissingParam']); + expect(mockParams).toStrictEqual({}); + expect(getParametersMock).not.toBeCalled(); + }); + + it('returns {} when no keys are supplied', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': { 'AmplifyAppId': 'mockedAppId' } } }); + const returnedFn = await getEnvParametersDownloadHandler(contextMock); + expect(returnedFn).toBeDefined(); + const mockParams = await returnedFn([]); + expect(mockParams).toStrictEqual({}); + expect(getParametersMock).not.toBeCalled(); + }); + + it('returns an async function which can invoke the SSM client', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': { 'AmplifyAppId': 'mockedAppId' } } }); + const singleResultMock: SSMType.GetParametersResult = { Parameters: [{ Name: '/amplify/mockAppId/mockEnv/key', Value: '"value"' }] }; + getParametersPromiseMock.mockResolvedValueOnce(singleResultMock); + + const returnedFn = await getEnvParametersDownloadHandler(contextMock); + expect(returnedFn).toBeDefined(); + const mockParams = await returnedFn(['key']); + expect(mockParams).toStrictEqual({ 'key': 'value' }); + expect(getParametersMock).toBeCalledTimes(1); + expect(getParametersPromiseMock).toBeCalledTimes(1); + }); + + it('returns function which can handle many parameters in a single request', async () => { + stateManagerMock.getMeta.mockReturnValueOnce({ 'providers': { 'awscloudformation': { 'AmplifyAppId': 'mockedAppId' } } }); + const keys: string[] = []; + const mockCloudParams: { Name: string; Value: string }[] = []; + const expectedParams = {}; + for (let i = 0; i < 12; ++i) { + const key = `key${i}`; + keys.push(key); + expectedParams[key] = 'value'; + mockCloudParams.push({ Name: `/amplify/mockedAppId/mockEnv/${key}`, Value: '"value"' }); + } + const expectedKeyPaths = keys.map(key => `/amplify/mockedAppId/mockEnv/${key}`); + getParametersPromiseMock.mockResolvedValueOnce({ Parameters: mockCloudParams.slice(0, 10) }); + getParametersPromiseMock.mockResolvedValueOnce({ Parameters: mockCloudParams.slice(10) }); + + const returnedFn = await getEnvParametersDownloadHandler(contextMock); + expect(returnedFn).toBeDefined(); + const mockParams = await returnedFn(keys); + expect(getParametersMock).toBeCalledTimes(2); + expect(getParametersMock).toBeCalledWith({ + Names: expectedKeyPaths.slice(0, 10), + WithDecryption: false, + }); + expect(getParametersMock).toBeCalledWith({ + Names: expectedKeyPaths.slice(10), + WithDecryption: false, + }); + expect(mockParams).toStrictEqual(expectedParams); + expect(getParametersPromiseMock).toBeCalledTimes(2); + }); +}); diff --git a/packages/amplify-provider-awscloudformation/src/index.ts b/packages/amplify-provider-awscloudformation/src/index.ts index 3ae8543125c..960559d872c 100644 --- a/packages/amplify-provider-awscloudformation/src/index.ts +++ b/packages/amplify-provider-awscloudformation/src/index.ts @@ -50,6 +50,11 @@ import { LocationService } from './aws-utils/aws-location-service'; import { hashDirectory } from './upload-appsync-files'; import { prePushCfnTemplateModifier } from './pre-push-cfn-processor/pre-push-cfn-modifier'; import { getApiKeyConfig } from './utils/api-key-helpers'; +import { deleteEnvironmentParametersFromService } from './utils/ssm-utils/delete-ssm-parameters'; +export { deleteEnvironmentParametersFromService } from './utils/ssm-utils/delete-ssm-parameters'; + +import { getEnvParametersUploadHandler, getEnvParametersDownloadHandler } from './utils/ssm-utils/env-parameter-ssm-helpers'; +export { getEnvParametersUploadHandler, getEnvParametersDownloadHandler } from './utils/ssm-utils/env-parameter-ssm-helpers'; function init(context) { return initializer.run(context); @@ -198,4 +203,7 @@ module.exports = { hashDirectory, prePushCfnTemplateModifier, getApiKeyConfig, + getEnvParametersDownloadHandler, + getEnvParametersUploadHandler, + deleteEnvironmentParametersFromService, }; diff --git a/packages/amplify-provider-awscloudformation/src/initialize-env.ts b/packages/amplify-provider-awscloudformation/src/initialize-env.ts index 7814299cde4..122fa96f816 100644 --- a/packages/amplify-provider-awscloudformation/src/initialize-env.ts +++ b/packages/amplify-provider-awscloudformation/src/initialize-env.ts @@ -12,9 +12,7 @@ /* eslint-disable no-restricted-syntax */ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable prefer-arrow/prefer-arrow-functions */ -import { - $TSContext, $TSMeta, JSONUtilities, PathConstants, stateManager, -} from 'amplify-cli-core'; +import { $TSContext, $TSMeta, JSONUtilities, PathConstants, stateManager } from 'amplify-cli-core'; import fs from 'fs-extra'; import glob from 'glob'; import _ from 'lodash'; diff --git a/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/delete-ssm-parameters.ts b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/delete-ssm-parameters.ts new file mode 100644 index 00000000000..bccf9878760 --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/delete-ssm-parameters.ts @@ -0,0 +1,83 @@ +import { $TSContext, AmplifyFault } from 'amplify-cli-core'; +import { printer } from 'amplify-prompts'; +import type { SSM as SSMType } from 'aws-sdk'; +import { SSM } from '../../aws-utils/aws-ssm'; +import { resolveAppId } from '../resolve-appId'; +import { executeSdkPromisesWithExponentialBackOff } from './exp-backoff-executor'; +import { getSsmSdkParametersDeleteParameters, getSsmSdkParametersGetParametersByPath } from './get-ssm-sdk-parameters'; + +/** + * Delete CloudFormation parameters from the service + */ +export const deleteEnvironmentParametersFromService = async (context: $TSContext, envName: string): Promise => { + let appId; + try { + appId = resolveAppId(context); + } catch { + printer.debug(`No AppId found when deleting parameters for environment ${envName}`); + return; + } + const { client } = await SSM.getInstance(context); + await deleteParametersFromParameterStore(appId, envName, client); +}; + +const deleteParametersFromParameterStore = async (appId: string, envName: string, ssmClient: SSMType): Promise => { + try { + const envKeysInParameterStore: Array = await getAllEnvParametersFromParameterStore(appId, envName, ssmClient); + if (!envKeysInParameterStore.length) { + return; + } + const chunkedKeys: Array> = chunkForParameterStore(envKeysInParameterStore); + const deleteKeysFromPSPromises = chunkedKeys.map(keys => { + const ssmArgument = getSsmSdkParametersDeleteParameters(keys); + return () => ssmClient.deleteParameters(ssmArgument).promise(); + }); + + await executeSdkPromisesWithExponentialBackOff(deleteKeysFromPSPromises); + + } catch (e) { + throw new AmplifyFault( + 'ParametersDeleteFault', + { + message: `Failed to delete parameters from the service`, + }, + e, + ); + } +}; + +function isAmplifyParameter(parameter: string) { + const keyPrefix = 'AMPLIFY_'; + const splitParam = parameter.split('/'); + const lastPartOfPath = splitParam.slice(-1).pop(); + return lastPartOfPath.startsWith(keyPrefix); +} + +const getAllEnvParametersFromParameterStore = async (appId: string, envName: string, ssmClient: SSMType): Promise> => { + const parametersUnderPath: Array = []; + let receivedNextToken = ''; + do { + const ssmArgument = getSsmSdkParametersGetParametersByPath(appId, envName, receivedNextToken); + const [data] = await executeSdkPromisesWithExponentialBackOff([ + () => ssmClient.getParametersByPath(ssmArgument).promise(), + ]); + parametersUnderPath.push(...data.Parameters.map(returnedParameter => returnedParameter.Name).filter(name => isAmplifyParameter(name))); + receivedNextToken = data.NextToken; + } while (receivedNextToken); + return parametersUnderPath; +}; + +const chunkForParameterStore = (keys: Array): Array> => { + const maxLength = 10; + const chunkedKeys: Array> = []; + let lastChunk: Array = []; + chunkedKeys.push(lastChunk); + keys.forEach(key => { + if (lastChunk.length === maxLength) { + lastChunk = []; + chunkedKeys.push(lastChunk); + } + lastChunk.push(key); + }); + return chunkedKeys; +}; diff --git a/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/env-parameter-ssm-helpers.ts b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/env-parameter-ssm-helpers.ts new file mode 100644 index 00000000000..1331e41178e --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/env-parameter-ssm-helpers.ts @@ -0,0 +1,119 @@ +import { $TSContext, AmplifyFault, stateManager } from 'amplify-cli-core'; +import { printer } from 'amplify-prompts'; +import type { SSM as SSMType } from 'aws-sdk'; +import { SSM } from '../../aws-utils/aws-ssm'; +import { resolveAppId } from '../resolve-appId'; +import { executeSdkPromisesWithExponentialBackOff } from './exp-backoff-executor'; + +/** + * Higher order function for uploading CloudFormation parameters to the service + */ +export const getEnvParametersUploadHandler = async ( + context: $TSContext, +): Promise<(key: string, value: string | boolean | number) => Promise> => { + let appId: string; + try { + appId = resolveAppId(context); + } catch { + printer.warn('Failed to resolve AppId, skipping parameter download.'); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return (__: string, ___: string | boolean | number) => new Promise(resolve => { + resolve(); + }); + } + const envName = stateManager.getCurrentEnvName(); + const { client } = await SSM.getInstance(context); + return uploadParameterToParameterStore(appId, envName, client); +}; + +const uploadParameterToParameterStore = ( + appId: string, + envName: string, + ssmClient: SSMType, +): ((key: string, value: string | boolean | number) => Promise) => { + return async (key: string, value: string | boolean | number): Promise => { + try { + const stringValue: string = JSON.stringify(value); + const sdkParameters = { + Name: `/amplify/${appId}/${envName}/${key}`, + Overwrite: true, + Tier: 'Standard', + Type: 'String', + Value: stringValue, + }; + await executeSdkPromisesWithExponentialBackOff([() => ssmClient.putParameter(sdkParameters).promise()]); + } catch (e) { + throw new AmplifyFault( + 'ParameterUploadFault', + { + message: `Failed to upload ${key} to ParameterStore`, + }, + e, + ); + } + }; +}; + +/** + * Higher order function for downloading CloudFormation parameters from the service + */ +export const getEnvParametersDownloadHandler = async ( + context: $TSContext, +): Promise<((keys: string[]) => Promise>)> => { + let appId: string; + try { + appId = resolveAppId(context); + } catch { + printer.warn('Failed to resolve AppId, skipping parameter download.'); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + return (__: string[]) => new Promise(resolve => { + resolve({}); + }); + } + const envName = stateManager.getCurrentEnvName(); + const { client } = await SSM.getInstance(context); + return downloadParametersFromParameterStore(appId, envName, client); +}; + +const downloadParametersFromParameterStore = ( + appId: string, + envName: string, + ssmClient: SSMType, +): ((keys: string[]) => Promise>) => { + return async (keys: string[]): Promise> => { + if (keys.length === 0) { + return {}; + } + try { + const keyPaths = keys.map(key => `/amplify/${appId}/${envName}/${key}`); + const sdkPromises = convertKeyPathsToSdkPromises(ssmClient, keyPaths); + const results = await executeSdkPromisesWithExponentialBackOff(sdkPromises); + return results.reduce((acc, { Parameters }) => { + Parameters.forEach((param) => { + const [/* leading slash */, /* amplify */, /* appId */, /* envName */, key] = param.Name.split('/'); + acc[key] = JSON.parse(param.Value); + }); + return acc; + }, {} as Record); + } catch (e) { + throw new AmplifyFault( + 'ParameterDownloadFault', + { + message: `Failed to download the following parameters from ParameterStore:\n ${keys.join('\n ')}`, + }, + e, + ); + } + }; +}; + +const convertKeyPathsToSdkPromises = (ssmClient: SSMType, keyPaths: string[]): (() => Promise)[] => { + const sdkParameterChunks = []; + for (let i = 0; i < keyPaths.length; i += 10) { + sdkParameterChunks.push({ + Names: keyPaths.slice(i, Math.min(i + 10, keyPaths.length)), + WithDecryption: false, + }); + } + return sdkParameterChunks.map((sdkParameters) => () => ssmClient.getParameters(sdkParameters).promise()); +}; diff --git a/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/exp-backoff-executor.ts b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/exp-backoff-executor.ts new file mode 100644 index 00000000000..f672e67fc87 --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/exp-backoff-executor.ts @@ -0,0 +1,33 @@ +export const executeSdkPromisesWithExponentialBackOff = async ( + sdkPromises: (() => Promise)[], +): Promise => { + const MAX_RETRIES = 5; + const MAX_BACK_OFF_IN_MS = 10 * 1000; // 10 seconds + let backOffSleepTimeInMs = 200; + let consecutiveRetries = 0; + + let i = 0; + const promiseResults = []; + while (i < sdkPromises.length) { + try { + promiseResults.push(await sdkPromises[i]()); + ++i; + // In case previously throttled, reset backoff + backOffSleepTimeInMs = 200; + consecutiveRetries = 0; + } catch (e) { + if (e?.code === 'ThrottlingException' || e?.code === 'Throttling') { + if (consecutiveRetries < MAX_RETRIES) { + ++consecutiveRetries; + await new Promise(resolve => setTimeout(resolve, backOffSleepTimeInMs)); + backOffSleepTimeInMs = 2 ** consecutiveRetries * backOffSleepTimeInMs; + backOffSleepTimeInMs = Math.min(Math.random() * backOffSleepTimeInMs, MAX_BACK_OFF_IN_MS); + continue; + } + } + throw e; + } + } + + return promiseResults; +}; diff --git a/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/get-ssm-sdk-parameters.ts b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/get-ssm-sdk-parameters.ts new file mode 100644 index 00000000000..956b1ed87a0 --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/utils/ssm-utils/get-ssm-sdk-parameters.ts @@ -0,0 +1,29 @@ +export const getSsmSdkParametersDeleteParameters = (keys: Array): SsmDeleteParameters => { + const sdkParameters = { + Names: keys, + }; + return sdkParameters; +}; + +export const getSsmSdkParametersGetParametersByPath = ( + appId: string, + envName: string, + nextToken?: string, +): SsmGetParametersByPathArgument => { + const sdkParameters: SsmGetParametersByPathArgument = { + Path: `/amplify/${appId}/${envName}/`, + }; + if (nextToken) { + sdkParameters.NextToken = nextToken; + } + return sdkParameters; +}; + +type SsmDeleteParameters = { + Names: Array; +}; + +type SsmGetParametersByPathArgument = { + Path: string; + NextToken?: string; +};