fix(api): add null placeholder for nested stack during api rebuild (#…

…11460)

* fix(api): add null placeholder for nested stack during api rebuild

packages/amplify-e2e-core/src/init/deleteProject.ts

@@ -1,12 +1,18 @@
 /* eslint-disable import/no-cycle */
 import { nspawn as spawn, retry, getCLIPath, describeCloudFormationStack } from '..';
 import { getBackendAmplifyMeta } from '../utils';
+import { $TSAny } from 'amplify-cli-core';
 
 /**
  * Runs `amplify delete`
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const deleteProject = async (cwd: string, profileConfig?: any, usingLatestCodebase = false): Promise<void> => {
+export const deleteProject = async (
+  cwd: string,
+  profileConfig?: $TSAny,
+  usingLatestCodebase = false,
+  noOutputTimeout: number = 1000 * 60 * 20,
+): Promise<void> => {
   // Read the meta from backend otherwise it could fail on non-pushed, just initialized projects
   try {
     const { StackName: stackName, Region: region } = getBackendAmplifyMeta(cwd).providers.awscloudformation;
@@ -15,7 +21,6 @@ export const deleteProject = async (cwd: string, profileConfig?: any, usingLates
       (stack) => stack.StackStatus.endsWith('_COMPLETE') || stack.StackStatus.endsWith('_FAILED'),
     );
 
-    const noOutputTimeout = 1000 * 60 * 20; // 20 minutes;
     await spawn(getCLIPath(usingLatestCodebase), ['delete'], { cwd, stripColors: true, noOutputTimeout })
       .wait('Are you sure you want to continue?')
       .sendYes()

packages/amplify-e2e-tests/schemas/relational_models_v2.graphql

@@ -0,0 +1,18 @@
+input AMPLIFY {
+  globalAuthRule: AuthRule = { allow: public }
+}
+type Todo @model {
+  id: ID!
+  name: String
+  description: String
+  tasks: [Task] @hasMany
+  assignee: Worker @hasOne
+}
+type Task @model {
+  id: ID!
+  todo: Todo @belongsTo
+}
+type Worker @model {
+  id: ID!
+  todo: Todo @belongsTo
+}

packages/amplify-e2e-tests/schemas/searchable_model_v2.graphql

@@ -0,0 +1,8 @@
+input AMPLIFY {
+  globalAuthRule: AuthRule = { allow: public }
+}
+type Todo @model @searchable {
+  id: ID!
+  name: String
+  description: String
+}

packages/amplify-e2e-tests/src/__tests__/api_6a.test.ts

@@ -5,40 +5,49 @@ import {
   amplifyPush,
   deleteProject,
   deleteProjectDir,
-  putItemInTable,
-  scanTable,
   rebuildApi,
   getProjectMeta,
+  updateApiSchema,
 } from '@aws-amplify/amplify-e2e-core';
+import { testTableAfterRebuildApi, testTableBeforeRebuildApi } from '../rebuild-test-helpers';
 
 const projName = 'apitest';
+
 let projRoot;
 beforeEach(async () => {
   projRoot = await createNewProjectDir(projName);
   await initJSProjectWithProfile(projRoot, { name: projName });
   await addApiWithoutSchema(projRoot, { transformerVersion: 2 });
-  await amplifyPush(projRoot);
 });
 afterEach(async () => {
   await deleteProject(projRoot);
   deleteProjectDir(projRoot);
 });
 
 describe('amplify rebuild api', () => {
-  it('recreates all model tables', async () => {
+  it('recreates single table', async () => {
+    await amplifyPush(projRoot);
     const projMeta = getProjectMeta(projRoot);
     const apiId = projMeta?.api?.[projName]?.output?.GraphQLAPIIdOutput;
     const region = projMeta?.providers?.awscloudformation?.Region;
     expect(apiId).toBeDefined();
     expect(region).toBeDefined();
-    const tableName = `Todo-${apiId}-integtest`;
-    await putItemInTable(tableName, region, { id: 'this is a test value' });
-    const scanResultBefore = await scanTable(tableName, region);
-    expect(scanResultBefore.Items.length).toBe(1);
-
+    await testTableBeforeRebuildApi(apiId, region, 'Todo');
     await rebuildApi(projRoot, projName);
+    await testTableAfterRebuildApi(apiId, region, 'Todo');
+  });
+  it('recreates tables for relational models', async () => {
+    await updateApiSchema(projRoot, projName, 'relational_models_v2.graphql');
+    await amplifyPush(projRoot);
+    const projMeta = getProjectMeta(projRoot);
+    const apiId = projMeta?.api?.[projName]?.output?.GraphQLAPIIdOutput;
+    const region = projMeta?.providers?.awscloudformation?.Region;
+    expect(apiId).toBeDefined();
+    expect(region).toBeDefined();
 
-    const scanResultAfter = await scanTable(tableName, region);
-    expect(scanResultAfter.Items.length).toBe(0);
+    const modelNames = ['Todo', 'Task', 'Worker'];
+    modelNames.forEach(async (modelName) => await testTableBeforeRebuildApi(apiId, region, modelName));
+    await rebuildApi(projRoot, projName);
+    modelNames.forEach(async (modelName) => await testTableAfterRebuildApi(apiId, region, modelName));
   });
 });

packages/amplify-e2e-tests/src/__tests__/api_6c.test.ts

@@ -0,0 +1,40 @@
+import {
+  createNewProjectDir,
+  initJSProjectWithProfile,
+  addApiWithoutSchema,
+  amplifyPush,
+  deleteProjectDir,
+  rebuildApi,
+  getProjectMeta,
+  updateApiSchema,
+  deleteProject,
+} from '@aws-amplify/amplify-e2e-core';
+import { testTableAfterRebuildApi, testTableBeforeRebuildApi } from '../rebuild-test-helpers';
+
+const projName = 'apitest';
+
+let projRoot;
+beforeEach(async () => {
+  projRoot = await createNewProjectDir(projName);
+});
+afterEach(async () => {
+  await deleteProject(projRoot, undefined, false, 1000 * 60 * 30);
+  deleteProjectDir(projRoot);
+});
+
+describe('amplify rebuild api', () => {
+  it('recreates tables for searchable models', async () => {
+    await initJSProjectWithProfile(projRoot, { name: projName });
+    await addApiWithoutSchema(projRoot, { transformerVersion: 2 });
+    await updateApiSchema(projRoot, projName, 'searchable_model_v2.graphql');
+    await amplifyPush(projRoot);
+    const projMeta = getProjectMeta(projRoot);
+    const apiId = projMeta?.api?.[projName]?.output?.GraphQLAPIIdOutput;
+    const region = projMeta?.providers?.awscloudformation?.Region;
+    expect(apiId).toBeDefined();
+    expect(region).toBeDefined();
+    await testTableBeforeRebuildApi(apiId, region, 'Todo');
+    await rebuildApi(projRoot, projName);
+    await testTableAfterRebuildApi(apiId, region, 'Todo');
+  });
+});

packages/amplify-e2e-tests/src/rebuild-test-helpers/index.ts

@@ -0,0 +1,14 @@
+import { putItemInTable, scanTable } from '@aws-amplify/amplify-e2e-core';
+
+export const testTableBeforeRebuildApi = async (apiId: string, region: string, modelName: string) => {
+  const tableName = `${modelName}-${apiId}-integtest`;
+  await putItemInTable(tableName, region, { id: 'this is a test value' });
+  const scanResultBefore = await scanTable(tableName, region);
+  expect(scanResultBefore.Items.length).toBe(1);
+};
+
+export const testTableAfterRebuildApi = async (apiId: string, region: string, modelName: string) => {
+  const tableName = `${modelName}-${apiId}-integtest`;
+  const scanResultAfter = await scanTable(tableName, region);
+  expect(scanResultAfter.Items.length).toBe(0);
+};

packages/amplify-provider-awscloudformation/src/graphql-resource-manager/amplify-graphql-resource-manager.ts

@@ -22,6 +22,9 @@ import { addGSI, getGSIDetails, removeGSI } from './dynamodb-gsi-helpers';
 import { loadConfiguration } from '../configuration-manager';
 
 const ROOT_LEVEL = 'root';
+const RESERVED_ROOT_STACK_TEMPLATE_STATE_KEY_NAME = '_root';
+const CONNECTION_STACK_NAME = 'ConnectionStack';
+const SEARCHABLE_STACK_NAME = 'SearchableStack';
 
 /**
  * Type for GQLResourceManagerProps
@@ -166,15 +169,25 @@ export class GraphQLResourceManager {
       fs.copySync(previousStepPath, stepPath);
       previousStepPath = stepPath;
 
-      const tables = this.templateState.getKeys();
+      const nestedStacks = this.templateState.getKeys().filter((k) => k !== RESERVED_ROOT_STACK_TEMPLATE_STATE_KEY_NAME);
       const tableNames = [];
-      tables.forEach((tableName) => {
-        tableNames.push(tableNameMap.get(tableName));
-        const tableNameStackFilePath = path.join(stepPath, 'stacks', `${tableName}.json`);
-        fs.ensureDirSync(path.dirname(tableNameStackFilePath));
-        JSONUtilities.writeJson(tableNameStackFilePath, this.templateState.pop(tableName));
+      nestedStacks.forEach((stackName) => {
+        if (stackName !== CONNECTION_STACK_NAME && stackName !== SEARCHABLE_STACK_NAME) {
+          // Connection stack is not provisioning dynamoDB table and need to be filtered
+          tableNames.push(tableNameMap.get(stackName));
+        }
+        const nestedStackFilePath = path.join(stepPath, 'stacks', `${stackName}.json`);
+        fs.ensureDirSync(path.dirname(nestedStackFilePath));
+        JSONUtilities.writeJson(nestedStackFilePath, this.templateState.pop(stackName));
       });
 
+      // Update the root stack template when it is changed in template state
+      if (this.templateState.has(RESERVED_ROOT_STACK_TEMPLATE_STATE_KEY_NAME)) {
+        const rootStackFilePath = path.join(stepPath, 'cloudformation-template.json');
+        fs.ensureDirSync(path.dirname(rootStackFilePath));
+        JSONUtilities.writeJson(rootStackFilePath, this.templateState.pop(RESERVED_ROOT_STACK_TEMPLATE_STATE_KEY_NAME));
+      }
+
       const deploymentRootKey = `${ROOT_APPSYNC_S3_KEY}/${buildHash}/states/${stepNumber}`;
       const deploymentStep: DeploymentOp = {
         stackTemplatePathOrUrl: `${deploymentRootKey}/cloudformation-template.json`,
@@ -303,16 +316,68 @@ export class GraphQLResourceManager {
   };
 
   private tableRecreationManagement = (currentState: DiffableProject) => {
-    this.getTablesBeingReplaced().forEach((tableMeta) => {
+    const recreatedTables = this.getTablesBeingReplaced();
+    recreatedTables.forEach((tableMeta) => {
       const ddbStack = this.getStack(tableMeta.stackName, currentState);
       this.dropTemplateResources(ddbStack);
-
       // clear any other states created by GSI updates as dropping and recreating supersedes those changes
       this.clearTemplateState(tableMeta.stackName);
       this.templateState.add(tableMeta.stackName, JSONUtilities.stringify(ddbStack));
     });
+
+    /**
+     * When rebuild api, the root stack needs to change the reference to nested stack output values to temporary null placeholder value
+     * as there will be no output from nested stacks.
+     */
+    if (this.rebuildAllTables) {
+      const rootStack = this.getStack(ROOT_LEVEL, currentState);
+      const connectionStack = this.getStack(CONNECTION_STACK_NAME, currentState);
+      const searchableStack = this.getStack(SEARCHABLE_STACK_NAME, currentState);
+      const allRecreatedNestedStackNames = recreatedTables.map((tableMeta) => tableMeta.stackName);
+      // Drop resources and outputs for connection stack if existed
+      if (connectionStack) {
+        allRecreatedNestedStackNames.push(CONNECTION_STACK_NAME);
+        this.dropTemplateResources(connectionStack);
+        this.templateState.add(CONNECTION_STACK_NAME, JSONUtilities.stringify(connectionStack));
+      }
+      // Drop resources and outputs for searchable stack if existed
+      if (searchableStack) {
+        allRecreatedNestedStackNames.push(SEARCHABLE_STACK_NAME);
+        this.dropTemplateResourcesForSearchableStack(searchableStack);
+        this.templateState.add(SEARCHABLE_STACK_NAME, JSONUtilities.stringify(searchableStack));
+      }
+      // Update nested stack params in root stack
+      this.replaceRecreatedNestedStackParamsInRootStackTemplate(allRecreatedNestedStackNames, rootStack);
+      this.templateState.add(RESERVED_ROOT_STACK_TEMPLATE_STATE_KEY_NAME, JSONUtilities.stringify(rootStack));
+    }
   };
 
+  /**
+   * Set recreated nested stack parameters to 'TemporaryPlaceholderValue' in root stack template
+   * @param recreatedNestedStackNames names of recreated stacks
+   * @param rootStack root stack template
+   */
+  private replaceRecreatedNestedStackParamsInRootStackTemplate(recreatedNestedStackNames: string[], rootStack: Template) {
+    recreatedNestedStackNames.forEach((stackName) => {
+      const stackParamsMap = rootStack.Resources[stackName].Properties.Parameters;
+      Object.keys(stackParamsMap).forEach((stackParamKey) => {
+        const paramObj = stackParamsMap[stackParamKey];
+        const paramObjKeys = Object.keys(paramObj);
+        if (paramObjKeys.length === 1 && paramObjKeys[0] === 'Fn::GetAtt') {
+          const paramObjValue = paramObj[paramObjKeys[0]];
+          if (
+            Array.isArray(paramObjValue) &&
+            paramObjValue.length === 2 &&
+            recreatedNestedStackNames.includes(paramObjValue[0]) &&
+            paramObjValue[1].startsWith('Outputs.')
+          ) {
+            stackParamsMap[stackParamKey] = 'TemporaryPlaceholderValue';
+          }
+        }
+      });
+    });
+  }
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   getTablesBeingReplaced = (): any => {
     const gqlDiff = getGQLDiff(this.backendApiProjectRoot, this.cloudBackendApiProjectRoot);
@@ -382,6 +447,18 @@ export class GraphQLResourceManager {
     template.Outputs = {};
   };
 
+  /**
+   * Remove all outputs and resources except for search domain for searchable stack
+   * @param template stack CFN tempalte
+   */
+  private dropTemplateResourcesForSearchableStack = (template: Template): void => {
+    const OpenSearchDomainLogicalID = 'OpenSearchDomain';
+    const searchDomain = template.Resources[OpenSearchDomainLogicalID];
+    template.Resources = {};
+    template.Resources[OpenSearchDomainLogicalID] = searchDomain;
+    template.Outputs = {};
+  };
+
   private clearTemplateState = (stackName: string) => {
     while (this.templateState.has(stackName)) {
       this.templateState.pop(stackName);