From e49880be9ecf3969dde044a9eb92ee20dd61ed35 Mon Sep 17 00:00:00 2001 From: Al Harris Date: Tue, 21 Feb 2023 16:57:07 -0800 Subject: [PATCH] fix: post-process cfn files for minification, consolidating concern of minification a bit --- .../amplify-e2e-core/src/init/amplifyPush.ts | 12 ++- .../amplify-e2e-core/src/utils/projectMeta.ts | 4 +- .../__tests__/minify-cloudformation.test.ts | 65 ++++++++++++++ .../src/__tests__/utils/minify-json.test.ts | 86 +++++++++++++++++++ .../src/push-resources.ts | 4 + .../src/upload-appsync-files.js | 9 +- .../src/utils/minify-json.ts | 27 ++++++ 7 files changed, 202 insertions(+), 5 deletions(-) create mode 100644 packages/amplify-e2e-tests/src/__tests__/minify-cloudformation.test.ts create mode 100644 packages/amplify-provider-awscloudformation/src/__tests__/utils/minify-json.test.ts create mode 100644 packages/amplify-provider-awscloudformation/src/utils/minify-json.ts diff --git a/packages/amplify-e2e-core/src/init/amplifyPush.ts b/packages/amplify-e2e-core/src/init/amplifyPush.ts index 97f75828220..f773f481fc8 100644 --- a/packages/amplify-e2e-core/src/init/amplifyPush.ts +++ b/packages/amplify-e2e-core/src/init/amplifyPush.ts @@ -29,16 +29,24 @@ export type LayerPushSettings = { usePreviousPermissions?: boolean; }; +export type PushOpts = { + minify?: boolean; +}; + /** * Function to test amplify push with verbose status */ -export const amplifyPush = async (cwd: string, testingWithLatestCodebase = false): Promise => { +export const amplifyPush = async (cwd: string, testingWithLatestCodebase = false, opts?: PushOpts): Promise => { // Test detailed status await spawn(getCLIPath(testingWithLatestCodebase), ['status', '-v'], { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS }) .wait(/.*/) .runAsync(); + const pushArgs = [ + 'push', + ...( opts?.minify ? ['--minify'] : [] ), + ]; // Test amplify push - await spawn(getCLIPath(testingWithLatestCodebase), ['push'], { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS }) + await spawn(getCLIPath(testingWithLatestCodebase), pushArgs, { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS }) .wait('Are you sure you want to continue?') .sendYes() .wait('Do you want to generate code for your newly created GraphQL API') diff --git a/packages/amplify-e2e-core/src/utils/projectMeta.ts b/packages/amplify-e2e-core/src/utils/projectMeta.ts index c174722516b..57a39e473ff 100644 --- a/packages/amplify-e2e-core/src/utils/projectMeta.ts +++ b/packages/amplify-e2e-core/src/utils/projectMeta.ts @@ -24,8 +24,10 @@ export const getAmplifyDirPath = (projectRoot: string): string => path.join(proj // eslint-disable-next-line spellcheck/spell-checker export const getAWSConfigIOSPath = (projectRoot: string): string => path.join(projectRoot, 'awsconfiguration.json'); +export const getProjectMetaPath = (projectRoot: string) => path.join(projectRoot, 'amplify', '#current-cloud-backend', 'amplify-meta.json'); + export const getProjectMeta = (projectRoot: string): $TSAny => { - const metaFilePath: string = path.join(projectRoot, 'amplify', '#current-cloud-backend', 'amplify-meta.json'); + const metaFilePath: string = getProjectMetaPath(projectRoot); return JSON.parse(fs.readFileSync(metaFilePath, 'utf8')); }; diff --git a/packages/amplify-e2e-tests/src/__tests__/minify-cloudformation.test.ts b/packages/amplify-e2e-tests/src/__tests__/minify-cloudformation.test.ts new file mode 100644 index 00000000000..d7de6f09e58 --- /dev/null +++ b/packages/amplify-e2e-tests/src/__tests__/minify-cloudformation.test.ts @@ -0,0 +1,65 @@ +import { + amplifyPush, + deleteProject, + initJSProjectWithProfile, + createNewProjectDir, + deleteProjectDir, + generateRandomShortId, + getProjectMetaPath, + addApiWithBlankSchema, + updateApiSchema, +} from '@aws-amplify/amplify-e2e-core'; +import fs from 'fs'; +import path from 'path'; + +describe('minify behavior', () => { + let projRoot: string; + let projFolderName: string; + + beforeEach(async () => { + projFolderName = `minify${generateRandomShortId()}`; + projRoot = await createNewProjectDir(projFolderName); + }); + + afterEach(async () => { + if (fs.existsSync(getProjectMetaPath(projRoot))) { + await deleteProject(projRoot); + } + deleteProjectDir(projRoot); + }); + + it('reduces file size when minify flag is provided', async () => { + const envName = 'devtest'; + const projName = `minify${generateRandomShortId()}`; + + // Configure and push app without minification + await initJSProjectWithProfile(projRoot, { name: projName, envName }); + await addApiWithBlankSchema(projRoot); + updateApiSchema(projRoot, projName, 'simple_model.graphql', false); + await amplifyPush(projRoot, false, { minify: false }); + + // Read Cfn file sizes for both nested API stacks and top-level stacks + const currentCloudBackendPath = path.join(projRoot, 'amplify', '#current-cloud-backend'); + + const nestedApiStackPath = path.join(currentCloudBackendPath, 'api', projName, 'build', 'stacks', 'Todo.json'); + const rootApiStackPath = path.join(currentCloudBackendPath, 'awscloudformation', 'build', 'api', projName, 'build', 'cloudformation-template.json'); + const rootStackPath = path.join(currentCloudBackendPath, 'awscloudformation', 'build', 'root-cloudformation-stack.json'); + + const unminifiedNestedApiStackDefinition = fs.readFileSync(nestedApiStackPath, 'utf-8'); + const unminifiedRootApiStackDefinition = fs.readFileSync(rootApiStackPath, 'utf-8'); + const unminifiedRootStackDefinition = fs.readFileSync(rootStackPath, 'utf-8'); + + // Tweak schema file and push with minification + updateApiSchema(projRoot, projName, 'simple_model.graphql', true); + await amplifyPush(projRoot, false, { minify: true }); + + // Read Cfn file sizes for both nested API stacks and top-level stacks, verify files are smaller than initial push. + const minifiedNestedApiStackDefinition = fs.readFileSync(nestedApiStackPath, 'utf-8'); + const minifiedRootApiStackDefinition = fs.readFileSync(rootApiStackPath, 'utf-8'); + const minifiedRootStackDefinition = fs.readFileSync(rootStackPath, 'utf-8'); + + expect(minifiedNestedApiStackDefinition.length).toBeLessThan(unminifiedNestedApiStackDefinition.length) + expect(minifiedRootApiStackDefinition.length).toBeLessThan(unminifiedRootApiStackDefinition.length) + expect(minifiedRootStackDefinition.length).toBeLessThan(unminifiedRootStackDefinition.length) + }); +}); diff --git a/packages/amplify-provider-awscloudformation/src/__tests__/utils/minify-json.test.ts b/packages/amplify-provider-awscloudformation/src/__tests__/utils/minify-json.test.ts new file mode 100644 index 00000000000..5a7e237b01d --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/__tests__/utils/minify-json.test.ts @@ -0,0 +1,86 @@ +import { minifyJSONFile, minifyAllJSONInFolderRecursively } from '../../utils/minify-json' +import path from 'path'; +import fs from 'fs-extra'; +import os from 'os'; + +const JSON_STRING = `{ + "city": "Seattle", + "state": "WA" +}`; + +const NON_JSON_STRING = `CITY = Seattle +STATE = WA +`; + +describe('minifyJSONFile', () => { + const writeFileMinifyAndReturnReadContents = (filePath: string, contents: string): string => { + const tmpFilePath = path.join(os.tmpdir(), filePath); + fs.writeFileSync(tmpFilePath, contents); + minifyJSONFile(tmpFilePath); + return fs.readFileSync(tmpFilePath, 'utf-8'); + }; + + it('minifies JSON files', () => { + const minifiedContents = writeFileMinifyAndReturnReadContents('testfile.json', JSON_STRING); + expect(minifiedContents.length).toBeLessThan(JSON_STRING.length); + }); + + + it('does not change a non-json file with json contents', () => { + const minifiedContents = writeFileMinifyAndReturnReadContents('testfile.dat', JSON_STRING); + expect(minifiedContents.length).toEqual(JSON_STRING.length); + }); + + it('does not change a non-json file with non-json contents', () => { + const minifiedContents = writeFileMinifyAndReturnReadContents('testfile.env', NON_JSON_STRING); + expect(minifiedContents.length).toEqual(NON_JSON_STRING.length); + }); +}); + +describe('minifyAllJSONInFolderRecursively', () => { + it('will minify json files, but leave non-json files alone', () => { + // This test will set up a directory structure as follows + // . + // ├── file.json + // ├── config.env + // └── nested_stacks + // ├── nested.env + // ├── nested_1.json + // ├── nested_2.json + // └── sub_nested_stacks + // ├── sub_nested.env + // └── sub_nested.json + // We will then ensure that after a single invocation file.json, nested_1.json, nested_2.json, and sub_nested.json are all minified + // and that config.env is the same length. + const testDirectory = path.join(os.tmpdir(), 'minifyTestDir'); + const nestedStacksDirectory = path.join(testDirectory, 'nested_stacks'); + const subNestedStacksDirectory = path.join(nestedStacksDirectory, 'sub_nested_stacks'); + + // Clean and setup directory structure + fs.removeSync(testDirectory); + fs.mkdirSync(testDirectory); + fs.mkdirSync(nestedStacksDirectory); + fs.mkdirSync(subNestedStacksDirectory); + + // Write files + fs.writeFileSync(path.join(testDirectory, 'file.json'), JSON_STRING); + fs.writeFileSync(path.join(testDirectory, 'config.env'), NON_JSON_STRING); + fs.writeFileSync(path.join(nestedStacksDirectory, 'nested_1.json'), JSON_STRING); + fs.writeFileSync(path.join(nestedStacksDirectory, 'nested_2.json'), JSON_STRING); + fs.writeFileSync(path.join(nestedStacksDirectory, 'nested.env'), NON_JSON_STRING); + fs.writeFileSync(path.join(subNestedStacksDirectory, 'sub_nested.json'), JSON_STRING); + fs.writeFileSync(path.join(subNestedStacksDirectory, 'sub_nested.env'), NON_JSON_STRING); + + // Apply recursive minification + minifyAllJSONInFolderRecursively(testDirectory); + + // Verify all `.json` files are minified, and `.env` files are not. + expect(fs.readFileSync(path.join(testDirectory, 'file.json')).length).toBeLessThan(JSON_STRING.length); + expect(fs.readFileSync(path.join(testDirectory, 'config.env')).length).toEqual(NON_JSON_STRING.length); + expect(fs.readFileSync(path.join(nestedStacksDirectory, 'nested_1.json')).length).toBeLessThan(JSON_STRING.length); + expect(fs.readFileSync(path.join(nestedStacksDirectory, 'nested_2.json')).length).toBeLessThan(JSON_STRING.length); + expect(fs.readFileSync(path.join(nestedStacksDirectory, 'nested.env')).length).toEqual(NON_JSON_STRING.length); + expect(fs.readFileSync(path.join(subNestedStacksDirectory, 'sub_nested.json')).length).toBeLessThan(JSON_STRING.length); + expect(fs.readFileSync(path.join(subNestedStacksDirectory, 'sub_nested.env')).length).toEqual(NON_JSON_STRING.length); + }); +}); diff --git a/packages/amplify-provider-awscloudformation/src/push-resources.ts b/packages/amplify-provider-awscloudformation/src/push-resources.ts index d2a3fd7de0c..62480d6b048 100644 --- a/packages/amplify-provider-awscloudformation/src/push-resources.ts +++ b/packages/amplify-provider-awscloudformation/src/push-resources.ts @@ -77,6 +77,7 @@ import { prePushTemplateDescriptionHandler } from './template-description-utils' import { buildOverridesEnabledResources } from './build-override-enabled-resources'; import { invokePostPushAnalyticsUpdate } from './plugin-client-api-analytics'; +import { minifyJSONFile } from './utils/minify-json'; const logger = fileLogger('push-resources'); @@ -767,6 +768,9 @@ export const uploadTemplateToS3 = async ( amplifyMeta: $TSMeta, ): Promise => { const cfnFile = path.parse(filePath).base; + if (context.input.options?.minify) { + minifyJSONFile(filePath); + } const s3 = await S3.getInstance(context); const s3Params = { diff --git a/packages/amplify-provider-awscloudformation/src/upload-appsync-files.js b/packages/amplify-provider-awscloudformation/src/upload-appsync-files.js index e42aac0d0ef..538961893bf 100644 --- a/packages/amplify-provider-awscloudformation/src/upload-appsync-files.js +++ b/packages/amplify-provider-awscloudformation/src/upload-appsync-files.js @@ -5,6 +5,8 @@ const path = require('path'); const TransformPackage = require('graphql-transformer-core'); const { S3 } = require('./aws-utils/aws-s3'); const { fileLogger } = require('./utils/aws-logger'); +const { minifyAllJSONInFolderRecursively } = require('./utils/minify-json'); + const logger = fileLogger('upload-appsync-files'); const ROOT_APPSYNC_S3_KEY = 'amplify-appsync-files'; @@ -111,7 +113,7 @@ async function uploadAppSyncFiles(context, resourcesToUpdate, allResources, opti if (personalParams.CreateAPIKey !== undefined && personalParams.APIKeyExpirationEpoch !== undefined) { context.print.warning( 'APIKeyExpirationEpoch and CreateAPIKey parameters should not used together because it can cause ' + - 'unwanted behavior. In the future APIKeyExpirationEpoch will be removed, use CreateAPIKey instead.', + 'unwanted behavior. In the future APIKeyExpirationEpoch will be removed, use CreateAPIKey instead.', ); } @@ -125,7 +127,7 @@ async function uploadAppSyncFiles(context, resourcesToUpdate, allResources, opti context.print.warning( "APIKeyExpirationEpoch parameter's -1 value is deprecated to disable " + - 'the API Key creation. In the future CreateAPIKey parameter replaces this behavior.', + 'the API Key creation. In the future CreateAPIKey parameter replaces this behavior.', ); } else { currentParameters.CreateAPIKey = 1; @@ -189,6 +191,9 @@ async function uploadAppSyncFiles(context, resourcesToUpdate, allResources, opti if (!fs.existsSync(resourceBuildDir)) { return; } + if (context.input.options?.minify) { + minifyAllJSONInFolderRecursively(resourceBuildDir); + } const spinner = new ora('Uploading files.'); spinner.start(); await TransformPackage.uploadAPIProject({ diff --git a/packages/amplify-provider-awscloudformation/src/utils/minify-json.ts b/packages/amplify-provider-awscloudformation/src/utils/minify-json.ts new file mode 100644 index 00000000000..c1e85361bab --- /dev/null +++ b/packages/amplify-provider-awscloudformation/src/utils/minify-json.ts @@ -0,0 +1,27 @@ +import * as fs from 'fs-extra'; +import * as path from 'path'; + +/** + * Given a path to a json file, minify that file by reading in as JSON + * and writing back out without any whitespace. + * @param jsonFilePath the path to the JSON file we are going to minify. + * @returns when the file has been minified and written back to disk. + */ +export const minifyJSONFile = (jsonFilePath: string): void => { + if (!jsonFilePath.includes('.json')) return; // Don't convert files not ending in `.json` + const originalJSON = fs.readFileSync(jsonFilePath, 'utf-8'); + const minifiedJSON = JSON.stringify(JSON.parse(originalJSON)); + fs.writeFileSync(jsonFilePath, minifiedJSON); +}; + +/** + * Recursively walk a folder, and minify (print without whitespace) all the json files you discover + * @param rootPath the top of the tree to walk + */ +export const minifyAllJSONInFolderRecursively = (rootPath: string): void => { + fs.readdirSync(rootPath).forEach(childHandle => { + const childPath = path.join(rootPath, childHandle); + if (fs.lstatSync(childPath).isDirectory()) minifyAllJSONInFolderRecursively(childPath); + if (fs.lstatSync(childPath).isFile() && childPath.includes('.json')) minifyJSONFile(childPath); + }); +};