chore(scripts): fix single-client generation (#4344)

.gitignore

@@ -20,7 +20,7 @@ jspm_packages
 .DS_Store
 .vscode/launch.json
 
-Makefile
+Makefile.private.mk
 lerna-debug.log
 package-lock.json
 
@@ -32,7 +32,9 @@ dist
 *.iws
 
 codegen/**/build
+codegen/**/build-single
 codegen/sdk-codegen/smithy-build.json
+codegen/sdk-codegen/smithy-build-*.json
 .gradle
 */out/
 */*/out/

CONTRIBUTING.md

@@ -126,7 +126,16 @@ lerna run build --scope [package name] --include-dependencies
 
 You don't need to run this command if the dependency packages are not changed.
 
-### Generating Service Clients
+## Build caching
+
+Build caching is optionally available via Turborepo. See `turbo.json`.
+Codegen build caching is not supported, but you can use it to run the
+basic `build`, `types`, and `docs` processes that have deterministic
+outputs in the `dist-*` folders.
+
+For usage and code, see [`./scripts/remote-cache`][build-cache].
+
+## Generating Service Clients
 
 If you made changes to either [AWS Smithy TypeScript generator](./codegen/smithy-aws-typescript-codegen)
 or [Smithy TypeScript generator][smithy typescript repo], you should include
@@ -145,17 +154,79 @@ the generated code change to your PR. Here's how to generate clients:
    ./gradlew clean build publishToMavenLocal
    ```
 
-1. Generate service clients in JavaScript SDK repo:
+1. Generate all service clients in JavaScript SDK repo:
+
    ```
    yarn generate-clients
    ```
+
    `generate-clients` is a util script to facilitate the code generation. For more
    information, please refer [codegen](./codegen/README.md)
 
+1. Generate a single client:
+   ```
+   # in the client package folder.
+   clients/client-X> yarn generate:client
+   ```
+
+### CLI dispatch helper
+
+There is an optional CLI helper.
+The CLI helper assists in the dispatch of commands to package contexts.
+
+To activate the default alias run:
+
+```
+. ./scripts/cli-dispatcher/set-alias.sh
+```
+
+This enables the command bin/exe
+
+```
+b
+```
+
+#### General Syntax
+
+```
+b (package name query) - (npm script query)
+```
+
+#### Syntax Examples:
+
+Usage examples
+
+```
+b s3 - b
+```
+
+yarn **b**uild in clients/client-**s3**
+
+```
+b mar ent - doc
+```
+
+yarn build:**doc**s in clients/client-**mar**ketplace-**ent**itlement-service
+
+```
+b m sign - t
+```
+
+yarn **t**est in packages/**m**iddleware-**sign**ing
+
+The package name query is used to find the package within clients, lib, or packages, and the npm script query is used to
+find a command to execute from within `package.json` `scripts`.
+
+In both queries, you can use space-separated substrings. They must occur in the matching package or command in linear order. Priority is given to whole-word matches, initial word matches, and shorter strings. If your instructions are ambiguous the first priority match will be executed. Use the dry-run or confirm options to check your command before execution.
+
+Additional options:
+--dry (dry run), --c (confirm before execution), --help
+
 [issues]: https://github.com/aws/aws-sdk-js-v3/issues
 [pr]: https://github.com/aws/aws-sdk-js-v3/pulls
 [license]: http://aws.amazon.com/apache2.0/
 [cla]: http://en.wikipedia.org/wiki/Contributor_License_Agreement
 [aws service models]: https://github.com/aws/aws-sdk-js-v3/tree/main/models
 [conventional commits]: https://www.conventionalcommits.org/
 [smithy typescript repo]: https://github.com/awslabs/smithy-typescript
+[build-cache]: https://github.com/aws/aws-sdk-js-v3/tree/main/scripts/remote-cache

Makefile

@@ -0,0 +1,30 @@
+# This is the public Makefile containing some build commands.
+# You can implement some additional personal commands such as login and sync in Makefile.private.mk (unversioned).
+
+# fetch AWS testing credentials
+login:
+	make -f Makefile.private.mk login
+
+# Sync your development fork with upstream.
+# Recommended contents:
+# gh repo sync {your_github_account_name}/aws-sdk-js-v3 -b main
+# git fetch --all
+sync:
+	make -f Makefile.private.mk sync
+
+# Runs build for all packages using Turborepo
+turbo-build:
+	(cd scripts/remote-cache && yarn)
+	node scripts/remote-cache/ start&
+	sleep 3
+	npx turbo run build --api="http://localhost:3000" --team="aws-sdk-js" --token="xyz"
+	node scripts/remote-cache/ stop
+
+# Runs single-client codegen for all clients using Turborepo
+turbo-generate-clients:
+	(cd scripts/remote-cache && yarn)
+	node scripts/remote-cache/ start&
+	sleep 3
+	npx turbo run generate:client --filter=@aws-sdk/client-* --api="http://localhost:3000" --team="aws-sdk-js" --token="xyz"
+	node scripts/remote-cache/ stop
+

codegen/sdk-codegen/build.gradle.kts

@@ -56,13 +56,22 @@ tasks.create<SmithyBuild>("buildSdk") {
     addRuntimeClasspath = true
 }
 
+configure<software.amazon.smithy.gradle.SmithyExtension> {
+    val clientNameProp: String? by project
+    if (!(clientNameProp?.isEmpty() ?: true)) {
+        smithyBuildConfigs = files("smithy-build-" + clientNameProp + ".json")
+        outputDirectory = file("build-single/" + clientNameProp)
+    }
+}
+
 // Generates a smithy-build.json file by creating a new projection for every
 // JSON file found in aws-models/. The generated smithy-build.json file is
 // not committed to git since it's rebuilt each time codegen is performed.
 tasks.register("generate-smithy-build") {
     doLast {
         val projectionsBuilder = Node.objectNodeBuilder()
         val modelsDirProp: String by project
+        val clientNameProp: String? by project
         val models = project.file(modelsDirProp);
 
         fileTree(models).filter { it.isFile }.files.forEach eachFile@{ file ->
@@ -108,7 +117,11 @@ tasks.register("generate-smithy-build") {
             projectionsBuilder.withMember(sdkId + "." + version.toLowerCase(), projectionContents)
         }
 
-        file("smithy-build.json").writeText(Node.prettyPrintJson(Node.objectNodeBuilder()
+        val buildFile = if (!(clientNameProp?.isEmpty() ?: true))
+            "smithy-build-" + clientNameProp + ".json"
+            else "smithy-build.json"
+
+        file(buildFile).writeText(Node.prettyPrintJson(Node.objectNodeBuilder()
                 .withMember("version", "1.0")
                 .withMember("projections", projectionsBuilder.build())
                 .build()))

scripts/generate-clients/code-gen.js

@@ -1,7 +1,6 @@
 // @ts-check
 const { basename, join, relative } = require("path");
-const { copySync, emptyDirSync } = require("fs-extra");
-const { copyFileSync } = require("fs");
+const { copySync, emptyDirSync, rmSync, copyFileSync } = require("fs-extra");
 const { spawnProcess } = require("../utils/spawn-process");
 const {
   CODE_GEN_ROOT,
@@ -14,19 +13,42 @@ const {
 const { getModelFilepaths } = require("./get-model-filepaths");
 
 const generateClient = async (clientName) => {
-  const TEMP_CODE_GEN_INPUT_DIR_SERVICE = join(TEMP_CODE_GEN_INPUT_DIR, clientName);
+  const retryable = async () => {
+    const TEMP_CODE_GEN_INPUT_DIR_SERVICE = join(TEMP_CODE_GEN_INPUT_DIR, clientName);
 
-  const options = [
-    ":sdk-codegen:build",
-    `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR_SERVICE)}`,
-  ];
+    const options = [
+      ":sdk-codegen:build",
+      "--stacktrace",
+      "--no-rebuild", // prevent dependency smithy-aws-typescript-codegen files from being rebuilt and possibly missing during multi-process
+      `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR_SERVICE)}`,
+      `-PclientNameProp=${clientName}`,
+    ];
 
-  emptyDirSync(TEMP_CODE_GEN_INPUT_DIR_SERVICE);
+    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR_SERVICE);
 
-  const filename = `${clientName}.json`;
-  copyFileSync(join(DEFAULT_CODE_GEN_INPUT_DIR, filename), join(TEMP_CODE_GEN_INPUT_DIR_SERVICE, filename));
+    const filename = `${clientName}.json`;
+    copyFileSync(join(DEFAULT_CODE_GEN_INPUT_DIR, filename), join(TEMP_CODE_GEN_INPUT_DIR_SERVICE, filename));
 
-  await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT });
+    await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT });
+  };
+  let attemptsRemaining = 3;
+
+  /**
+   * Retries are used here because multi-process codegen can be flaky.
+   */
+  while (attemptsRemaining-- > 0) {
+    try {
+      await retryable();
+      attemptsRemaining = 0;
+    } catch (e) {
+      if (attemptsRemaining <= 0) {
+        throw new Error(`Unable to complete codegen for ${clientName}: ` + e);
+      }
+      console.warn(`Retrying codegen for ${clientName} with ${attemptsRemaining} attempts remaining.`);
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+  rmSync(join(__dirname, "..", "..", "codegen", "sdk-codegen", `smithy-build-${clientName}.json`));
 };
 
 const generateClients = async (models, batchSize) => {

scripts/generate-clients/copy-to-clients.js

@@ -83,7 +83,7 @@ const mergeManifest = (fromContent = {}, toContent = {}) => {
   return merged;
 };
 
-const copyToClients = async (sourceDir, destinationDir) => {
+const copyToClients = async (sourceDir, destinationDir, solo) => {
   for (const modelName of readdirSync(sourceDir)) {
     if (modelName === "source") continue;
 
@@ -98,6 +98,10 @@ const copyToClients = async (sourceDir, destinationDir) => {
     const packageName = packageManifest.name;
     const clientName = packageName.replace("@aws-sdk/", "");
 
+    if (solo && clientName !== `client-${solo}`) {
+      continue;
+    }
+
     console.log(`copying ${packageName} from ${artifactPath} to ${destinationDir}`);
     const destPath = join(destinationDir, clientName);
 

scripts/generate-clients/single-service.js

@@ -1,12 +1,11 @@
 const yargs = require("yargs");
-const path = require("path");
+const { normalize, join } = require("path");
 const { generateClient } = require("./code-gen");
 const { codeOrdering } = require("./code-ordering");
 const { copyToClients } = require("./copy-to-clients");
-const { CODE_GEN_SDK_OUTPUT_DIR } = require("./code-gen-dir");
 const { spawnProcess } = require("../utils/spawn-process");
 
-const SDK_CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients"));
+const SDK_CLIENTS_DIR = normalize(join(__dirname, "..", "..", "clients"));
 
 const { solo } = yargs(process.argv.slice(2))
   .string("solo")
@@ -15,32 +14,31 @@ const { solo } = yargs(process.argv.slice(2))
 
 (async () => {
   try {
-    // generation and copy
     await generateClient(solo);
-    await copyToClients(CODE_GEN_SDK_OUTPUT_DIR, SDK_CLIENTS_DIR);
+    await copyToClients(
+      normalize(join(__dirname, "..", "..", "codegen", "sdk-codegen", "build-single", solo)),
+      SDK_CLIENTS_DIR,
+      solo
+    );
+    await codeOrdering(join(SDK_CLIENTS_DIR, `client-${solo}`));
 
     // post-generation transforms
-    const clientFolder = path.join(SDK_CLIENTS_DIR, `client-${solo}`);
-    await codeOrdering(clientFolder);
+    const clientFolder = join(SDK_CLIENTS_DIR, `client-${solo}`);
 
     console.log("================ starting eslint ================", "\n", new Date().toString(), solo);
     try {
-      await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "eslint"), [
-        "--fix",
-        "--quiet",
-        `${clientFolder}/src/**/*.{ts,js,json}`,
-      ]);
+      await spawnProcess("npx", ["eslint", "--quiet", "--fix", `${clientFolder}/src/**/*`]);
     } catch (ignored) {}
 
     console.log("================ starting prettier ================", "\n", new Date().toString(), solo);
-    await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "prettier"), [
+    await spawnProcess("npx", [
+      "prettier",
       "--write",
-      `${clientFolder}/src/**/*.{ts,js,md,json}`,
-    ]);
-    await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "prettier"), [
-      "--write",
-      `${clientFolder}/README.md`,
+      "--loglevel",
+      "warn",
+      `${clientFolder}/src/**/*.{md,js,ts,json}`,
     ]);
+    await spawnProcess("npx", ["prettier", "--write", "--loglevel", "warn", `${clientFolder}/README.md`]);
 
     const compress = require("../endpoints-ruleset/compress");
     compress(solo);

scripts/remote-cache/.gitignore

@@ -0,0 +1 @@
+tmp

scripts/remote-cache/index.js

@@ -41,3 +41,8 @@ module.exports = {
 if (process.argv.includes("start")) {
   module.exports.start();
 }
+
+if (process.argv.includes("stop")) {
+  const { spawnProcess } = require("../utils/spawn-process");
+  spawnProcess("npx", ["kill-port", port.toString()]);
+}