chore(scripts): fix single-client generation

.gitignore

@@ -32,7 +32,9 @@ dist
 *.iws
 
 codegen/**/build
+codegen/**/build-single
 codegen/sdk-codegen/smithy-build.json
+codegen/sdk-codegen/smithy-build-*.json
 .gradle
 */out/
 */*/out/

codegen/sdk-codegen/build.gradle.kts

@@ -56,13 +56,22 @@ tasks.create<SmithyBuild>("buildSdk") {
     addRuntimeClasspath = true
 }
 
+configure<software.amazon.smithy.gradle.SmithyExtension> {
+    val clientNameProp: String? by project
+    if (!(clientNameProp?.isEmpty() ?: true)) {
+        smithyBuildConfigs = files("smithy-build-" + clientNameProp + ".json")
+        outputDirectory = file("build-single/" + clientNameProp)
+    }
+}
+
 // Generates a smithy-build.json file by creating a new projection for every
 // JSON file found in aws-models/. The generated smithy-build.json file is
 // not committed to git since it's rebuilt each time codegen is performed.
 tasks.register("generate-smithy-build") {
     doLast {
         val projectionsBuilder = Node.objectNodeBuilder()
         val modelsDirProp: String by project
+        val clientNameProp: String? by project
         val models = project.file(modelsDirProp);
 
         fileTree(models).filter { it.isFile }.files.forEach eachFile@{ file ->
@@ -108,7 +117,11 @@ tasks.register("generate-smithy-build") {
             projectionsBuilder.withMember(sdkId + "." + version.toLowerCase(), projectionContents)
         }
 
-        file("smithy-build.json").writeText(Node.prettyPrintJson(Node.objectNodeBuilder()
+        val buildFile = if (!(clientNameProp?.isEmpty() ?: true))
+            "smithy-build-" + clientNameProp + ".json"
+            else "smithy-build.json"
+
+        file(buildFile).writeText(Node.prettyPrintJson(Node.objectNodeBuilder()
                 .withMember("version", "1.0")
                 .withMember("projections", projectionsBuilder.build())
                 .build()))

scripts/generate-clients/code-gen.js

@@ -1,7 +1,6 @@
 // @ts-check
 const { basename, join, relative } = require("path");
-const { copySync, emptyDirSync } = require("fs-extra");
-const { copyFileSync } = require("fs");
+const { copySync, emptyDirSync, rmSync, copyFileSync } = require("fs-extra");
 const { spawnProcess } = require("../utils/spawn-process");
 const {
   CODE_GEN_ROOT,
@@ -14,19 +13,42 @@ const {
 const { getModelFilepaths } = require("./get-model-filepaths");
 
 const generateClient = async (clientName) => {
-  const TEMP_CODE_GEN_INPUT_DIR_SERVICE = join(TEMP_CODE_GEN_INPUT_DIR, clientName);
+  const retryable = async () => {
+    const TEMP_CODE_GEN_INPUT_DIR_SERVICE = join(TEMP_CODE_GEN_INPUT_DIR, clientName);
 
-  const options = [
-    ":sdk-codegen:build",
-    `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR_SERVICE)}`,
-  ];
+    const options = [
+      ":sdk-codegen:build",
+      "--stacktrace",
+      "--no-rebuild", // prevent dependency smithy-aws-typescript-codegen files from being rebuilt and possibly missing during multi-process
+      `-PmodelsDirProp=${relative(CODE_GEN_SDK_ROOT, TEMP_CODE_GEN_INPUT_DIR_SERVICE)}`,
+      `-PclientNameProp=${clientName}`,
+    ];
 
-  emptyDirSync(TEMP_CODE_GEN_INPUT_DIR_SERVICE);
+    emptyDirSync(TEMP_CODE_GEN_INPUT_DIR_SERVICE);
 
-  const filename = `${clientName}.json`;
-  copyFileSync(join(DEFAULT_CODE_GEN_INPUT_DIR, filename), join(TEMP_CODE_GEN_INPUT_DIR_SERVICE, filename));
+    const filename = `${clientName}.json`;
+    copyFileSync(join(DEFAULT_CODE_GEN_INPUT_DIR, filename), join(TEMP_CODE_GEN_INPUT_DIR_SERVICE, filename));
 
-  await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT });
+    await spawnProcess("./gradlew", options, { cwd: CODE_GEN_ROOT });
+  };
+  let attemptsRemaining = 3;
+
+  /**
+   * Retries are used here because multi-process codegen can be flaky.
+   */
+  while (attemptsRemaining-- > 0) {
+    try {
+      await retryable();
+      attemptsRemaining = 0;
+    } catch (e) {
+      if (attemptsRemaining <= 0) {
+        throw new Error(`Unable to complete codegen for ${clientName}: ` + e);
+      }
+      console.warn(`Retrying codegen for ${clientName} with ${attemptsRemaining} attempts remaining.`);
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+  rmSync(join(__dirname, "..", "..", "codegen", "sdk-codegen", `smithy-build-${clientName}.json`));
 };
 
 const generateClients = async (models, batchSize) => {

scripts/generate-clients/copy-to-clients.js

@@ -83,7 +83,7 @@ const mergeManifest = (fromContent = {}, toContent = {}) => {
   return merged;
 };
 
-const copyToClients = async (sourceDir, destinationDir) => {
+const copyToClients = async (sourceDir, destinationDir, solo) => {
   for (const modelName of readdirSync(sourceDir)) {
     if (modelName === "source") continue;
 
@@ -98,6 +98,10 @@ const copyToClients = async (sourceDir, destinationDir) => {
     const packageName = packageManifest.name;
     const clientName = packageName.replace("@aws-sdk/", "");
 
+    if (solo && clientName !== `client-${solo}`) {
+      continue;
+    }
+
     console.log(`copying ${packageName} from ${artifactPath} to ${destinationDir}`);
     const destPath = join(destinationDir, clientName);
 

scripts/generate-clients/single-service.js

@@ -1,12 +1,11 @@
 const yargs = require("yargs");
-const path = require("path");
+const { normalize, join } = require("path");
 const { generateClient } = require("./code-gen");
 const { codeOrdering } = require("./code-ordering");
 const { copyToClients } = require("./copy-to-clients");
-const { CODE_GEN_SDK_OUTPUT_DIR } = require("./code-gen-dir");
 const { spawnProcess } = require("../utils/spawn-process");
 
-const SDK_CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients"));
+const SDK_CLIENTS_DIR = normalize(join(__dirname, "..", "..", "clients"));
 
 const { solo } = yargs(process.argv.slice(2))
   .string("solo")
@@ -15,32 +14,31 @@ const { solo } = yargs(process.argv.slice(2))
 
 (async () => {
   try {
-    // generation and copy
     await generateClient(solo);
-    await copyToClients(CODE_GEN_SDK_OUTPUT_DIR, SDK_CLIENTS_DIR);
+    await copyToClients(
+      normalize(join(__dirname, "..", "..", "codegen", "sdk-codegen", "build-single", solo)),
+      SDK_CLIENTS_DIR,
+      solo
+    );
+    await codeOrdering(join(SDK_CLIENTS_DIR, `client-${solo}`));
 
     // post-generation transforms
-    const clientFolder = path.join(SDK_CLIENTS_DIR, `client-${solo}`);
-    await codeOrdering(clientFolder);
+    const clientFolder = join(SDK_CLIENTS_DIR, `client-${solo}`);
 
     console.log("================ starting eslint ================", "\n", new Date().toString(), solo);
     try {
-      await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "eslint"), [
-        "--fix",
-        "--quiet",
-        `${clientFolder}/src/**/*.{ts,js,json}`,
-      ]);
+      await spawnProcess("npx", ["eslint", "--quiet", "--fix", `${clientFolder}/src/**/*`]);
     } catch (ignored) {}
 
     console.log("================ starting prettier ================", "\n", new Date().toString(), solo);
-    await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "prettier"), [
+    await spawnProcess("npx", [
+      "prettier",
       "--write",
-      `${clientFolder}/src/**/*.{ts,js,md,json}`,
-    ]);
-    await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "prettier"), [
-      "--write",
-      `${clientFolder}/README.md`,
+      "--loglevel",
+      "warn",
+      `${clientFolder}/src/**/*.{md,js,ts,json}`,
     ]);
+    await spawnProcess("npx", ["prettier", "--write", "--loglevel", "warn", `${clientFolder}/README.md`]);
 
     const compress = require("../endpoints-ruleset/compress");
     compress(solo);