Adapt publish command for multi-variant packages (#431)

* Update octokit * Handle multi-variant packages for publish * Update tests * Fix tests * Remove unused code * Fix upstream version * Increase version for variants * Update flags description * Renamed file to file name for content-hash * Remove comment * Squashed commit of the following: commit cfbdb0f Author: dappnodedev <[email protected]> Date: Tue Jun 18 17:28:29 2024 +0200 0.3.24 commit 2ff654a Author: dappnodedev <[email protected]> Date: Tue Jun 18 17:21:54 2024 +0200 Bugfix: Create Github releases (#434) * Bump octokit version * Fix Github release creation * Update all github requests * Do not upload release assets * Use octokit.rest * Improve Github error handling commit cc60a42 Author: dappnodedev <[email protected]> Date: Fri Jun 14 16:17:01 2024 +0200 0.3.23 commit 93ebd23 Author: dappnodedev <[email protected]> Date: Fri Jun 14 16:05:05 2024 +0200 Fix build for new upstream format (#433) * Fix build for new upstream format * Compare with string status 404 (not number) * Update remote IPFS API for tests * Fix BuildVariantsMap import * Add "NoThrow" to the generate release notes function * Copy prometheus targets to release dir * Handle variant dir not existent * Declare var outside switch * SImplify release body
dappnode · Jul 8, 2024 · c36e00a · c36e00a
1 parent cfbdb0f
commit c36e00a
Show file tree

Hide file tree

Showing 39 changed files with 793 additions and 382 deletions.
diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts
@@ -1,5 +1,3 @@
-import path from "path";
-import chalk from "chalk";
 import Listr from "listr";
 import { buildAndUpload } from "../../tasks/buildAndUpload/index.js";
 import { ListrContextBuild } from "../../types.js";
@@ -9,7 +7,7 @@ import {
   defaultVariantsDirName
 } from "../../params.js";
 import { BuildCommandOptions, VerbosityOptions } from "./types.js";
-import { getValidVariantNames } from "./variants.js";
+import { getVariantOptions } from "./variants.js";
 import { BuildAndUploadOptions } from "../../tasks/buildAndUpload/types.js";
 
 export async function buildHandler({
@@ -31,10 +29,6 @@ export async function buildHandler({
 }: BuildCommandOptions): Promise<ListrContextBuild> {
   const skipUpload = skip_upload || skipSave;
 
-  const multiVariantMode = Boolean(
-    allVariants || (variants && variants?.length > 0)
-  );
-
   const buildOptions: BuildAndUploadOptions = {
     dir,
     contentProvider,
@@ -45,12 +39,13 @@ export async function buildHandler({
     composeFileName,
     requireGitData,
     deleteOldPins,
-    ...(multiVariantMode &&
-      getVariantOptions({
-        variantsStr: variants,
-        rootDir: dir,
-        variantsDirName
-      }))
+    ...getVariantOptions({
+      allVariants: Boolean(allVariants),
+      variantsStr: variants,
+      rootDir: dir,
+      variantsDirName,
+      composeFileName
+    })
   };
 
   const verbosityOptions: VerbosityOptions = {
@@ -61,32 +56,3 @@ export async function buildHandler({
 
   return await buildTasks.run();
 }
-
-function getVariantOptions({
-  variantsStr,
-  rootDir,
-  variantsDirName
-}: {
-  variantsStr: string | undefined;
-  rootDir: string;
-  variantsDirName: string;
-}): { variants: string[]; variantsDirPath: string } {
-  const variantsDirPath = path.join(rootDir, variantsDirName);
-  const variantNames = getValidVariantNames({
-    variantsDirPath,
-    variants: variantsStr
-  });
-
-  if (variantNames.length === 0)
-    throw new Error(
-      `No valid variants specified. They must be included in: ${variantsDirPath}`
-    );
-
-  console.log(
-    `${chalk.dim(
-      `Building package from template for variant(s) ${variantsStr}...`
-    )}`
-  );
-
-  return { variants: variantNames, variantsDirPath };
-}
diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts
@@ -40,7 +40,7 @@ export const build: CommandModule<CliGlobalOptions, BuildCommandOptions> = {
     },
     all_variants: {
       alias: "all-variants",
-      description: `It will use the dappnode_package.json and docker-compose.yml files in the root of the project together with the specific ones defined for each package variant to build all of them`,
+      description: `Build all package variants at once, by merging the dappnode_package.json and docker-compose.yml files in the root of the project with the specific ones defined for each package variant`,
       type: "boolean"
     },
     variants_dir_name: {
@@ -51,7 +51,7 @@ export const build: CommandModule<CliGlobalOptions, BuildCommandOptions> = {
     },
     variants: {
       alias: "variant",
-      description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. If not specified, all variants will be built. Example: "variant1,variant2"`,
+      description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. Example: "variant1,variant2"`,
       type: "string"
     }
   },

diff --git a/src/commands/build/variants.ts b/src/commands/build/variants.ts
@@ -1,5 +1,56 @@
 import chalk from "chalk";
 import { getAllVariantsInPath } from "../../files/variants/getAllPackageVariants.js";
+import path from "path";
+import { BuildVariantsMap } from "../../types.js";
+import { buildVariantMap } from "../../tasks/buildAndUpload/buildVariantMap.js";
+
+export function getVariantOptions({
+  allVariants,
+  variantsStr,
+  rootDir,
+  variantsDirName,
+  composeFileName
+}: {
+  allVariants: boolean;
+  variantsStr?: string;
+  rootDir: string;
+  variantsDirName: string;
+  composeFileName: string;
+}): { variantsMap: BuildVariantsMap; variantsDirPath: string } {
+  const variantsDirPath = path.join(rootDir, variantsDirName);
+
+  const buildVariantMapArgs = { rootDir, variantsDirPath, composeFileName };
+
+  if (!allVariants && !variantsStr)
+    return {
+      variantsMap: buildVariantMap({ ...buildVariantMapArgs, variants: null }),
+      variantsDirPath
+    };
+
+  const validVariantNames = getValidVariantNames({
+    variantsDirPath,
+    variants: variantsStr
+  });
+
+  if (validVariantNames.length === 0)
+    throw new Error(
+      `No valid variants specified. They must be included in: ${variantsDirPath}`
+    );
+
+  console.log(
+    `${chalk.dim(
+      `Building package from template for variant(s) ${variantsStr}...`
+    )}`
+  );
+
+  return {
+    variantsMap: buildVariantMap({
+      ...buildVariantMapArgs,
+      variants: validVariantNames
+    }),
+    variantsDirPath
+  };
+}
 
 /**
  * Main function to retrieve the valid variant names based on the specified variants and available directories.

diff --git a/src/commands/from_github.ts b/src/commands/from_github.ts
@@ -13,11 +13,11 @@ import {
 } from "../utils/githubGetReleases.js";
 import { ipfsAddDirFromUrls } from "../releaseUploader/ipfsNode/addDirFromUrls.js";
 import { verifyIpfsConnection } from "../releaseUploader/ipfsNode/verifyConnection.js";
-import { CliGlobalOptions, contentHashFile } from "../types.js";
+import { CliGlobalOptions } from "../types.js";
 import { Manifest, defaultArch, releaseFiles } from "@dappnode/types";
 import { getLegacyImagePath } from "../utils/getLegacyImagePath.js";
 import { getImageFileName } from "../utils/getImageFileName.js";
-import { releaseFilesDefaultNames } from "../params.js";
+import { contentHashFileName, releaseFilesDefaultNames } from "../params.js";
 
 interface CliCommandOptions extends CliGlobalOptions {
   repoSlug: string;
@@ -111,7 +111,7 @@ export async function fromGithubHandler({
   }
 
   const files = release.assets
-    .filter(asset => asset.name !== contentHashFile)
+    .filter(asset => asset.name !== contentHashFileName)
     .map(asset => ({
       filepath: path.join("release", asset.name),
       url: asset.browser_download_url,
@@ -147,7 +147,7 @@ export async function fromGithubHandler({
 
   // Verify that the resulting release hash matches the one in Github
   const contentHashAsset = release.assets.find(
-    asset => asset.name === contentHashFile
+    asset => asset.name === contentHashFileName
   );
   if (contentHashAsset) {
     const contentHash = await got(contentHashAsset.browser_download_url).text();

diff --git a/src/commands/publish/handler.ts b/src/commands/publish/handler.ts
@@ -1,10 +1,15 @@
 import Listr from "listr";
-import { defaultComposeFileName, defaultDir } from "../../params.js";
+import {
+  defaultComposeFileName,
+  defaultDir,
+  defaultVariantsDirName
+} from "../../params.js";
 import { ListrContextPublish } from "../../types.js";
 import { VerbosityOptions } from "../build/types.js";
 import { PublishCommandOptions } from "./types.js";
 import { publish } from "../../tasks/publish/index.js";
 import { parseReleaseType } from "./parseReleaseType.js";
+import { getVariantOptions } from "../build/variants.js";
 
 /**
  * Common handler for CLI and programatic usage
@@ -25,7 +30,10 @@ export async function publishHandler({
   dir = defaultDir,
   compose_file_name: composeFileName = defaultComposeFileName,
   silent,
-  verbose
+  verbose,
+  variants_dir_name: variantsDirName = defaultVariantsDirName,
+  all_variants: allVariants,
+  variants
 }: PublishCommandOptions): Promise<ListrContextPublish> {
   let ethProvider = provider || eth_provider;
   let contentProvider = provider || content_provider;
@@ -65,7 +73,14 @@ export async function publishHandler({
       deleteOldPins,
       developerAddress,
       githubRelease,
-      verbosityOptions
+      verbosityOptions,
+      ...getVariantOptions({
+        allVariants: Boolean(allVariants),
+        variantsStr: variants,
+        rootDir: dir,
+        variantsDirName,
+        composeFileName
+      })
     }),
     verbosityOptions
   );

diff --git a/src/commands/publish/index.ts b/src/commands/publish/index.ts
@@ -10,6 +10,7 @@ import { printObject } from "../../utils/print.js";
 import { UploadTo } from "../../releaseUploader/index.js";
 import { publishHandler } from "./handler.js";
 import { PublishCommandOptions } from "./types.js";
+import { defaultVariantsDirName } from "../../params.js";
 
 export const publish: CommandModule<CliGlobalOptions, PublishCommandOptions> = {
   command: "publish [type]",
@@ -66,6 +67,22 @@ export const publish: CommandModule<CliGlobalOptions, PublishCommandOptions> = {
       alias: "dappnode-team-preset",
       description: `Specific set of options used for internal DAppNode releases. Caution: options may change without notice.`,
       type: "boolean"
+    },
+    all_variants: {
+      alias: "all-variants",
+      description: `It will use the dappnode_package.json and docker-compose.yml files in the root of the project together with the specific ones defined for each package variant to build all of them`,
+      type: "boolean"
+    },
+    variants_dir_name: {
+      alias: "variants-dir-name",
+      description: `Name of the directory where the package variants are located (only for packages that support it and combined with either "--all-variants" or "--variants"). By default, it is ${defaultVariantsDirName}`,
+      type: "string",
+      default: defaultVariantsDirName
+    },
+    variants: {
+      alias: "variant",
+      description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. If not specified, all variants will be built. Example: "variant1,variant2"`,
+      type: "string"
     }
   },
 

diff --git a/src/commands/publish/types.ts b/src/commands/publish/types.ts
@@ -13,4 +13,7 @@ export interface PublishCommandOptions extends CliGlobalOptions {
   dappnode_team_preset?: boolean;
   require_git_data?: boolean;
   delete_old_pins?: boolean;
+  variants_dir_name?: string;
+  variants?: string;
+  all_variants?: boolean;
 }
diff --git a/src/params.ts b/src/params.ts
@@ -27,6 +27,7 @@ export const upstreamImageLabel = "dappnode.dnp.upstreamImage";
 export const PINATA_URL = "https://api.pinata.cloud";
 // The build_sdk.env file is used by "slaves" DAppNode packages to define the UPSTREAM_PROJECT and UPSTREAM_VERSION used in the gha
 export const buildSdkEnvFileName = "build_sdk.env";
+export const contentHashFileName = "content-hash" as const;
 
 export const releaseFilesDefaultNames: {
   [P in keyof typeof releaseFiles]: string;

diff --git a/src/providers/github/Github.ts b/src/providers/github/Github.ts
@@ -202,7 +202,7 @@ export class Github {
         owner: this.owner,
         repo: this.repo,
         tag_name: tag,
-        name: tag,
+        name: this.buildReleaseNameFromTag(tag),
         body,
         prerelease,
         generate_release_notes: true,
@@ -216,6 +216,29 @@ export class Github {
       });
   }
 
+  /**
+   * Receives a tag and returns a prettified release name
+   *
+   * For single-variant packages:
+   *  Tag: "v0.2.0" => Release name: "v0.2.0"
+   *
+   * For multi-variant packages:
+   *  Tag: "[email protected][email protected][email protected]" => Release name: "Gnosis(v0.1.2), Holesky(v1.2.3), Mainnet(v3.21.1)"
+   *
+   * @param tag
+   */
+  private buildReleaseNameFromTag(tag: string): string {
+    const variants = tag.split("_").map(variant => {
+      const [name, version] = variant.split("@");
+
+      // If the variant is a single-variant package
+      if (!version) return name;
+
+      return `${name}(${version})`;
+    });
+    return variants.join(", ");
+  }
+
   /**
    * Open a Github pull request
    */

diff --git a/src/tasks/buildAndUpload/buildVariantMap.ts b/src/tasks/buildAndUpload/buildVariantMap.ts
@@ -7,25 +7,25 @@ import {
   parseComposeUpstreamVersion,
   readManifest
 } from "../../files/index.js";
-import { VariantsMap, VariantsMapEntry } from "./types.js";
 import { Compose, Manifest } from "@dappnode/types";
 import { defaultComposeFileName } from "../../params.js";
+import { BuildVariantsMap, BuildVariantsMapEntry } from "../../types.js";
 
 export function buildVariantMap({
   variants,
   rootDir,
   variantsDirPath,
   composeFileName = defaultComposeFileName
 }: {
-  variants?: string[];
+  variants: string[] | null;
   rootDir: string;
   variantsDirPath: string;
   composeFileName?: string;
-}): VariantsMap {
+}): BuildVariantsMap {
   if (!variants || variants.length === 0)
     return { default: createVariantMapEntry({ rootDir, composeFileName }) };
 
-  const map: VariantsMap = {};
+  const map: BuildVariantsMap = {};
 
   for (const variant of variants) {
     const variantPath = path.join(variantsDirPath, variant);
@@ -47,7 +47,7 @@ export function createVariantMapEntry({
   rootDir: string;
   composeFileName: string;
   variantPath?: string;
-}): VariantsMapEntry {
+}): BuildVariantsMapEntry {
   const { manifest, format } = variantPath
     ? readManifest([{ dir: rootDir }, { dir: variantPath }])
     : readManifest([{ dir: rootDir }]);
@@ -61,16 +61,14 @@ export function createVariantMapEntry({
 
   const compose = variantPath
     ? readCompose([
-        { dir: rootDir, composeFileName },
-        { dir: variantPath, composeFileName }
-      ])
+      { dir: rootDir, composeFileName },
+      { dir: variantPath, composeFileName }
+    ])
     : readCompose([{ dir: rootDir, composeFileName }]);
 
-  // Only parse this upstream version for legacy format (upstreamVersion field defined instead of upstream object)
-  if (!manifest.upstream) {
-    const upstreamVersion = getUpstreamVersion({ compose, manifest });
-    manifest.upstreamVersion = upstreamVersion;
-  }
+  // TODO: Handle upstream object defined case
+  if (!manifest.upstream)
+    manifest.upstreamVersion = getUpstreamVersion({ compose, manifest });
 
   return {
     manifest,