From c36e00acbff13f64acee079c2f474c5183a72165 Mon Sep 17 00:00:00 2001 From: dappnodedev <144998261+dappnodedev@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:01:39 +0200 Subject: [PATCH] Adapt publish command for multi-variant packages (#431) * Update octokit * Handle multi-variant packages for publish * Update tests * Fix tests * Remove unused code * Fix upstream version * Increase version for variants * Update flags description * Renamed file to file name for content-hash * Remove comment * Squashed commit of the following: commit cfbdb0fe5b383641bc7cf94a44164f1e185e9286 Author: dappnodedev Date: Tue Jun 18 17:28:29 2024 +0200 0.3.24 commit 2ff654a50bbf771fdeb28708d7bbcf863fce5144 Author: dappnodedev <144998261+dappnodedev@users.noreply.github.com> Date: Tue Jun 18 17:21:54 2024 +0200 Bugfix: Create Github releases (#434) * Bump octokit version * Fix Github release creation * Update all github requests * Do not upload release assets * Use octokit.rest * Improve Github error handling commit cc60a42dad188b77177da7ea1b8ba53af7a2b44e Author: dappnodedev Date: Fri Jun 14 16:17:01 2024 +0200 0.3.23 commit 93ebd234d7fba5e9043a9ee7d362efe41eed334a Author: dappnodedev <144998261+dappnodedev@users.noreply.github.com> Date: Fri Jun 14 16:05:05 2024 +0200 Fix build for new upstream format (#433) * Fix build for new upstream format * Compare with string status 404 (not number) * Update remote IPFS API for tests * Fix BuildVariantsMap import * Add "NoThrow" to the generate release notes function * Copy prometheus targets to release dir * Handle variant dir not existent * Declare var outside switch * SImplify release body --- src/commands/build/handler.ts | 50 +---- src/commands/build/index.ts | 4 +- src/commands/build/variants.ts | 51 +++++ src/commands/from_github.ts | 8 +- src/commands/publish/handler.ts | 21 +- src/commands/publish/index.ts | 17 ++ src/commands/publish/types.ts | 3 + src/params.ts | 1 + src/providers/github/Github.ts | 25 ++- src/tasks/buildAndUpload/buildVariantMap.ts | 24 +-- src/tasks/buildAndUpload/getBuildTasks.ts | 7 +- .../buildAndUpload/getDeleteOldPinsTask.ts | 5 +- src/tasks/buildAndUpload/getFileCopyTask.ts | 43 +++- .../buildAndUpload/getFileValidationTask.ts | 11 +- .../getReleaseDirCreationTask.ts | 7 +- .../getSaveUploadResultsTask.ts | 5 +- src/tasks/buildAndUpload/getUploadTasks.ts | 5 +- src/tasks/buildAndUpload/index.ts | 11 +- src/tasks/buildAndUpload/types.ts | 27 +-- .../buildReleaseDetailsMap.ts | 26 +++ .../createGithubRelease/getNextGitTag.ts | 36 ++++ src/tasks/createGithubRelease/index.ts | 10 +- .../subtasks/getCreateReleaseTask.ts | 196 ++++++++++++------ .../subtasks/getHandleTagsTask.ts | 10 +- .../subtasks/getNextGitTag.ts | 7 - src/tasks/createGithubRelease/types.ts | 11 + .../getRegistryAddressFromEns.ts | 0 .../index.ts | 144 ++++++++----- .../types.ts | 0 src/tasks/publish/index.ts | 25 ++- .../subtasks/getCreateGithubReleaseTask.ts | 25 +-- .../subtasks/getFetchApmVersionsTask.ts | 137 ++++++++++++ .../publish/subtasks/getGenerateTxTask.ts | 52 ----- .../publish/subtasks/getGenerateTxsTask.ts | 34 +++ src/tasks/publish/types.ts | 4 +- src/types.ts | 28 ++- .../buildAndUpload/buildVariantMap.test.ts | 3 +- test/tasks/generatePublishTx.test.ts | 65 +++--- test/tasks/gitTags.test.ts | 37 ++++ 39 files changed, 793 insertions(+), 382 deletions(-) create mode 100644 src/tasks/createGithubRelease/buildReleaseDetailsMap.ts create mode 100644 src/tasks/createGithubRelease/getNextGitTag.ts delete mode 100644 src/tasks/createGithubRelease/subtasks/getNextGitTag.ts create mode 100644 src/tasks/createGithubRelease/types.ts rename src/tasks/{generatePublishTx => generatePublishTxs}/getRegistryAddressFromEns.ts (100%) rename src/tasks/{generatePublishTx => generatePublishTxs}/index.ts (71%) rename src/tasks/{generatePublishTx => generatePublishTxs}/types.ts (100%) create mode 100644 src/tasks/publish/subtasks/getFetchApmVersionsTask.ts delete mode 100644 src/tasks/publish/subtasks/getGenerateTxTask.ts create mode 100644 src/tasks/publish/subtasks/getGenerateTxsTask.ts create mode 100644 test/tasks/gitTags.test.ts diff --git a/src/commands/build/handler.ts b/src/commands/build/handler.ts index 584d9ef3..446d993d 100644 --- a/src/commands/build/handler.ts +++ b/src/commands/build/handler.ts @@ -1,5 +1,3 @@ -import path from "path"; -import chalk from "chalk"; import Listr from "listr"; import { buildAndUpload } from "../../tasks/buildAndUpload/index.js"; import { ListrContextBuild } from "../../types.js"; @@ -9,7 +7,7 @@ import { defaultVariantsDirName } from "../../params.js"; import { BuildCommandOptions, VerbosityOptions } from "./types.js"; -import { getValidVariantNames } from "./variants.js"; +import { getVariantOptions } from "./variants.js"; import { BuildAndUploadOptions } from "../../tasks/buildAndUpload/types.js"; export async function buildHandler({ @@ -31,10 +29,6 @@ export async function buildHandler({ }: BuildCommandOptions): Promise { const skipUpload = skip_upload || skipSave; - const multiVariantMode = Boolean( - allVariants || (variants && variants?.length > 0) - ); - const buildOptions: BuildAndUploadOptions = { dir, contentProvider, @@ -45,12 +39,13 @@ export async function buildHandler({ composeFileName, requireGitData, deleteOldPins, - ...(multiVariantMode && - getVariantOptions({ - variantsStr: variants, - rootDir: dir, - variantsDirName - })) + ...getVariantOptions({ + allVariants: Boolean(allVariants), + variantsStr: variants, + rootDir: dir, + variantsDirName, + composeFileName + }) }; const verbosityOptions: VerbosityOptions = { @@ -61,32 +56,3 @@ export async function buildHandler({ return await buildTasks.run(); } - -function getVariantOptions({ - variantsStr, - rootDir, - variantsDirName -}: { - variantsStr: string | undefined; - rootDir: string; - variantsDirName: string; -}): { variants: string[]; variantsDirPath: string } { - const variantsDirPath = path.join(rootDir, variantsDirName); - const variantNames = getValidVariantNames({ - variantsDirPath, - variants: variantsStr - }); - - if (variantNames.length === 0) - throw new Error( - `No valid variants specified. They must be included in: ${variantsDirPath}` - ); - - console.log( - `${chalk.dim( - `Building package from template for variant(s) ${variantsStr}...` - )}` - ); - - return { variants: variantNames, variantsDirPath }; -} diff --git a/src/commands/build/index.ts b/src/commands/build/index.ts index 35d5bf43..623b0711 100644 --- a/src/commands/build/index.ts +++ b/src/commands/build/index.ts @@ -40,7 +40,7 @@ export const build: CommandModule = { }, all_variants: { alias: "all-variants", - description: `It will use the dappnode_package.json and docker-compose.yml files in the root of the project together with the specific ones defined for each package variant to build all of them`, + description: `Build all package variants at once, by merging the dappnode_package.json and docker-compose.yml files in the root of the project with the specific ones defined for each package variant`, type: "boolean" }, variants_dir_name: { @@ -51,7 +51,7 @@ export const build: CommandModule = { }, variants: { alias: "variant", - description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. If not specified, all variants will be built. Example: "variant1,variant2"`, + description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. Example: "variant1,variant2"`, type: "string" } }, diff --git a/src/commands/build/variants.ts b/src/commands/build/variants.ts index 0b6dee0f..9abcb418 100644 --- a/src/commands/build/variants.ts +++ b/src/commands/build/variants.ts @@ -1,5 +1,56 @@ import chalk from "chalk"; import { getAllVariantsInPath } from "../../files/variants/getAllPackageVariants.js"; +import path from "path"; +import { BuildVariantsMap } from "../../types.js"; +import { buildVariantMap } from "../../tasks/buildAndUpload/buildVariantMap.js"; + +export function getVariantOptions({ + allVariants, + variantsStr, + rootDir, + variantsDirName, + composeFileName +}: { + allVariants: boolean; + variantsStr?: string; + rootDir: string; + variantsDirName: string; + composeFileName: string; +}): { variantsMap: BuildVariantsMap; variantsDirPath: string } { + const variantsDirPath = path.join(rootDir, variantsDirName); + + const buildVariantMapArgs = { rootDir, variantsDirPath, composeFileName }; + + if (!allVariants && !variantsStr) + return { + variantsMap: buildVariantMap({ ...buildVariantMapArgs, variants: null }), + variantsDirPath + }; + + const validVariantNames = getValidVariantNames({ + variantsDirPath, + variants: variantsStr + }); + + if (validVariantNames.length === 0) + throw new Error( + `No valid variants specified. They must be included in: ${variantsDirPath}` + ); + + console.log( + `${chalk.dim( + `Building package from template for variant(s) ${variantsStr}...` + )}` + ); + + return { + variantsMap: buildVariantMap({ + ...buildVariantMapArgs, + variants: validVariantNames + }), + variantsDirPath + }; +} /** * Main function to retrieve the valid variant names based on the specified variants and available directories. diff --git a/src/commands/from_github.ts b/src/commands/from_github.ts index e40b2d14..f0d2f082 100644 --- a/src/commands/from_github.ts +++ b/src/commands/from_github.ts @@ -13,11 +13,11 @@ import { } from "../utils/githubGetReleases.js"; import { ipfsAddDirFromUrls } from "../releaseUploader/ipfsNode/addDirFromUrls.js"; import { verifyIpfsConnection } from "../releaseUploader/ipfsNode/verifyConnection.js"; -import { CliGlobalOptions, contentHashFile } from "../types.js"; +import { CliGlobalOptions } from "../types.js"; import { Manifest, defaultArch, releaseFiles } from "@dappnode/types"; import { getLegacyImagePath } from "../utils/getLegacyImagePath.js"; import { getImageFileName } from "../utils/getImageFileName.js"; -import { releaseFilesDefaultNames } from "../params.js"; +import { contentHashFileName, releaseFilesDefaultNames } from "../params.js"; interface CliCommandOptions extends CliGlobalOptions { repoSlug: string; @@ -111,7 +111,7 @@ export async function fromGithubHandler({ } const files = release.assets - .filter(asset => asset.name !== contentHashFile) + .filter(asset => asset.name !== contentHashFileName) .map(asset => ({ filepath: path.join("release", asset.name), url: asset.browser_download_url, @@ -147,7 +147,7 @@ export async function fromGithubHandler({ // Verify that the resulting release hash matches the one in Github const contentHashAsset = release.assets.find( - asset => asset.name === contentHashFile + asset => asset.name === contentHashFileName ); if (contentHashAsset) { const contentHash = await got(contentHashAsset.browser_download_url).text(); diff --git a/src/commands/publish/handler.ts b/src/commands/publish/handler.ts index 9316ca66..4961a7e0 100644 --- a/src/commands/publish/handler.ts +++ b/src/commands/publish/handler.ts @@ -1,10 +1,15 @@ import Listr from "listr"; -import { defaultComposeFileName, defaultDir } from "../../params.js"; +import { + defaultComposeFileName, + defaultDir, + defaultVariantsDirName +} from "../../params.js"; import { ListrContextPublish } from "../../types.js"; import { VerbosityOptions } from "../build/types.js"; import { PublishCommandOptions } from "./types.js"; import { publish } from "../../tasks/publish/index.js"; import { parseReleaseType } from "./parseReleaseType.js"; +import { getVariantOptions } from "../build/variants.js"; /** * Common handler for CLI and programatic usage @@ -25,7 +30,10 @@ export async function publishHandler({ dir = defaultDir, compose_file_name: composeFileName = defaultComposeFileName, silent, - verbose + verbose, + variants_dir_name: variantsDirName = defaultVariantsDirName, + all_variants: allVariants, + variants }: PublishCommandOptions): Promise { let ethProvider = provider || eth_provider; let contentProvider = provider || content_provider; @@ -65,7 +73,14 @@ export async function publishHandler({ deleteOldPins, developerAddress, githubRelease, - verbosityOptions + verbosityOptions, + ...getVariantOptions({ + allVariants: Boolean(allVariants), + variantsStr: variants, + rootDir: dir, + variantsDirName, + composeFileName + }) }), verbosityOptions ); diff --git a/src/commands/publish/index.ts b/src/commands/publish/index.ts index a1d1e7eb..ce023a7b 100644 --- a/src/commands/publish/index.ts +++ b/src/commands/publish/index.ts @@ -10,6 +10,7 @@ import { printObject } from "../../utils/print.js"; import { UploadTo } from "../../releaseUploader/index.js"; import { publishHandler } from "./handler.js"; import { PublishCommandOptions } from "./types.js"; +import { defaultVariantsDirName } from "../../params.js"; export const publish: CommandModule = { command: "publish [type]", @@ -66,6 +67,22 @@ export const publish: CommandModule = { alias: "dappnode-team-preset", description: `Specific set of options used for internal DAppNode releases. Caution: options may change without notice.`, type: "boolean" + }, + all_variants: { + alias: "all-variants", + description: `It will use the dappnode_package.json and docker-compose.yml files in the root of the project together with the specific ones defined for each package variant to build all of them`, + type: "boolean" + }, + variants_dir_name: { + alias: "variants-dir-name", + description: `Name of the directory where the package variants are located (only for packages that support it and combined with either "--all-variants" or "--variants"). By default, it is ${defaultVariantsDirName}`, + type: "string", + default: defaultVariantsDirName + }, + variants: { + alias: "variant", + description: `Specify the package variants to build (only for packages that support it). Defined by comma-separated list of variant names. If not specified, all variants will be built. Example: "variant1,variant2"`, + type: "string" } }, diff --git a/src/commands/publish/types.ts b/src/commands/publish/types.ts index ba99547d..a1bcc421 100644 --- a/src/commands/publish/types.ts +++ b/src/commands/publish/types.ts @@ -13,4 +13,7 @@ export interface PublishCommandOptions extends CliGlobalOptions { dappnode_team_preset?: boolean; require_git_data?: boolean; delete_old_pins?: boolean; + variants_dir_name?: string; + variants?: string; + all_variants?: boolean; } diff --git a/src/params.ts b/src/params.ts index ad5a661d..1da09560 100644 --- a/src/params.ts +++ b/src/params.ts @@ -27,6 +27,7 @@ export const upstreamImageLabel = "dappnode.dnp.upstreamImage"; export const PINATA_URL = "https://api.pinata.cloud"; // The build_sdk.env file is used by "slaves" DAppNode packages to define the UPSTREAM_PROJECT and UPSTREAM_VERSION used in the gha export const buildSdkEnvFileName = "build_sdk.env"; +export const contentHashFileName = "content-hash" as const; export const releaseFilesDefaultNames: { [P in keyof typeof releaseFiles]: string; diff --git a/src/providers/github/Github.ts b/src/providers/github/Github.ts index d7ca992b..e234ad5a 100644 --- a/src/providers/github/Github.ts +++ b/src/providers/github/Github.ts @@ -202,7 +202,7 @@ export class Github { owner: this.owner, repo: this.repo, tag_name: tag, - name: tag, + name: this.buildReleaseNameFromTag(tag), body, prerelease, generate_release_notes: true, @@ -216,6 +216,29 @@ export class Github { }); } + /** + * Receives a tag and returns a prettified release name + * + * For single-variant packages: + * Tag: "v0.2.0" => Release name: "v0.2.0" + * + * For multi-variant packages: + * Tag: "gnosis@v0.1.2_holesky@v1.2.3_mainnet@v3.21.1" => Release name: "Gnosis(v0.1.2), Holesky(v1.2.3), Mainnet(v3.21.1)" + * + * @param tag + */ + private buildReleaseNameFromTag(tag: string): string { + const variants = tag.split("_").map(variant => { + const [name, version] = variant.split("@"); + + // If the variant is a single-variant package + if (!version) return name; + + return `${name}(${version})`; + }); + return variants.join(", "); + } + /** * Open a Github pull request */ diff --git a/src/tasks/buildAndUpload/buildVariantMap.ts b/src/tasks/buildAndUpload/buildVariantMap.ts index 73356776..34b2fefe 100644 --- a/src/tasks/buildAndUpload/buildVariantMap.ts +++ b/src/tasks/buildAndUpload/buildVariantMap.ts @@ -7,9 +7,9 @@ import { parseComposeUpstreamVersion, readManifest } from "../../files/index.js"; -import { VariantsMap, VariantsMapEntry } from "./types.js"; import { Compose, Manifest } from "@dappnode/types"; import { defaultComposeFileName } from "../../params.js"; +import { BuildVariantsMap, BuildVariantsMapEntry } from "../../types.js"; export function buildVariantMap({ variants, @@ -17,15 +17,15 @@ export function buildVariantMap({ variantsDirPath, composeFileName = defaultComposeFileName }: { - variants?: string[]; + variants: string[] | null; rootDir: string; variantsDirPath: string; composeFileName?: string; -}): VariantsMap { +}): BuildVariantsMap { if (!variants || variants.length === 0) return { default: createVariantMapEntry({ rootDir, composeFileName }) }; - const map: VariantsMap = {}; + const map: BuildVariantsMap = {}; for (const variant of variants) { const variantPath = path.join(variantsDirPath, variant); @@ -47,7 +47,7 @@ export function createVariantMapEntry({ rootDir: string; composeFileName: string; variantPath?: string; -}): VariantsMapEntry { +}): BuildVariantsMapEntry { const { manifest, format } = variantPath ? readManifest([{ dir: rootDir }, { dir: variantPath }]) : readManifest([{ dir: rootDir }]); @@ -61,16 +61,14 @@ export function createVariantMapEntry({ const compose = variantPath ? readCompose([ - { dir: rootDir, composeFileName }, - { dir: variantPath, composeFileName } - ]) + { dir: rootDir, composeFileName }, + { dir: variantPath, composeFileName } + ]) : readCompose([{ dir: rootDir, composeFileName }]); - // Only parse this upstream version for legacy format (upstreamVersion field defined instead of upstream object) - if (!manifest.upstream) { - const upstreamVersion = getUpstreamVersion({ compose, manifest }); - manifest.upstreamVersion = upstreamVersion; - } + // TODO: Handle upstream object defined case + if (!manifest.upstream) + manifest.upstreamVersion = getUpstreamVersion({ compose, manifest }); return { manifest, diff --git a/src/tasks/buildAndUpload/getBuildTasks.ts b/src/tasks/buildAndUpload/getBuildTasks.ts index 3307c139..5032ca0c 100644 --- a/src/tasks/buildAndUpload/getBuildTasks.ts +++ b/src/tasks/buildAndUpload/getBuildTasks.ts @@ -1,11 +1,10 @@ import path from "path"; import Listr, { ListrTask } from "listr/index.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, BuildVariantsMapEntry, ListrContextBuild } from "../../types.js"; import { buildWithBuildx } from "./buildWithBuildx.js"; import { buildWithCompose } from "./buildWithCompose.js"; import { Architecture, defaultArch } from "@dappnode/types"; import { getImageFileName } from "../../utils/getImageFileName.js"; -import { VariantsMap, VariantsMapEntry } from "./types.js"; /** * The naming scheme for multiarch exported images must be @@ -20,7 +19,7 @@ export function getBuildTasks({ skipSave, rootDir }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; buildTimeout: number; skipSave?: boolean; rootDir: string; @@ -49,7 +48,7 @@ function createBuildTask({ skipSave, rootDir }: { - variantSpecs: VariantsMapEntry; + variantSpecs: BuildVariantsMapEntry; architecture: Architecture; buildTimeout: number; skipSave?: boolean; diff --git a/src/tasks/buildAndUpload/getDeleteOldPinsTask.ts b/src/tasks/buildAndUpload/getDeleteOldPinsTask.ts index c20ddb79..d9b00d03 100644 --- a/src/tasks/buildAndUpload/getDeleteOldPinsTask.ts +++ b/src/tasks/buildAndUpload/getDeleteOldPinsTask.ts @@ -1,17 +1,16 @@ import { ListrTask } from "listr/index.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, ListrContextBuild } from "../../types.js"; import { getGitHead } from "../../utils/git.js"; import { fetchPinsWithBranchToDelete } from "../../pinStrategy/index.js"; import { PinataPinManager } from "../../providers/pinata/pinManager.js"; import { ReleaseUploaderProvider } from "../../releaseUploader/index.js"; -import { VariantsMap } from "./types.js"; export function getDeleteOldPinsTask({ variantsMap, deleteOldPins, releaseUploaderProvider }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; deleteOldPins: boolean; releaseUploaderProvider: ReleaseUploaderProvider; }): ListrTask { diff --git a/src/tasks/buildAndUpload/getFileCopyTask.ts b/src/tasks/buildAndUpload/getFileCopyTask.ts index e44d2ae4..6deb1291 100644 --- a/src/tasks/buildAndUpload/getFileCopyTask.ts +++ b/src/tasks/buildAndUpload/getFileCopyTask.ts @@ -1,3 +1,4 @@ +import fs from "fs"; import path from "path"; import { ListrTask } from "listr/index.js"; import { verifyAvatar } from "../../utils/verifyAvatar.js"; @@ -6,7 +7,7 @@ import { defaultComposeFileName, releaseFilesDefaultNames } from "../../params.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, BuildVariantsMapEntry, ListrContextBuild } from "../../types.js"; import { getGitHeadIfAvailable } from "../../utils/git.js"; import { updateComposeImageTags, @@ -14,15 +15,16 @@ import { writeManifest } from "../../files/index.js"; import { Compose, Manifest, releaseFiles } from "@dappnode/types"; -import { VariantsMap, VariantsMapEntry } from "./types.js"; export function getFileCopyTask({ variantsMap, + variantsDirPath, rootDir, composeFileName, requireGitData }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; + variantsDirPath: string; rootDir: string; composeFileName: string; requireGitData?: boolean; @@ -32,6 +34,7 @@ export function getFileCopyTask({ task: async () => copyFilesToReleaseDir({ variantsMap, + variantsDirPath, rootDir, composeFileName, requireGitData @@ -41,21 +44,25 @@ export function getFileCopyTask({ async function copyFilesToReleaseDir({ variantsMap, + variantsDirPath, rootDir, composeFileName, requireGitData }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; + variantsDirPath: string; rootDir: string; composeFileName: string; requireGitData?: boolean; }): Promise { - for (const [, variant] of Object.entries(variantsMap)) { - await copyVariantFilesToReleaseDir({ variant, rootDir, composeFileName }); + for (const [variantName, variantProps] of Object.entries(variantsMap)) { + + const variantDirPath = path.join(variantsDirPath, variantName); + await copyVariantFilesToReleaseDir({ variantProps, variantDirPath: variantDirPath, rootDir, composeFileName }); // Verify avatar (throws) const avatarPath = path.join( - variant.releaseDir, + variantProps.releaseDir, releaseFilesDefaultNames.avatar ); verifyAvatar(avatarPath); @@ -66,17 +73,22 @@ async function copyFilesToReleaseDir({ } async function copyVariantFilesToReleaseDir({ - variant, + variantProps, + variantDirPath, rootDir, composeFileName }: { - variant: VariantsMapEntry; + variantProps: BuildVariantsMapEntry; + variantDirPath: string; rootDir: string; composeFileName: string; }): Promise { - const { manifest, manifestFormat, releaseDir, compose } = variant; + const { manifest, manifestFormat, releaseDir, compose } = variantProps; for (const [fileId, fileConfig] of Object.entries(releaseFiles)) { + // For single variant packages, the targets are in the root dir + const dirsToCopy = fs.existsSync(variantDirPath) ? [rootDir, variantDirPath] : [rootDir]; + switch (fileId as keyof typeof releaseFiles) { case "manifest": writeManifest(manifest, manifestFormat, { dir: releaseDir }); @@ -86,6 +98,17 @@ async function copyVariantFilesToReleaseDir({ writeReleaseCompose({ compose, composeFileName, manifest, releaseDir }); break; + case "prometheusTargets": + // Copy the targets in root and in the variant dir + for (const dir of dirsToCopy) { + copyReleaseFile({ + fileConfig: { ...fileConfig, id: fileId }, + fromDir: dir, + toDir: releaseDir + }); + } + break; + default: copyReleaseFile({ fileConfig: { ...fileConfig, id: fileId }, diff --git a/src/tasks/buildAndUpload/getFileValidationTask.ts b/src/tasks/buildAndUpload/getFileValidationTask.ts index d44e7606..0d5520d8 100644 --- a/src/tasks/buildAndUpload/getFileValidationTask.ts +++ b/src/tasks/buildAndUpload/getFileValidationTask.ts @@ -1,5 +1,5 @@ import { ListrTask } from "listr/index.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, BuildVariantsMapEntry, ListrContextBuild } from "../../types.js"; import { validateComposeSchema, validateManifestSchema, @@ -7,14 +7,13 @@ import { validateDappnodeCompose } from "@dappnode/schemas"; import { readSetupWizardIfExists } from "../../files/index.js"; -import { VariantsMap, VariantsMapEntry } from "./types.js"; import { CliError } from "../../params.js"; export function getFileValidationTask({ variantsMap, rootDir }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; rootDir: string; }): ListrTask { return { @@ -27,7 +26,7 @@ async function validatePackageFiles({ variantsMap, rootDir }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; rootDir: string; }): Promise { const setupWizard = readSetupWizardIfExists(rootDir); @@ -38,7 +37,9 @@ async function validatePackageFiles({ await validateVariantFiles(variant); } -async function validateVariantFiles(variant: VariantsMapEntry): Promise { +async function validateVariantFiles( + variant: BuildVariantsMapEntry +): Promise { const { manifest, compose, composePaths } = variant; console.log( diff --git a/src/tasks/buildAndUpload/getReleaseDirCreationTask.ts b/src/tasks/buildAndUpload/getReleaseDirCreationTask.ts index f3970f4e..f9e47217 100644 --- a/src/tasks/buildAndUpload/getReleaseDirCreationTask.ts +++ b/src/tasks/buildAndUpload/getReleaseDirCreationTask.ts @@ -2,14 +2,13 @@ import fs from "fs"; import path from "path"; import { ListrTask } from "listr/index.js"; import rimraf from "rimraf"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, ListrContextBuild } from "../../types.js"; import { getImageFileName } from "../../utils/getImageFileName.js"; -import { VariantsMap } from "./types.js"; export function getReleaseDirCreationTask({ variantsMap }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; }): ListrTask { return { title: `Create release directories`, @@ -22,7 +21,7 @@ function createReleaseDirs({ variantsMap }: { ctx: ListrContextBuild; - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; }): void { for (const [ variant, diff --git a/src/tasks/buildAndUpload/getSaveUploadResultsTask.ts b/src/tasks/buildAndUpload/getSaveUploadResultsTask.ts index d75f85dc..eb68fbb4 100644 --- a/src/tasks/buildAndUpload/getSaveUploadResultsTask.ts +++ b/src/tasks/buildAndUpload/getSaveUploadResultsTask.ts @@ -1,8 +1,7 @@ import { ListrTask } from "listr/index.js"; import { addReleaseRecord } from "../../utils/releaseRecord.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, ListrContextBuild } from "../../types.js"; import { pruneCache } from "../../utils/cache.js"; -import { VariantsMap } from "./types.js"; import path from "path"; export function getSaveUploadResultsTask({ @@ -12,7 +11,7 @@ export function getSaveUploadResultsTask({ contentProvider, skipUpload }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; rootDir: string; variantsDirPath: string; contentProvider: string; diff --git a/src/tasks/buildAndUpload/getUploadTasks.ts b/src/tasks/buildAndUpload/getUploadTasks.ts index 8ed65f88..9c26bb35 100644 --- a/src/tasks/buildAndUpload/getUploadTasks.ts +++ b/src/tasks/buildAndUpload/getUploadTasks.ts @@ -1,11 +1,10 @@ import { ListrTask } from "listr/index.js"; -import { ListrContextBuild } from "../../types.js"; +import { BuildVariantsMap, ListrContextBuild } from "../../types.js"; import { getGitHeadIfAvailable } from "../../utils/git.js"; import { getPinMetadata } from "../../pinStrategy/index.js"; import { PinKeyvaluesDefault } from "../../releaseUploader/pinata/index.js"; import { IReleaseUploader } from "../../releaseUploader/index.js"; import { composeDeleteBuildProperties } from "../../files/index.js"; -import { VariantsMap } from "./types.js"; export function getUploadTasks({ variantsMap, @@ -14,7 +13,7 @@ export function getUploadTasks({ requireGitData, composeFileName }: { - variantsMap: VariantsMap; + variantsMap: BuildVariantsMap; skipUpload?: boolean; releaseUploader: IReleaseUploader; requireGitData: boolean; diff --git a/src/tasks/buildAndUpload/index.ts b/src/tasks/buildAndUpload/index.ts index ac0156bb..139b2b61 100644 --- a/src/tasks/buildAndUpload/index.ts +++ b/src/tasks/buildAndUpload/index.ts @@ -7,7 +7,6 @@ import { cliArgsToReleaseUploaderProvider } from "../../releaseUploader/index.js"; import { BuildAndUploadOptions } from "./types.js"; -import { buildVariantMap } from "./buildVariantMap.js"; import { getVerifyConnectionTask } from "./getVerifyConnectionTask.js"; import { getReleaseDirCreationTask } from "./getReleaseDirCreationTask.js"; import { getFileValidationTask } from "./getFileValidationTask.js"; @@ -28,7 +27,7 @@ export function buildAndUpload({ composeFileName, dir, variantsDirPath = defaultVariantsDirName, - variants + variantsMap }: BuildAndUploadOptions): ListrTask[] { const buildTimeout = parseTimeout(userTimeout); @@ -39,19 +38,13 @@ export function buildAndUpload({ }); const releaseUploader = getReleaseUploader(releaseUploaderProvider); - const variantsMap = buildVariantMap({ - variants, - rootDir: dir, - variantsDirPath, - composeFileName - }); - return [ getFileValidationTask({ variantsMap, rootDir: dir }), getVerifyConnectionTask({ releaseUploader, skipUpload }), getReleaseDirCreationTask({ variantsMap }), getFileCopyTask({ variantsMap, + variantsDirPath, rootDir: dir, composeFileName, requireGitData diff --git a/src/tasks/buildAndUpload/types.ts b/src/tasks/buildAndUpload/types.ts index 05e809d5..cdac9d2d 100644 --- a/src/tasks/buildAndUpload/types.ts +++ b/src/tasks/buildAndUpload/types.ts @@ -1,7 +1,5 @@ -import { PackageImage } from "../../types.js"; +import { BuildVariantsMap, PackageImage } from "../../types.js"; import { UploadTo } from "../../releaseUploader/index.js"; -import { Architecture, Compose, Manifest } from "@dappnode/types"; -import { ManifestFormat } from "../../files/manifest/types.js"; export interface BuildAndUploadOptions { contentProvider: string; @@ -14,26 +12,5 @@ export interface BuildAndUploadOptions { composeFileName: string; dir: string; variantsDirPath?: string; - variants?: string[]; -} - -export interface VariantsMapEntry { - // Manifest-related - manifest: Manifest; - manifestFormat: ManifestFormat; - - // Compose file - compose: Compose; - - // File paths - releaseDir: string; - composePaths: string[]; - - // Package information - images: PackageImage[]; - architectures: Architecture[]; -} - -export interface VariantsMap { - [variant: string]: VariantsMapEntry; + variantsMap: BuildVariantsMap; } diff --git a/src/tasks/createGithubRelease/buildReleaseDetailsMap.ts b/src/tasks/createGithubRelease/buildReleaseDetailsMap.ts new file mode 100644 index 00000000..5a0caa73 --- /dev/null +++ b/src/tasks/createGithubRelease/buildReleaseDetailsMap.ts @@ -0,0 +1,26 @@ +import { ListrContextPublish } from "../../types.js"; +import { ReleaseDetailsMap } from "./types.js"; + +export function buildReleaseDetailsMap( + ctx: ListrContextPublish +): ReleaseDetailsMap { + const releaseDetailsMap: ReleaseDetailsMap = {}; + + for (const [ + dnpName, + { nextVersion, releaseMultiHash, txData, releaseDir, variant } + ] of Object.entries(ctx)) { + if (!nextVersion || !releaseMultiHash || !txData || !releaseDir || !variant) + throw new Error(`Missing required release details for ${dnpName}`); + + releaseDetailsMap[dnpName] = { + nextVersion, + releaseMultiHash, + txData, + releaseDir, + variant + }; + } + + return releaseDetailsMap; +} diff --git a/src/tasks/createGithubRelease/getNextGitTag.ts b/src/tasks/createGithubRelease/getNextGitTag.ts new file mode 100644 index 00000000..7c569adb --- /dev/null +++ b/src/tasks/createGithubRelease/getNextGitTag.ts @@ -0,0 +1,36 @@ +import { ReleaseDetailsMap } from "./types.js"; + +// Only variant and nextVersion are needed from the ReleaseDetailsMap +type GitTagDetailsMap = { + [K in keyof ReleaseDetailsMap]: Pick< + ReleaseDetailsMap[K], + "variant" | "nextVersion" + >; +}; + +/** + * Returns the next git tag based on the next version defined in the context + * + * - If the package is a multi-variant package, the tag will be in the format: + * `gnosis@v0.1.2_holesky@v1.2.3_mainnet@v3.21.1` sorted alphabetically by variant + * + * - If the package is a single-variant package, the tag will be in the format: + * `v0.1.2` + */ +export function getNextGitTag(releaseDetailsMap: GitTagDetailsMap): string { + const variantVersions = Object.entries( + releaseDetailsMap + ).map(([, { variant, nextVersion }]) => ({ variant, nextVersion })); + + if (variantVersions.length === 0) + throw Error("Could not generate git tag. Missing variant or nextVersion"); + + // Not a multi-variant package + if (variantVersions.length === 1) return `v${variantVersions[0].nextVersion}`; + + // Multi-variant package + return variantVersions + .sort((a, b) => a.variant.localeCompare(b.variant)) // Sort alphabetically by variant + .map(({ variant, nextVersion }) => `${variant}@${nextVersion}`) // Map to string + .join("_"); // Join into a single string +} diff --git a/src/tasks/createGithubRelease/index.ts b/src/tasks/createGithubRelease/index.ts index cfeb3b17..4ef9833a 100644 --- a/src/tasks/createGithubRelease/index.ts +++ b/src/tasks/createGithubRelease/index.ts @@ -10,29 +10,23 @@ import { getCreateReleaseTask } from "./subtasks/getCreateReleaseTask.js"; * Create (or edit) a Github release, then upload all assets */ export function createGithubRelease({ - dir = defaultDir, + dir: rootDir = defaultDir, compose_file_name: composeFileName, - buildDir, - releaseMultiHash, verbosityOptions }: { - buildDir: string; - releaseMultiHash: string; verbosityOptions: VerbosityOptions; } & CliGlobalOptions): Listr { // OAuth2 token from Github if (!process.env.GITHUB_TOKEN) throw Error("GITHUB_TOKEN ENV (OAuth2) is required"); - const github = Github.fromLocal(dir); + const github = Github.fromLocal(rootDir); return new Listr( [ getHandleTagsTask({ github }), getCreateReleaseTask({ github, - buildDir, - releaseMultiHash, composeFileName }) ], diff --git a/src/tasks/createGithubRelease/subtasks/getCreateReleaseTask.ts b/src/tasks/createGithubRelease/subtasks/getCreateReleaseTask.ts index cdbfee93..c925a06b 100644 --- a/src/tasks/createGithubRelease/subtasks/getCreateReleaseTask.ts +++ b/src/tasks/createGithubRelease/subtasks/getCreateReleaseTask.ts @@ -1,11 +1,7 @@ import path from "path"; import fs from "fs"; import { Github } from "../../../providers/github/Github.js"; -import { - ListrContextPublish, - TxData, - contentHashFile -} from "../../../types.js"; +import { ListrContextPublish, TxData } from "../../../types.js"; import { ListrTask } from "listr"; import { compactManifestIfCore, @@ -15,7 +11,10 @@ import { getInstallDnpLink, getPublishTxLink } from "../../../utils/getLinks.js"; -import { getNextGitTag } from "./getNextGitTag.js"; +import { getNextGitTag } from "../getNextGitTag.js"; +import { contentHashFileName } from "../../../params.js"; +import { ReleaseDetailsMap } from "../types.js"; +import { buildReleaseDetailsMap } from "../buildReleaseDetailsMap.js"; /** * Create release @@ -24,50 +23,71 @@ import { getNextGitTag } from "./getNextGitTag.js"; */ export function getCreateReleaseTask({ github, - buildDir, - releaseMultiHash, composeFileName }: { github: Github; - buildDir: string; - releaseMultiHash: string; composeFileName?: string; }): ListrTask { return { title: `Create release`, task: async (ctx, task) => { - // TODO: Do this for each release - const [, { nextVersion, txData }] = Object.entries(ctx)[0]; + const releaseDetailsMap = buildReleaseDetailsMap(ctx); - const tag = getNextGitTag(nextVersion); + const tag = getNextGitTag(releaseDetailsMap); task.output = "Deleting existing release..."; await github.deleteReleaseAndAssets(tag); - const contentHashPath = writeContentHashToFile({ - buildDir, - releaseMultiHash + const contentHashPaths = await handleReleaseVariantFiles({ + releaseDetailsMap, + composeFileName }); - compactManifestIfCore(buildDir); - - composeDeleteBuildProperties({ dir: buildDir, composeFileName }); - - if (txData) { - task.output = `Creating release for tag ${tag}...`; - await github.createRelease(tag, { - body: getReleaseBody(txData), - prerelease: true // Until it is actually published to mainnet - }); - } + task.output = `Creating release for tag ${tag}...`; + await github.createRelease(tag, { + body: await getReleaseBody({ releaseDetailsMap }), + prerelease: true, // Until it is actually published to mainnet + }); // Clean content hash file so the directory uploaded to IPFS is the same // as the local build_* dir. User can then `ipfs add -r` and get the same hash - fs.unlinkSync(contentHashPath); + contentHashPaths.map(contentHashPath => fs.unlinkSync(contentHashPath)); } }; } +async function handleReleaseVariantFiles({ + releaseDetailsMap, + composeFileName +}: { + releaseDetailsMap: ReleaseDetailsMap; + composeFileName?: string; +}): Promise { + const contentHashPaths: string[] = []; + + for (const [, { variant, releaseDir, releaseMultiHash }] of Object.entries( + releaseDetailsMap + )) { + if (!releaseMultiHash) { + throw new Error( + `Release hash not found for variant ${variant} of ${name}` + ); + } + + const contentHashPath = writeContentHashToFile({ + releaseDir, + releaseMultiHash + }); + + contentHashPaths.push(contentHashPath); + + compactManifestIfCore(releaseDir); + composeDeleteBuildProperties({ dir: releaseDir, composeFileName }); + } + + return contentHashPaths; +} + /** * Plain text file which should contain the IPFS hash of the release * Necessary for the installer script to fetch the latest content hash @@ -75,13 +95,13 @@ export function getCreateReleaseTask({ * to install an eth client when the user does not want to use a remote node */ function writeContentHashToFile({ - buildDir, + releaseDir, releaseMultiHash }: { - buildDir: string; + releaseDir: string; releaseMultiHash: string; }): string { - const contentHashPath = path.join(buildDir, contentHashFile); + const contentHashPath = path.join(releaseDir, contentHashFileName); fs.writeFileSync(contentHashPath, releaseMultiHash); return contentHashPath; } @@ -89,39 +109,89 @@ function writeContentHashToFile({ /** * Write the release body * - * TODO: Extend this to automatically write the body of the changelog */ -function getReleaseBody(txData: TxData) { - const link = getPublishTxLink(txData); - const changelog = ""; - const installLink = getInstallDnpLink(txData.releaseMultiHash); +async function getReleaseBody({ + releaseDetailsMap, +}: { + releaseDetailsMap: ReleaseDetailsMap; +}) { + return ` + ## Package versions + + ${getPackageVersionsTable(releaseDetailsMap)} + + `.trim(); +} + +function getPackageVersionsTable(releaseDetailsMap: ReleaseDetailsMap) { + return ` + Package | Version | Hash | Install | Publish + --- | --- | --- | --- | --- + ${Object.entries(releaseDetailsMap) + .map(([dnpName, { nextVersion, releaseMultiHash, txData }]) => + getPackageVersionsRow({ + dnpName, + nextVersion, + releaseMultiHash, + txData + }) + ) + .join("\n")} + `.trim(); +} + +function getPackageVersionsRow({ + dnpName, + nextVersion, + releaseMultiHash, + txData +}: { + dnpName: string; + nextVersion: string; + releaseMultiHash: string; + txData: TxData; +}): string { + const prettyDnp = prettyDnpName(dnpName); + return ` - ##### Changelog - - ${changelog} - - --- - - ##### For package mantainer - - Authorized developer account may execute this transaction [from a pre-filled link](${link})[.](${installLink}) - -
Release details -

- - \`\`\` - To: ${txData.to} - Value: ${txData.value} - Data: ${txData.data} - Gas limit: ${txData.gasLimit} - \`\`\` - - \`\`\` - ${txData.releaseMultiHash} - \`\`\` - -

-
- + ${prettyDnp} | ${nextVersion} | \`${releaseMultiHash}\` | [:inbox_tray:](${getInstallDnpLink( + releaseMultiHash + )}) | [:mega:](${getPublishTxLink(txData)}) `.trim(); } + +/** + * Pretifies a ENS name + * "bitcoin.dnp.dappnode.eth" => "Bitcoin" + * "raiden-testnet.dnp.dappnode.eth" => "Raiden Testnet" + * + * TODO: This is duplicated from dappmanager + * + * @param name ENS name + * @returns pretty name + */ +function prettyDnpName(dnpName: string): string { + if (!dnpName || typeof dnpName !== "string") return dnpName; + return ( + dnpName + .split(".")[0] + // Convert all "-" and "_" to spaces + .replace(new RegExp("-", "g"), " ") + .replace(new RegExp("_", "g"), " ") + .split(" ") + .map(capitalize) + .join(" ") + ); +} + +/** + * Capitalizes a string + * @param string = "hello world" + * @returns "Hello world" + * + * TODO: This is duplicated from dappmanager + */ +function capitalize(s: string): string { + if (!s || typeof s !== "string") return s; + return s.charAt(0).toUpperCase() + s.slice(1); +} diff --git a/src/tasks/createGithubRelease/subtasks/getHandleTagsTask.ts b/src/tasks/createGithubRelease/subtasks/getHandleTagsTask.ts index 80d16680..5e1ebc5f 100644 --- a/src/tasks/createGithubRelease/subtasks/getHandleTagsTask.ts +++ b/src/tasks/createGithubRelease/subtasks/getHandleTagsTask.ts @@ -2,7 +2,8 @@ import { getGitHead } from "../../../utils/git.js"; import { ListrContextPublish } from "../../../types.js"; import { ListrTask } from "listr"; import { Github } from "../../../providers/github/Github.js"; -import { getNextGitTag } from "./getNextGitTag.js"; +import { getNextGitTag } from "../getNextGitTag.js"; +import { buildReleaseDetailsMap } from "../buildReleaseDetailsMap.js"; /** * Handle tags * @@ -23,13 +24,12 @@ export function getHandleTagsTask({ return { title: `Handle tags`, task: async (ctx, task) => { - // TODO: Do this for each release - const [, { nextVersion }] = Object.entries(ctx)[0]; - // Sanity check, make sure repo exists await github.assertRepoExists(); - const tag = getNextGitTag(nextVersion); + const releaseDetailsMap = buildReleaseDetailsMap(ctx); + + const tag = getNextGitTag(releaseDetailsMap); // If the release is triggered in CI, // the trigger tag must be removed ("release/patch") diff --git a/src/tasks/createGithubRelease/subtasks/getNextGitTag.ts b/src/tasks/createGithubRelease/subtasks/getNextGitTag.ts deleted file mode 100644 index a82941cf..00000000 --- a/src/tasks/createGithubRelease/subtasks/getNextGitTag.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Returns the next git tag based on the next version defined in the context - */ -export function getNextGitTag(nextVersion?: string): string { - if (!nextVersion) throw Error("Missing ctx.nextVersion"); - return `v${nextVersion}`; -} diff --git a/src/tasks/createGithubRelease/types.ts b/src/tasks/createGithubRelease/types.ts new file mode 100644 index 00000000..dc03fe36 --- /dev/null +++ b/src/tasks/createGithubRelease/types.ts @@ -0,0 +1,11 @@ +import { TxData } from "../../types.js"; + +export interface ReleaseDetailsMap { + [dnpName: string]: { + nextVersion: string; + releaseMultiHash: string; + txData: TxData; + releaseDir: string; + variant: string; + }; +} diff --git a/src/tasks/generatePublishTx/getRegistryAddressFromEns.ts b/src/tasks/generatePublishTxs/getRegistryAddressFromEns.ts similarity index 100% rename from src/tasks/generatePublishTx/getRegistryAddressFromEns.ts rename to src/tasks/generatePublishTxs/getRegistryAddressFromEns.ts diff --git a/src/tasks/generatePublishTx/index.ts b/src/tasks/generatePublishTxs/index.ts similarity index 71% rename from src/tasks/generatePublishTx/index.ts rename to src/tasks/generatePublishTxs/index.ts index a3cd530c..ed9573e2 100644 --- a/src/tasks/generatePublishTx/index.ts +++ b/src/tasks/generatePublishTxs/index.ts @@ -4,22 +4,21 @@ import { getEthereumUrl } from "../../utils/getEthereumUrl.js"; import { getPublishTxLink } from "../../utils/getLinks.js"; import { addReleaseTx } from "../../utils/releaseRecord.js"; import { defaultDir, YargsError } from "../../params.js"; -import { - CliGlobalOptions, - ListrContextPublish, - TxData -} from "../../types.js"; +import { BuildVariantsMap, CliGlobalOptions, ListrContextPublish, TxData } from "../../types.js"; import { ApmRepository } from "@dappnode/toolkit"; import registryAbi from "../../contracts/ApmRegistryAbi.json" assert { type: "json" }; import { semverToArray } from "../../utils/semverToArray.js"; import repoAbi from "../../contracts/RepoAbi.json" assert { type: "json" }; import { VerbosityOptions } from "../../commands/build/types.js"; -import { Manifest } from "@dappnode/types"; import { getRegistryAddressFromDnpName } from "./getRegistryAddressFromEns.js"; import { Repo } from "./types.js"; const isZeroAddress = (address: string): boolean => parseInt(address) === 0; +type GenerateTxVariantsMap = { + [K in keyof BuildVariantsMap]: Pick; +} + /** * Generates the transaction data necessary to publish the package. * It will check if the repository exists first: @@ -31,71 +30,61 @@ const isZeroAddress = (address: string): boolean => parseInt(address) === 0; * - Show it on screen */ -export function generatePublishTx({ - dir = defaultDir, - manifest, - releaseMultiHash, +export function generatePublishTxs({ + dir: rootDir = defaultDir, developerAddress, ethProvider, - verbosityOptions + verbosityOptions, + variantsMap }: { - manifest: Manifest; - releaseMultiHash: string; developerAddress?: string; ethProvider: string; verbosityOptions: VerbosityOptions; + variantsMap: GenerateTxVariantsMap; } & CliGlobalOptions): Listr { // Init APM instance const ethereumUrl = getEthereumUrl(ethProvider); const apm = new ApmRepository(ethereumUrl); - // Compute tx data - const contentURI = - "0x" + Buffer.from(releaseMultiHash, "utf8").toString("hex"); - const contractAddress = "0x0000000000000000000000000000000000000000"; - - const { name: dnpName, version } = manifest; - return new Listr( [ { - title: "Generate transaction", - task: async ctx => { - const repository = await getRepoContractIfExists({ - apm, - ensName: dnpName - }); + title: "Generate transactions", + task: async (ctx, task) => { + const contractAddress = "0x0000000000000000000000000000000000000000"; + + for (const [, { manifest }] of Object.entries(variantsMap)) { - const txData: TxData = repository - ? await getTxDataForExistingRepo({ - repository, - version, - contractAddress, - contentURI, - releaseMultiHash, - dnpName: dnpName - }) - : await getTxDataForNewRepo({ - dnpName: dnpName, - version, - contractAddress, - contentURI, - releaseMultiHash, - developerAddress, - ethereumUrl - }); + const { name: dnpName, version } = manifest; + const releaseMultiHash = ctx[dnpName]?.releaseMultiHash; - ctx[dnpName] = ctx[dnpName] || {}; - ctx[dnpName].txData = txData; + if (!releaseMultiHash) { + task.output += `No release hash found for ${dnpName}. Skipping it...\n`; + continue; + } - /** - * Write Tx data in a file for future reference - */ - addReleaseTx({ - dir, - version: manifest.version, - link: getPublishTxLink(txData) - }); + const txData = await buildTxData({ + apm, + contractAddress, + dnpName, + version, + releaseMultiHash, + developerAddress: developerAddress || "", + ethereumUrl + }); + + ctx[dnpName] = ctx[dnpName] || {}; + ctx[dnpName].txData = txData; + + /** + * Write Tx data in a file for future reference + */ + addReleaseTx({ + dir: rootDir, + version, + link: getPublishTxLink(txData) + }); + } } } ], @@ -103,6 +92,53 @@ export function generatePublishTx({ ); } +// Exported to test it +export async function buildTxData({ + apm, + contractAddress, + dnpName, + version, + releaseMultiHash, + developerAddress, + ethereumUrl +}: { + apm: ApmRepository; + contractAddress: string; + dnpName: string; + version: string; + releaseMultiHash: string; + developerAddress: string; + ethereumUrl: string; +}): Promise { + // Compute tx data + const contentURI = + "0x" + Buffer.from(releaseMultiHash, "utf8").toString("hex"); + + const repository = await getRepoContractIfExists({ + apm, + ensName: dnpName + }); + + return repository + ? await getTxDataForExistingRepo({ + repository, + version, + contractAddress, + contentURI, + releaseMultiHash, + dnpName: dnpName + }) + : await getTxDataForNewRepo({ + dnpName: dnpName, + version, + contractAddress, + contentURI, + releaseMultiHash, + developerAddress, + ethereumUrl + }); +} + async function getRepoContractIfExists({ apm, ensName diff --git a/src/tasks/generatePublishTx/types.ts b/src/tasks/generatePublishTxs/types.ts similarity index 100% rename from src/tasks/generatePublishTx/types.ts rename to src/tasks/generatePublishTxs/types.ts diff --git a/src/tasks/publish/index.ts b/src/tasks/publish/index.ts index c1fd5c90..067aa342 100644 --- a/src/tasks/publish/index.ts +++ b/src/tasks/publish/index.ts @@ -1,9 +1,9 @@ import { ListrTask } from "listr"; import { PublishOptions } from "./types.js"; import { ListrContextPublish } from "../../types.js"; -import { getFetchApmVersionTask } from "./subtasks/getFetchApmVersionTask.js"; +import { getFetchApmVersionsTask } from "./subtasks/getFetchApmVersionsTask.js"; import { getBuildAndUploadTask } from "./subtasks/getBuildAndUploadTask.js"; -import { getGenerateTxTask } from "./subtasks/getGenerateTxTask.js"; +import { getGenerateTxTask } from "./subtasks/getGenerateTxsTask.js"; import { getCreateGithubReleaseTask } from "./subtasks/getCreateGithubReleaseTask.js"; import { getVerifyEthConnectionTask } from "./subtasks/getVerifyEthConnectionTask.js"; @@ -19,11 +19,20 @@ export function publish({ deleteOldPins, developerAddress, githubRelease, - verbosityOptions + verbosityOptions, + variantsDirPath, + variantsMap }: PublishOptions): ListrTask[] { return [ getVerifyEthConnectionTask({ ethProvider }), - getFetchApmVersionTask({ releaseType, ethProvider, dir, composeFileName }), + getFetchApmVersionsTask({ + releaseType, + ethProvider, + rootDir: dir, + variantsDirPath, + composeFileName, + variantsMap + }), getBuildAndUploadTask({ buildOptions: { dir, @@ -32,8 +41,9 @@ export function publish({ uploadTo, userTimeout, requireGitData, - deleteOldPins - // TODO: Add multi-variant package build options here + deleteOldPins, + variantsMap, + variantsDirPath }, verbosityOptions }), @@ -42,7 +52,8 @@ export function publish({ composeFileName, developerAddress, ethProvider, - verbosityOptions + verbosityOptions, + variantsMap }), getCreateGithubReleaseTask({ dir, diff --git a/src/tasks/publish/subtasks/getCreateGithubReleaseTask.ts b/src/tasks/publish/subtasks/getCreateGithubReleaseTask.ts index ce3e78c5..503683e7 100644 --- a/src/tasks/publish/subtasks/getCreateGithubReleaseTask.ts +++ b/src/tasks/publish/subtasks/getCreateGithubReleaseTask.ts @@ -16,23 +16,12 @@ export function getCreateGithubReleaseTask({ }): ListrTask { return { title: "Release on github", - enabled: () => githubRelease, // Only create release if requested - task: (ctx, task) => { - // TODO: Generate 1 tx per package - const [, { releaseMultiHash, releaseDir }] = Object.entries(ctx)[0]; - - if (releaseMultiHash && releaseDir) { - return createGithubRelease({ - dir, - compose_file_name: composeFileName, - buildDir: releaseDir, - releaseMultiHash, - verbosityOptions - }); - } else { - // TODO: Check if this is the correct way to handle this case - task.output = "No release hash found. Skipping release creation."; - } - } + skip: () => !githubRelease, + task: () => + createGithubRelease({ + dir, + compose_file_name: composeFileName, + verbosityOptions + }) }; } diff --git a/src/tasks/publish/subtasks/getFetchApmVersionsTask.ts b/src/tasks/publish/subtasks/getFetchApmVersionsTask.ts new file mode 100644 index 00000000..fbb9a634 --- /dev/null +++ b/src/tasks/publish/subtasks/getFetchApmVersionsTask.ts @@ -0,0 +1,137 @@ +import { ListrTask } from "listr"; +import { BuildVariantsMap, ListrContextPublish, ReleaseType } from "../../../types.js"; +import { + writeManifest, + readCompose, + updateComposeImageTags, + writeCompose, + readManifest +} from "../../../files/index.js"; +import { getNextVersionFromApm } from "../../../utils/versions/getNextVersionFromApm.js"; +import path from "path"; + +export function getFetchApmVersionsTask({ + releaseType, + ethProvider, + rootDir, + variantsDirPath, + composeFileName, + variantsMap +}: { + releaseType: ReleaseType; + ethProvider: string; + rootDir: string; + variantsDirPath: string; + composeFileName: string; + variantsMap: BuildVariantsMap; +}): ListrTask { + return { + title: "Fetch current versions from APM", + task: async ctx => { + for (const [ + variant, + { + manifest: { name, version } + } + ] of Object.entries(variantsMap)) + await setNextVersionToContext({ + ctx, + releaseType, + ethProvider, + dir: rootDir, + variantsDirPath, + composeFileName, + variant: variant === "default" ? null : variant, + name, + version + }); + } + }; +} + +async function setNextVersionToContext({ + ctx, + releaseType, + ethProvider, + dir, + variantsDirPath, + composeFileName, + variant, + name, + version +}: { + ctx: ListrContextPublish; + releaseType: ReleaseType; + ethProvider: string; + dir: string; + variantsDirPath: string; + composeFileName: string; + variant: string | null; + name: string; + version: string; +}): Promise { + ctx[name] = ctx[name] || {}; + + try { + ctx[name].nextVersion = await increaseFromApmVersion({ + type: releaseType, + ethProvider, + dir, + composeFileName, + variant, + variantsDirPath, + ensName: name + }); + } catch (e) { + if (e.message.includes("NOREPO")) ctx[name].nextVersion = version; + else throw e; + } +} + +// TODO: Try to test this without exporting the function (not used anywhere else) +export async function increaseFromApmVersion({ + type, + ethProvider, + dir, + composeFileName, + variant, + variantsDirPath, + ensName +}: { + type: ReleaseType; + ethProvider: string; + dir: string; + composeFileName: string; + variant: string | null; + variantsDirPath: string; + ensName: string; +}): Promise { + const variantDir = variant ? path.join(variantsDirPath, variant) : dir; + + // Check variables + const nextVersion = await getNextVersionFromApm({ + type, + ethProvider, + ensName + }); + + const { manifest, format } = readManifest([{ dir: variantDir }]); + + // Increase the version + manifest.version = nextVersion; + + // Modify and write the manifest and docker-compose + writeManifest(manifest, format, { dir: variantDir }); + + const compose = readCompose([{ dir: variantDir, composeFileName }]); + const newCompose = updateComposeImageTags(compose, { + name: ensName, + version: nextVersion + }); + writeCompose(newCompose, { + dir: variantDir, + composeFileName + }); + + return nextVersion; +} diff --git a/src/tasks/publish/subtasks/getGenerateTxTask.ts b/src/tasks/publish/subtasks/getGenerateTxTask.ts deleted file mode 100644 index 9dd74822..00000000 --- a/src/tasks/publish/subtasks/getGenerateTxTask.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { ListrTask } from "listr"; -import { VerbosityOptions } from "../../../commands/build/types.js"; -import { ListrContextPublish } from "../../../types.js"; -import { generatePublishTx } from "../../generatePublishTx/index.js"; -import { readManifest } from "../../../files/index.js"; - -export function getGenerateTxTask({ - dir, - composeFileName, - developerAddress, - ethProvider, - verbosityOptions -}: { - dir: string; - composeFileName: string; - developerAddress?: string; - ethProvider: string; - verbosityOptions: VerbosityOptions; -}): ListrTask { - return { - title: "Generate transaction", - task: (ctx, task) => { - const releaseHashes = Object.entries(ctx).map( - ([, { releaseMultiHash }]) => releaseMultiHash - ); - - if (releaseHashes.length < 1) - throw new Error( - "Could not get release hash from previous task while trying to generate the publish tx." - ); - - // TODO: Generate 1 tx per package - const releaseMultiHash = releaseHashes[0]; - - const { manifest } = readManifest([{ dir }]); - - if (releaseMultiHash) { - return generatePublishTx({ - dir, - compose_file_name: composeFileName, - releaseMultiHash, - developerAddress, - ethProvider, - verbosityOptions, - manifest - }); - } else { - task.output = "No release hash found. Skipping transaction generation."; - } - } - }; -} diff --git a/src/tasks/publish/subtasks/getGenerateTxsTask.ts b/src/tasks/publish/subtasks/getGenerateTxsTask.ts new file mode 100644 index 00000000..5ffe70cb --- /dev/null +++ b/src/tasks/publish/subtasks/getGenerateTxsTask.ts @@ -0,0 +1,34 @@ +import { ListrTask } from "listr"; +import { VerbosityOptions } from "../../../commands/build/types.js"; +import { BuildVariantsMap, ListrContextPublish } from "../../../types.js"; +import { generatePublishTxs } from "../../generatePublishTxs/index.js"; + +export function getGenerateTxTask({ + dir, + composeFileName, + developerAddress, + ethProvider, + verbosityOptions, + variantsMap +}: { + dir: string; + composeFileName: string; + developerAddress?: string; + ethProvider: string; + verbosityOptions: VerbosityOptions; + variantsMap: BuildVariantsMap; +}): ListrTask { + return { + title: "Generate transaction", + task: () => { + return generatePublishTxs({ + dir, + compose_file_name: composeFileName, + developerAddress, + ethProvider, + verbosityOptions, + variantsMap + }); + } + }; +} diff --git a/src/tasks/publish/types.ts b/src/tasks/publish/types.ts index 612c85d8..21bfa357 100644 --- a/src/tasks/publish/types.ts +++ b/src/tasks/publish/types.ts @@ -1,6 +1,6 @@ import { VerbosityOptions } from "../../commands/build/types.js"; import { UploadTo } from "../../releaseUploader/index.js"; -import { ReleaseType } from "../../types.js"; +import { BuildVariantsMap, ReleaseType } from "../../types.js"; export interface PublishOptions { releaseType: ReleaseType; @@ -15,4 +15,6 @@ export interface PublishOptions { developerAddress?: string; githubRelease?: boolean; verbosityOptions: VerbosityOptions; + variantsDirPath: string; + variantsMap: BuildVariantsMap; } diff --git a/src/types.ts b/src/types.ts index 0e973cd9..21409dfe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,5 @@ -// CLI types +import { Architecture, Compose, Manifest } from "@dappnode/types"; +import { ManifestFormat } from "./files/manifest/types.js"; export interface CliGlobalOptions { dir?: string; @@ -26,6 +27,29 @@ export interface ListrContextPublish { [dnpName: string]: ListrContextPublishItem; } +interface PublishVariantsMapEntry { + // Manifest-related + manifest: Manifest; + manifestFormat: ManifestFormat; +} + +export interface BuildVariantsMapEntry extends PublishVariantsMapEntry { + // Compose file + compose: Compose; + + // File paths + releaseDir: string; + composePaths: string[]; + + // Package information + images: PackageImage[]; + architectures: Architecture[]; +} + +export interface BuildVariantsMap { + [variant: string]: BuildVariantsMapEntry; +} + // Interal types export type ReleaseType = "major" | "minor" | "patch"; @@ -59,5 +83,3 @@ export interface TxDataShortKeys { h: string; // hash d?: string; // developerAddress } - -export const contentHashFile = "content-hash" as const; diff --git a/test/tasks/buildAndUpload/buildVariantMap.test.ts b/test/tasks/buildAndUpload/buildVariantMap.test.ts index b6e8a2a1..1b57c31c 100644 --- a/test/tasks/buildAndUpload/buildVariantMap.test.ts +++ b/test/tasks/buildAndUpload/buildVariantMap.test.ts @@ -27,7 +27,8 @@ describe("buildVariantMap", function () { it("should return a map with only default variant", function () { const result = buildVariantMap({ rootDir: testDir, - variantsDirPath: defaultVariantsDirName + variantsDirPath: defaultVariantsDirName, + variants: null }); expect(result).to.have.all.keys("default"); diff --git a/test/tasks/generatePublishTx.test.ts b/test/tasks/generatePublishTx.test.ts index bcbc17b1..b4dc72f9 100644 --- a/test/tasks/generatePublishTx.test.ts +++ b/test/tasks/generatePublishTx.test.ts @@ -1,8 +1,10 @@ import { expect } from "chai"; import { defaultManifestFormat } from "../../src/params.js"; -import { generatePublishTx } from "../../src/tasks/generatePublishTx/index.js"; +import { buildTxData } from "../../src/tasks/generatePublishTxs/index.js"; import { writeManifest } from "../../src/files/index.js"; import { testDir, cleanTestDir } from "../testUtils.js"; +import { getEthereumUrl } from "../../src/utils/getEthereumUrl.js"; +import { ApmRepository } from "@dappnode/toolkit"; // This test will create the following fake files // ./dappnode_package.json => fake manifest @@ -11,9 +13,14 @@ import { testDir, cleanTestDir } from "../testUtils.js"; // Then it will expect the function to generate transaction data // and output it to the console and to ./dnp_0.0.0/deploy.txt -describe("generatePublishTx", function () { +describe("generatePublishTx", async function () { this.timeout(60 * 1000); + const burnAddress = "0x0000000000000000000000000000000000000000"; + const ethProvider = "infura"; + const ethereumUrl = getEthereumUrl(ethProvider); + const apm = new ApmRepository(ethereumUrl); + before("Clean testDir", () => cleanTestDir()); after("Clean testDir", () => cleanTestDir()); @@ -24,34 +31,30 @@ describe("generatePublishTx", function () { }; writeManifest(manifest, defaultManifestFormat, { dir: testDir }); - const generatePublishTxTasks = generatePublishTx({ - dir: testDir, - manifest, - releaseMultiHash: "/ipfs/Qm", + const { name, version } = manifest; + + const txData = await buildTxData({ + apm, + contractAddress: burnAddress, developerAddress: "0xAb5801a7D398351b8bE11C439e05C5B3259aeC9B", - ethProvider: "infura", - verbosityOptions: { renderer: "verbose" } + dnpName: name, + releaseMultiHash: "/ipfs/Qm", + version: version, + ethereumUrl }); - // TODO: Fix when publish is adapted to multi-variant packages - const generateTxResults = await generatePublishTxTasks.run(); - const { txData } = generateTxResults[manifest.name]; - expect(txData).to.be.an("object"); - // admin.dnp.dappnode.eth ==> 0xEe66C4765696C922078e8670aA9E6d4F6fFcc455 expect(txData).to.deep.equal({ - to: "0xEe66C4765696C922078e8670aA9E6d4F6fFcc455", + to: "0xEe66C4765696C922078e8670aA9E6d4F6fFcc455", // admin.dnp.dappnode.eth resolves to this address value: 0, data: "0x73053410000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000082f697066732f516d000000000000000000000000000000000000000000000000", gasLimit: 300000, - ensName: "admin.dnp.dappnode.eth", - currentVersion: "0.1.0", + ensName: name, + currentVersion: version, releaseMultiHash: "/ipfs/Qm" }); - // I am not sure if the Data property will be the same - expect(txData?.data).to.be.a("string"); }); it("Should generate a publish TX", async function () { @@ -61,32 +64,30 @@ describe("generatePublishTx", function () { }; writeManifest(manifest, defaultManifestFormat, { dir: testDir }); - const generatePublishTxTasks = generatePublishTx({ - dir: testDir, - manifest, - releaseMultiHash: "/ipfs/Qm", + const { name, version } = manifest; + + const txData = await buildTxData({ + apm, + contractAddress: burnAddress, developerAddress: "0xAb5801a7D398351b8bE11C439e05C5B3259aeC9B", - ethProvider: "infura", - verbosityOptions: { renderer: "verbose" } + dnpName: name, + releaseMultiHash: "/ipfs/Qm", + version: version, + ethereumUrl }); - const publishResult = await generatePublishTxTasks.run(); - - const { txData } = publishResult[manifest.name]; expect(txData).to.be.an("object"); expect(txData).to.deep.equal({ - to: "0x266BFdb2124A68beB6769dC887BD655f78778923", + to: "0x266BFdb2124A68beB6769dC887BD655f78778923", // Registry contract address value: 0, data: "0x32ab6af000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000ab5801a7d398351b8be11c439e05c5b3259aec9b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000000086e65772d7265706f00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000082f697066732f516d000000000000000000000000000000000000000000000000", gasLimit: 1100000, - ensName: "new-repo.dnp.dappnode.eth", - currentVersion: "0.1.0", + ensName: name, + currentVersion: version, releaseMultiHash: "/ipfs/Qm", developerAddress: "0xAb5801a7D398351b8bE11C439e05C5B3259aeC9B" }); - // I am not sure if the Data property will be the same - expect(txData?.data).to.be.a("string"); }); }); diff --git a/test/tasks/gitTags.test.ts b/test/tasks/gitTags.test.ts new file mode 100644 index 00000000..2be13979 --- /dev/null +++ b/test/tasks/gitTags.test.ts @@ -0,0 +1,37 @@ +// Import the necessary modules +import { expect } from "chai"; +import { describe, it } from "mocha"; +import { getNextGitTag } from "../../src/tasks/createGithubRelease/getNextGitTag.js"; + +// Describe your test suite +describe("getNextGitTag", () => { + it("should format a single-variant package correctly", () => { + const ctx = { + "geth.dnp.dapnode.eth": { + variant: "mainnet", + nextVersion: "0.1.2" + } + }; + const result = getNextGitTag(ctx); + expect(result).to.equal("v0.1.2"); + }); + + it("should format and sort a multi-variant package correctly", () => { + const ctx = { + "geth.dnp.dapnode.eth": { + variant: "mainnet", + nextVersion: "0.1.3" + }, + "goerli-geth.dnp.dapnode.eth": { + variant: "goerli", + nextVersion: "0.1.2" + }, + "holesky-geth.dnp.dapnode.eth": { + variant: "holesky", + nextVersion: "0.1.1" + } + }; + const result = getNextGitTag(ctx); + expect(result).to.equal("goerli@0.1.2_holesky@0.1.1_mainnet@0.1.3"); + }); +});