From b3ee50e0a60ae132f1ca086aaadfac21139a1834 Mon Sep 17 00:00:00 2001 From: Lenin Mehedy Date: Tue, 5 Mar 2024 12:55:54 +1100 Subject: [PATCH 1/2] fix: do checksum check when downloading and extract platform zip file in the container Signed-off-by: Lenin Mehedy --- resources/extract-jar.sh | 16 ------- resources/extract-platform.sh | 39 +++++++++++++++++ src/commands/node.mjs | 2 +- src/core/k8.mjs | 8 +++- src/core/platform_installer.mjs | 40 ++++-------------- test/e2e/core/platform_installer_e2e.test.mjs | 42 +++++++++++-------- 6 files changed, 78 insertions(+), 69 deletions(-) delete mode 100644 resources/extract-jar.sh create mode 100755 resources/extract-platform.sh diff --git a/resources/extract-jar.sh b/resources/extract-jar.sh deleted file mode 100644 index 9237bab8f..000000000 --- a/resources/extract-jar.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Extract a jar file to a destination directory -# Usage: ./extract.sh - -if [ -z "${1}" ]; then - echo "jarFile arg is required"; - exit 1 -fi - -if [ -z "${2}" ]; then - echo "destDir arg is required"; - exit 1 -fi - -[[ -d "${2}" ]] || mkdir -p "${2}" -cd "${2}" && jar xvf "${1}" diff --git a/resources/extract-platform.sh b/resources/extract-platform.sh new file mode 100755 index 000000000..99a81a55d --- /dev/null +++ b/resources/extract-platform.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# This script fetch the build.zip file and checksum file from builds.hedera.com and then extract it into HapiApp2 directory +# Usage extract-platform +# e.g. extract-platform v0.42.5 + +readonly tag="${1}" +if [ -z "${tag}" ]; then + echo "Release tag is required (e.g. v0.42.5)"; + exit 1 +fi + +readonly HAPI_DIR=/opt/hgcapp/services-hedera/HapiApp2.0 +readonly HEDERA_USER_HOME_DIR=/home/hedera +readonly HEDERA_BUILDS_URL='https://builds.hedera.com' +readonly RELEASE_DIR="$(awk -F'.' '{print $1"."$2}' <<< "${tag}")" +readonly BUILD_ZIP_FILE="${HEDERA_USER_HOME_DIR}/build-${tag}.zip" +readonly BUILD_ZIP_URL="${HEDERA_BUILDS_URL}/node/software/${RELEASE_DIR}/build-${tag}.zip" +readonly CHECKSUM_FILE="${HEDERA_USER_HOME_DIR}/build-${tag}.sha384" +readonly CHECKSUM_URL="${HEDERA_BUILDS_URL}/node/software/${RELEASE_DIR}/build-${tag}.sha384" + +# download +echo "Downloading ${BUILD_ZIP_URL}" +[ -f "${BUILD_ZIP_FILE}" ] || curl -sSf "${BUILD_ZIP_URL}" -o "${BUILD_ZIP_FILE}" +[ $? == 0 ] || exit 1 +echo "Downloading ${CHECKSUM_URL}" +[ -f "${CHECKSUM_FILE}" ] || curl -sSf "${CHECKSUM_URL}" -o "${CHECKSUM_FILE}" +[ $? == 0 ] || exit 1 +readonly sum="$(openssl dgst -sha384 ${BUILD_ZIP_FILE} | awk '{print $2}')" +readonly expected_sum="$(awk '{print $1}' < "${CHECKSUM_FILE}")" +if [ "${sum}" != "${expected_sum}" ]; then + echo "SHA sum of ${BUILD_ZIP_FILE} does not match. Aborting." + exit 1 +fi + +# extract +echo "Extracting ${BUILD_ZIP_FILE}" +[[ -d "${HAPI_DIR}" ]] || mkdir -p "${HAPI_DIR}" +cd "${HAPI_DIR}" && jar xvf "${BUILD_ZIP_FILE}" +[ $? == 0 ] || exit 1 diff --git a/src/commands/node.mjs b/src/commands/node.mjs index 15f8441c8..a8c7fca8a 100644 --- a/src/commands/node.mjs +++ b/src/commands/node.mjs @@ -357,7 +357,7 @@ export class NodeCommand extends BaseCommand { // set up the sub-tasks return task.newListr(subTasks, { - concurrent: false, // parallel uploading of the zip file seems to be unreliable, so we just upload in sequence + concurrent: true, // since we download in the container directly, we want this to be in parallel across all nodes rendererOptions: { collapseSubtasks: false } diff --git a/src/core/k8.mjs b/src/core/k8.mjs index 74023666e..04c832be1 100644 --- a/src/core/k8.mjs +++ b/src/core/k8.mjs @@ -20,7 +20,7 @@ import net from 'net' import os from 'os' import path from 'path' import { flags } from '../commands/index.mjs' -import { FullstackTestingError, MissingArgumentError } from './errors.mjs' +import { FullstackTestingError, IllegalArgumentError, MissingArgumentError } from './errors.mjs' import * as sb from 'stream-buffers' import * as tar from 'tar' import { v4 as uuid4 } from 'uuid' @@ -472,7 +472,10 @@ export class K8 { // zip the source file const tmpFile = this._tempFileFor(srcFile) - await tar.c({ file: tmpFile, cwd: srcDir }, [srcFile]) + await tar.c({ + file: tmpFile, + cwd: srcDir + }, [srcFile]) const self = this return new Promise((resolve, reject) => { @@ -614,6 +617,7 @@ export class K8 { if (!Array.isArray(command)) { command = command.split(' ') } + if (!await this.getPodByName(podName)) throw new IllegalArgumentError(`Invalid pod ${podName}`) const self = this return new Promise((resolve, reject) => { diff --git a/src/core/platform_installer.mjs b/src/core/platform_installer.mjs index bfd9476fe..c29c156a0 100644 --- a/src/core/platform_installer.mjs +++ b/src/core/platform_installer.mjs @@ -100,50 +100,24 @@ export class PlatformInstaller { } /** - * Fetch platform code into the container - * - * It uses curl to fetch the platform code directly inside the /home/hedera directory. - * + * Fetch and extract platform code into the container * @param podName pod name * @param tag platform release tag * @return {Promise} */ + async fetchPlatform (podName, tag) { if (!podName) throw new MissingArgumentError('podName is required') if (!tag) throw new MissingArgumentError('tag is required') try { - const releaseDir = Templates.prepareReleasePrefix(tag) - const packageURL = `${constants.HEDERA_BUILDS_URL}/node/software/${releaseDir}/build-${tag}.zip` - const buildZip = path.join(constants.HEDERA_USER_HOME_DIR, `build-${tag}.zip`) - await this.k8.execContainer(podName, constants.ROOT_CONTAINER, `curl -s ${packageURL} -o ${buildZip}`) - return this.extractPlatform(podName, buildZip) - } catch (e) { - throw new FullstackTestingError(`failed to copy platform code in to pod '${podName}': ${e.message}`, e) - } - } + const scriptName = 'extract-platform.sh' + const sourcePath = path.join(constants.RESOURCES_DIR, scriptName) // script source path + await this.copyFiles(podName, [sourcePath], constants.HEDERA_USER_HOME_DIR) - async extractPlatform (podName, buildZipSrc) { - if (!podName) throw new MissingArgumentError('podName is required') - if (!buildZipSrc) throw new MissingArgumentError('buildZipSrc is required') - - const buildZipFileName = path.basename(buildZipSrc) - const buildZip = path.join(constants.HEDERA_USER_HOME_DIR, buildZipFileName) // inside the container - const extractScriptName = 'extract-jar.sh' - const extractScriptSrc = path.join(constants.RESOURCES_DIR, extractScriptName) - const extractScript = path.join(constants.HEDERA_USER_HOME_DIR, extractScriptName) // inside the container - - this.logger.debug(`Extracting platform code in pod ${podName}`, { - extractScript, - buildZip, - dest: constants.HEDERA_HAPI_PATH - }) - - try { - await this.copyFiles(podName, [extractScriptSrc], constants.HEDERA_USER_HOME_DIR) + const extractScript = path.join(constants.HEDERA_USER_HOME_DIR, scriptName) // inside the container await this.k8.execContainer(podName, constants.ROOT_CONTAINER, `chmod +x ${extractScript}`) - await this.k8.execContainer(podName, constants.ROOT_CONTAINER, [extractScript, buildZip, constants.HEDERA_HAPI_PATH]) - + await this.k8.execContainer(podName, constants.ROOT_CONTAINER, [extractScript, tag]) return true } catch (e) { throw new FullstackTestingError(`failed to extract platform code in this pod '${podName}': ${e.message}`, e) diff --git a/test/e2e/core/platform_installer_e2e.test.mjs b/test/e2e/core/platform_installer_e2e.test.mjs index 929de289a..75ead3961 100644 --- a/test/e2e/core/platform_installer_e2e.test.mjs +++ b/test/e2e/core/platform_installer_e2e.test.mjs @@ -16,7 +16,6 @@ */ import { beforeAll, describe, expect, it } from '@jest/globals' import { - PackageDownloader, PlatformInstaller, constants, Templates, @@ -33,7 +32,6 @@ describe('PackageInstallerE2E', () => { const configManager = new ConfigManager(testLogger) const k8 = new K8(configManager, testLogger) const installer = new PlatformInstaller(testLogger, k8) - const downloader = new PackageDownloader(testLogger) const testCacheDir = getTestCacheDir() const podName = 'network-node0-0' const packageVersion = 'v0.42.5' @@ -58,28 +56,38 @@ describe('PackageInstallerE2E', () => { }) }) - describe('copyPlatform', () => { - it('should succeed fetching platform release', async () => { - const releasePrefix = Templates.prepareReleasePrefix(packageVersion) - const destPath = `${testCacheDir}/${releasePrefix}/build-${packageVersion}.zip` - await expect(downloader.fetchPlatform(packageVersion, testCacheDir)).resolves.toBe(destPath) - expect(fs.existsSync(destPath)).toBeTruthy() - testLogger.showUser(destPath) + describe('fetchPlatform', () => { + it('should fail with invalid pod', async () => { + expect.assertions(2) + try { + await installer.fetchPlatform('', packageVersion) + } catch (e) { + expect(e.message.includes('podName is required')).toBeTruthy() + } - // do not delete the cache dir - }, 200000) + try { + await installer.fetchPlatform('INVALID', packageVersion) + } catch (e) { + expect(e.message + .includes('failed to extract platform code in this pod')) + .toBeTruthy() + } + }, 20000) - it('should succeed with valid tag and pod', async () => { + it('should fail with invalid tag', async () => { expect.assertions(1) try { - await expect(installer.fetchPlatform(podName, packageVersion)).resolves.toBeTruthy() - const outputs = await k8.execContainer(podName, constants.ROOT_CONTAINER, `ls -la ${constants.HEDERA_HAPI_PATH}`) - testLogger.showUser(outputs) + await installer.fetchPlatform(podName, 'INVALID') } catch (e) { - console.error(e) - expect(e).toBeNull() + expect(e.message.includes('curl: (22) The requested URL returned error: 404')).toBeTruthy() } }, 20000) + + it('should succeed with valid tag and pod', async () => { + await expect(installer.fetchPlatform(podName, packageVersion)).resolves.toBeTruthy() + const outputs = await k8.execContainer(podName, constants.ROOT_CONTAINER, `ls -la ${constants.HEDERA_HAPI_PATH}`) + testLogger.showUser(outputs) + }, 60000) }) describe('prepareConfigTxt', () => { From 6cda7e16a099799011239847ef82bb1fa80f335a Mon Sep 17 00:00:00 2001 From: Lenin Mehedy Date: Tue, 5 Mar 2024 13:08:33 +1100 Subject: [PATCH 2/2] style: fix codacy errors Signed-off-by: Lenin Mehedy --- resources/extract-platform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/extract-platform.sh b/resources/extract-platform.sh index 99a81a55d..205829ebb 100755 --- a/resources/extract-platform.sh +++ b/resources/extract-platform.sh @@ -25,7 +25,7 @@ echo "Downloading ${BUILD_ZIP_URL}" echo "Downloading ${CHECKSUM_URL}" [ -f "${CHECKSUM_FILE}" ] || curl -sSf "${CHECKSUM_URL}" -o "${CHECKSUM_FILE}" [ $? == 0 ] || exit 1 -readonly sum="$(openssl dgst -sha384 ${BUILD_ZIP_FILE} | awk '{print $2}')" +readonly sum="$(openssl dgst -sha384 "${BUILD_ZIP_FILE}" | awk '{print $2}')" readonly expected_sum="$(awk '{print $1}' < "${CHECKSUM_FILE}")" if [ "${sum}" != "${expected_sum}" ]; then echo "SHA sum of ${BUILD_ZIP_FILE} does not match. Aborting."