feat(tools): substrate test ledger

Substrate-based blockchains like Polkadot are becoming increasingly relevant.
However, Cactus lacked tools to support developing Substrate-based connectors,
such as a test ledger.

This commit defines a Substrate test ledger that allows to programmatically instantiate
the Substrate Contracts node template (https://github.com/paritytech/substrate-contracts-node).
Thus, it contains a Dockerfile with the test ledger
and the test-ledger file that contains the execution logic.

Signed-off-by: Rafael Belchior <[email protected]>
Signed-off-by: Catarina Pedreira <[email protected]>
Signed-off-by: Peter Somogyvari <[email protected]>

.cspell.json

@@ -21,6 +21,7 @@
         "cids",
         "Corda",
         "Cordapp",
+        "couchdb",
         "dclm",
         "DHTAPI",
         "DockerOde",

packages/cactus-test-tooling/package.json

@@ -51,6 +51,16 @@
       "email": "[email protected]",
       "url": "https://example.com"
     },
+    {
+      "name": "Catarina Pedreira",
+      "email": "[email protected]",
+      "url": "https://github.com/CatarinaPedreira"
+    },
+    {
+      "name": "Rafael Belchior",
+      "email": "[email protected]",
+      "url": "https://rafaelapb.github.io/"
+    },
     {
       "name": "Peter Somogyvari",
       "email": "[email protected]",

packages/cactus-test-tooling/src/main/typescript/public-api.ts

@@ -131,6 +131,11 @@ export {
   RustcContainer,
 } from "./rustc-container/rustc-container";
 
+export {
+  ISubstrateTestLedgerOptions,
+  SubstrateTestLedger,
+} from "./substrate-test-ledger/substrate-test-ledger";
+
 export { RustcBuildCmd } from "./rustc-container/rustc-build-cmd";
 
 export { Streams } from "./common/streams";

packages/cactus-test-tooling/src/main/typescript/substrate-test-ledger/substrate-test-ledger.ts

@@ -0,0 +1,202 @@
+import type { EventEmitter } from "events";
+import { Optional } from "typescript-optional";
+import { RuntimeError } from "run-time-error";
+import type { Container, ContainerInfo } from "dockerode";
+import Docker from "dockerode";
+import { Logger, Checks, Bools } from "@hyperledger/cactus-common";
+import type { LogLevelDesc } from "@hyperledger/cactus-common";
+import { LoggerProvider } from "@hyperledger/cactus-common";
+import { Containers } from "../common/containers";
+
+export interface ISubstrateTestLedgerOptions {
+  readonly publishAllPorts: boolean;
+  readonly logLevel?: LogLevelDesc;
+  readonly imageName?: string;
+  readonly imageTag?: string;
+  readonly emitContainerLogs?: boolean;
+  readonly envVars?: Map<string, string>;
+}
+
+export class SubstrateTestLedger {
+  public static readonly CLASS_NAME = "SubstrateTestLedger";
+
+  public readonly logLevel: LogLevelDesc;
+  public readonly imageName: string;
+  public readonly imageTag: string;
+  public readonly imageFqn: string;
+  public readonly log: Logger;
+  public readonly emitContainerLogs: boolean;
+  public readonly publishAllPorts: boolean;
+  public readonly envVars: Map<string, string>;
+
+  private _containerId: Optional<string>;
+
+  public get containerId(): Optional<string> {
+    return this._containerId;
+  }
+
+  public get container(): Optional<Container> {
+    const docker = new Docker();
+    return this.containerId.isPresent()
+      ? Optional.ofNonNull(docker.getContainer(this.containerId.get()))
+      : Optional.empty();
+  }
+
+  public get className(): string {
+    return SubstrateTestLedger.CLASS_NAME;
+  }
+
+  constructor(public readonly opts: ISubstrateTestLedgerOptions) {
+    const fnTag = `${this.className}#constructor()`;
+    Checks.truthy(opts, `${fnTag} arg options`);
+
+    this.publishAllPorts = opts.publishAllPorts;
+    this._containerId = Optional.empty();
+    this.imageName =
+      opts.imageName || "ghcr.io/hyperledger/cactus-substrate-all-in-one";
+    this.imageTag = opts.imageTag || "2021-09-24---feat-1274";
+    this.imageFqn = `${this.imageName}:${this.imageTag}`;
+    this.envVars = opts.envVars || new Map();
+    this.emitContainerLogs = Bools.isBooleanStrict(opts.emitContainerLogs)
+      ? (opts.emitContainerLogs as boolean)
+      : true;
+
+    this.logLevel = opts.logLevel || "INFO";
+
+    const level = this.logLevel;
+    const label = this.className;
+    this.log = LoggerProvider.getOrCreate({ level, label });
+
+    this.log.debug(`Created instance of ${this.className} OK`);
+  }
+  public getContainerImageName(): string {
+    return `${this.imageName}:${this.imageTag}`;
+  }
+  public async start(omitPull = false): Promise<Container> {
+    const docker = new Docker();
+    if (this.containerId.isPresent()) {
+      this.log.debug(`Container ID provided. Will not start new one.`);
+      const container = docker.getContainer(this.containerId.get());
+      return container;
+    }
+    if (!omitPull) {
+      this.log.debug(`Pulling image ${this.imageFqn}...`);
+      await Containers.pullImage(this.imageFqn);
+      this.log.debug(`Pulled image ${this.imageFqn} OK`);
+    }
+
+    const dockerEnvVars: string[] = new Array(...this.envVars).map(
+      (pairs) => `${pairs[0]}=${pairs[1]}`,
+    );
+
+    // TODO: dynamically expose ports for custom port mapping
+    const createOptions = {
+      Env: dockerEnvVars,
+      Healthcheck: {
+        Test: [
+          "CMD-SHELL",
+          `rustup --version && rustc --version && cargo --version`,
+        ],
+        Interval: 1000000000, // 1 second
+        Timeout: 3000000000, // 3 seconds
+        Retries: 10,
+        StartPeriod: 1000000000, // 1 second
+      },
+      ExposedPorts: {
+        "9944/tcp": {}, // OpenSSH Server - TCP
+      },
+      HostConfig: {
+        AutoRemove: true,
+        PublishAllPorts: this.publishAllPorts,
+        Privileged: false,
+        PortBindings: {
+          "9944/tcp": [{ HostPort: "9944" }],
+        },
+      },
+    };
+
+    this.log.debug(`Starting ${this.imageFqn} with options: `, createOptions);
+
+    return new Promise<Container>((resolve, reject) => {
+      const eventEmitter: EventEmitter = docker.run(
+        this.imageFqn,
+        [],
+        [],
+        createOptions,
+        {},
+        (err: Error) => {
+          if (err) {
+            const errorMessage = `Failed to start container ${this.imageFqn}`;
+            const exception = new RuntimeError(errorMessage, err);
+            this.log.error(exception);
+            reject(exception);
+          }
+        },
+      );
+
+      eventEmitter.once("start", async (container: Container) => {
+        const { id } = container;
+        this.log.debug(`Started ${this.imageFqn} successfully. ID=${id}`);
+        this._containerId = Optional.ofNonNull(id);
+
+        if (this.emitContainerLogs) {
+          const logOptions = { follow: true, stderr: true, stdout: true };
+          const logStream = await container.logs(logOptions);
+          logStream.on("data", (data: Buffer) => {
+            const fnTag = `[${this.imageFqn}]`;
+            this.log.debug(`${fnTag} %o`, data.toString("utf-8"));
+          });
+        }
+        this.log.debug(`Registered container log stream callbacks OK`);
+
+        try {
+          this.log.debug(`Starting to wait for healthcheck... `);
+          await Containers.waitForHealthCheck(this.containerId.get());
+          this.log.debug(`Healthcheck passed OK`);
+          resolve(container);
+        } catch (ex) {
+          this.log.error(ex);
+          reject(ex);
+        }
+      });
+    });
+  }
+
+  public async stop(): Promise<unknown> {
+    return Containers.stop(this.container.get());
+  }
+
+  public async destroy(): Promise<unknown> {
+    return this.container.get().remove();
+  }
+
+  public async getContainerIpAddress(): Promise<string> {
+    const containerInfo = await this.getContainerInfo();
+    return Containers.getContainerInternalIp(containerInfo);
+  }
+
+  protected async getContainerInfo(): Promise<ContainerInfo> {
+    const fnTag = "FabricTestLedgerV1#getContainerInfo()";
+    const docker = new Docker();
+    const image = this.getContainerImageName();
+    const containerInfos = await docker.listContainers({});
+
+    let aContainerInfo;
+    if (this.containerId !== undefined) {
+      aContainerInfo = containerInfos.find(
+        (ci) => ci.Id == this.containerId.toString(),
+      );
+    }
+
+    if (aContainerInfo) {
+      return aContainerInfo;
+    } else {
+      throw new Error(`${fnTag} no image "${image}"`);
+    }
+  }
+
+  // ./scripts/docker_run.sh ./target/release/node-template purge-chain --dev
+  protected async purgeDevChain(): Promise<void> {
+    throw new Error("TODO");
+  }
+}

packages/cactus-test-tooling/src/test/typescript/integration/substrate/substrate-test-ledger-constructor.test.ts

@@ -0,0 +1,40 @@
+import test, { Test } from "tape-promise/tape";
+import { LogLevelDesc } from "@hyperledger/cactus-common";
+import { SubstrateTestLedger } from "../../../../main/typescript/substrate-test-ledger/substrate-test-ledger";
+import { pruneDockerAllIfGithubAction } from "../../../../main/typescript/github-actions/prune-docker-all-if-github-action";
+
+const testCase = "Instantiate plugin";
+const logLevel: LogLevelDesc = "TRACE";
+
+test("BEFORE " + testCase, async (t: Test) => {
+  const pruning = pruneDockerAllIfGithubAction({ logLevel });
+  await t.doesNotReject(pruning, "Pruning didn't throw OK");
+  t.end();
+});
+
+test(testCase, async (t: Test) => {
+  const options = {
+    publishAllPorts: true,
+    logLevel: logLevel,
+    emitContainerLogs: true,
+    envVars: new Map([
+      ["WORKING_DIR", "/var/www/node-template"],
+      ["CONTAINER_NAME", "contracts-node-template-cactus"],
+      ["PORT", "9944"],
+      ["DOCKER_PORT", "9944"],
+      ["CARGO_HOME", "/var/www/node-template/.cargo"],
+    ]),
+  };
+
+  const ledger = new SubstrateTestLedger(options);
+  const tearDown = async () => {
+    await ledger.stop();
+    await pruneDockerAllIfGithubAction({ logLevel });
+  };
+
+  test.onFinish(tearDown);
+  await ledger.start();
+  t.ok(ledger);
+
+  t.end();
+});

tools/docker/substrate-all-in-one/Dockerfile

@@ -0,0 +1,39 @@
+FROM paritytech/ci-linux:production
+LABEL AUTHORS="Rafael Belchior, Catarina Pedreira" 
+LABEL VERSION="2021-09-10"
+LABEL org.opencontainers.image.source=https://github.com/hyperledger/cactus
+
+WORKDIR /
+ARG WORKING_DIR=/var/www/node-template
+ARG CONTAINER_NAME=contracts-node-template-cactus
+ARG PORT=9944
+ARG DOCKER_PORT=9944
+ARG CARGO_HOME=/var/www/node-template/.cargo
+
+ENV CARGO_HOME=${CARGO_HOME}
+ENV CACTUS_CFG_PATH=/etc/hyperledger/cactus
+VOLUME .:/var/www/node-template
+
+RUN apt update
+
+# Get ubuntu and rust packages
+RUN apt install -y build-essential pkg-config git clang curl libssl-dev llvm libudev-dev
+
+ENV CACTUS_CFG_PATH=/etc/hyperledger/cactus
+RUN mkdir -p $CACTUS_CFG_PATH
+
+RUN set -e
+
+RUN echo "*** Instaling Rust environment ***"
+RUN curl https://sh.rustup.rs -y -sSf | sh
+RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
+RUN rustup default nightly
+
+RUN echo "*** Initializing WASM build environment"
+RUN rustup target add wasm32-unknown-unknown --toolchain nightly
+
+RUN echo "*** Installing Substrate node environment ***"
+RUN cargo install contracts-node --git https://github.com/paritytech/substrate-contracts-node.git --force --locked
+
+RUN echo "*** Start Substrate node template ***"
+CMD [ "/var/www/node-template/.cargo/bin/substrate-contracts-node", "--dev"]

tools/docker/substrate-all-in-one/README.md

@@ -0,0 +1,20 @@
+# @hyperledger/cactus-substrate-all-in-one<!-- omit in toc -->
+
+A container image that can holds the default Substrate test ledger (and the corresponding front-end).
+This image can be used for development of Substrate-based chains (including but not limited to pallets, smart contracts) and connectors.
+
+## Table of Contents<!-- omit in toc -->
+
+- [Usage](#usage)
+- [Build](#build)
+
+## Usage
+```sh
+docker run -t -p 9944:9944 --name substrate-contracts-node saio:latest
+```
+
+## Build
+
+```sh
+DOCKER_BUILDKIT=1 docker build -f ./tools/docker/substrate-all-in-one/Dockerfile . --tag saio
+```

tools/docker/substrate-all-in-one/hooks/post_push

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+
+SHORTHASH="$(git rev-parse --short HEAD)"
+TODAYS_DATE="$(date +%F)"
+
+#
+# We tag every image with today's date and also the git short hash
+# Today's date helps humans quickly intuit which version is older/newer
+# And the short hash helps identify the exact git revision that the image was
+# built from in case you are chasing some exotic bug that requires this sort of
+# rabbithole diving where you are down to comparing the images at this level.
+#
+DOCKER_TAG="$TODAYS_DATE-$SHORTHASH"
+
+
+docker tag $IMAGE_NAME $DOCKER_REPO:$DOCKER_TAG
+docker push $DOCKER_REPO:$DOCKER_TAG

yarn.lock

@@ -24505,4 +24505,4 @@ [email protected]:
 zone.js@~0.10.3:
   version "0.10.3"
   resolved "https://registry.yarnpkg.com/zone.js/-/zone.js-0.10.3.tgz#3e5e4da03c607c9dcd92e37dd35687a14a140c16"
-  integrity sha512-LXVLVEq0NNOqK/fLJo3d0kfzd4sxwn2/h67/02pjCjfKDxgx1i9QqpvtHD8CrBnSSwMw5+dy11O7FRX5mkO7Cg==
+  integrity sha512-LXVLVEq0NNOqK/fLJo3d0kfzd4sxwn2/h67/02pjCjfKDxgx1i9QqpvtHD8CrBnSSwMw5+dy11O7FRX5mkO7Cg==