From b4065bb949301ee3b82e94c86d13cdc92ad63929 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 11:57:03 +0100 Subject: [PATCH 01/38] try to replace self-host with github runner --- .github/workflows/build_and_run_test.yml | 30 ++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 3fa6d18dd3..1ba1576504 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -56,11 +56,10 @@ jobs: run: make clippy build-docker: - if: "!contains(github.event.commits[0].message, '[SKIP CI]')" concurrency: group: ${{ github.ref }} cancel-in-progress: true - runs-on: self-hosted + runs-on: ubuntu-latest needs: check-cargo-fmt steps: - name: Checkout codes @@ -74,9 +73,18 @@ jobs: ./scripts/build-docker.sh echo "=============================" docker images + docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + + # to share data between jobs + - name: Cache docker image + uses: actions/cache@v2 + path: litentry-parachain.tar + key: docker-${{ hashFiles('litentry-parachain.tar') }} + restore-keys: | + docker- run-ts-tests: - runs-on: self-hosted + runs-on: ubuntu-latest needs: build-docker strategy: matrix: @@ -84,6 +92,10 @@ jobs: - litmus - litentry steps: + - name: Load docker images + run: | + docker load -i litentry-parachain.tar + - name: Run ts tests for ${{ matrix.chain }} timeout-minutes: 20 run: | @@ -146,9 +158,19 @@ jobs: run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib push-docker-image: - runs-on: self-hosted + runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] steps: + - name: Load docker images + run: | + docker load -i litentry-parachain.tar + + - name: Dockerhub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push docker image on dev branch if test passes if: ${{ success() && (github.event_name == 'push') && (github.ref == 'refs/heads/dev') }} run: From 07196f4ba2f72073477112144bb8cd7fa592f31d Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 11:59:27 +0100 Subject: [PATCH 02/38] rectify the usage of cache --- .github/workflows/build_and_run_test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 1ba1576504..10dd614c06 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -78,10 +78,11 @@ jobs: # to share data between jobs - name: Cache docker image uses: actions/cache@v2 - path: litentry-parachain.tar - key: docker-${{ hashFiles('litentry-parachain.tar') }} - restore-keys: | - docker- + with: + path: litentry-parachain.tar + key: docker-${{ hashFiles('litentry-parachain.tar') }} + restore-keys: | + docker- run-ts-tests: runs-on: ubuntu-latest From a561d75fbb80b24cccf8924820aef80c9edae31a Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 13:05:39 +0100 Subject: [PATCH 03/38] remove time limit --- .github/workflows/build_and_run_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 10dd614c06..da942310ca 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -68,7 +68,7 @@ jobs: fetch-depth: 0 - name: Build docker image - timeout-minutes: 40 + # timeout-minutes: 40 run: | ./scripts/build-docker.sh echo "=============================" From e5806ee3cbe81962ab42f1cc0dbddc8c7bfce523 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 15:34:16 +0100 Subject: [PATCH 04/38] use upload/download --- .github/workflows/build_and_run_test.yml | 29 +++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index da942310ca..151440ae16 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -59,7 +59,9 @@ jobs: concurrency: group: ${{ github.ref }} cancel-in-progress: true - runs-on: ubuntu-latest + # run the docker build on our self-hosted runner, which takes < 20min + # on a standard github runner it takes ~1 hour + runs-on: self-hosted needs: check-cargo-fmt steps: - name: Checkout codes @@ -68,21 +70,18 @@ jobs: fetch-depth: 0 - name: Build docker image - # timeout-minutes: 40 + timeout-minutes: 80 run: | ./scripts/build-docker.sh echo "=============================" docker images docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - # to share data between jobs - - name: Cache docker image - uses: actions/cache@v2 + - name: Upload docker image + uses: actions/upload-artifact@v3 with: + name: docker-artifact path: litentry-parachain.tar - key: docker-${{ hashFiles('litentry-parachain.tar') }} - restore-keys: | - docker- run-ts-tests: runs-on: ubuntu-latest @@ -93,7 +92,11 @@ jobs: - litmus - litentry steps: - - name: Load docker images + - uses: actions/download-artifact@v3 + with: + path: docker-artifact + + - name: Load docker image run: | docker load -i litentry-parachain.tar @@ -103,7 +106,7 @@ jobs: make test-ts-docker-${{ matrix.chain }} - name: Archive logs if test fails - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: ${{ matrix.chain }}-ts-tests-artifacts @@ -162,7 +165,11 @@ jobs: runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] steps: - - name: Load docker images + - uses: actions/download-artifact@v3 + with: + path: docker-artifact + + - name: Load docker image run: | docker load -i litentry-parachain.tar From 226582b876a97865de2a2fe6097a7c46e00202f6 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 16:16:39 +0100 Subject: [PATCH 05/38] fix download path --- .github/workflows/build_and_run_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 151440ae16..7487ea481c 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -94,7 +94,7 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - path: docker-artifact + name: docker-artifact - name: Load docker image run: | @@ -167,7 +167,7 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - path: docker-artifact + name: docker-artifact - name: Load docker image run: | From 7e7af385c3a2430bce9b3bc4ab588bdf19c42733 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 16:43:23 +0100 Subject: [PATCH 06/38] missing checkout --- .github/workflows/build_and_run_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 7487ea481c..64553256d1 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -92,6 +92,11 @@ jobs: - litmus - litentry steps: + - name: Checkout codes + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v3 with: name: docker-artifact From 80de5d9d23900268128175403789f4550d29931e Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 18:01:13 +0100 Subject: [PATCH 07/38] use ul/dl version 3 --- .github/workflows/build_and_run_test.yml | 3 +++ .github/workflows/build_docker_with_args.yml | 2 +- .github/workflows/build_wasm.yml | 2 +- .github/workflows/create_release_draft.yml | 6 +++--- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 64553256d1..0db6b19a2e 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -75,6 +75,9 @@ jobs: ./scripts/build-docker.sh echo "=============================" docker images + + - name: Save docker image + run: | docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image diff --git a/.github/workflows/build_docker_with_args.yml b/.github/workflows/build_docker_with_args.yml index a7a2d6450b..d8c3825dd2 100644 --- a/.github/workflows/build_docker_with_args.yml +++ b/.github/workflows/build_docker_with_args.yml @@ -41,7 +41,7 @@ jobs: docker cp $(docker create --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }}):/usr/local/bin/litentry-collator . - name: Upload the client binary - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: litentry-collator path: | diff --git a/.github/workflows/build_wasm.yml b/.github/workflows/build_wasm.yml index d4986ed27f..4591027e4b 100644 --- a/.github/workflows/build_wasm.yml +++ b/.github/workflows/build_wasm.yml @@ -43,7 +43,7 @@ jobs: cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ github.event.inputs.chain }}-parachain-runtime.compact.compressed.wasm - name: Upload wasm artefacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ github.event.inputs.chain }}-parachain-runtime path: | diff --git a/.github/workflows/create_release_draft.yml b/.github/workflows/create_release_draft.yml index 8047e7328e..a3f1a8a066 100644 --- a/.github/workflows/create_release_draft.yml +++ b/.github/workflows/create_release_draft.yml @@ -62,7 +62,7 @@ jobs: cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ env.CHAIN }}-parachain-runtime.compact.compressed.wasm - name: Upload wasm artefacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ env.CHAIN }}-parachain-runtime path: | @@ -105,7 +105,7 @@ jobs: docker cp $(docker create --rm litentry/litentry-parachain:${{ env.RELEASE_TAG }}):/usr/local/bin/litentry-collator . - name: Upload the client binary - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: litentry-collator if-no-files-found: ignore @@ -133,7 +133,7 @@ jobs: fetch-depth: 0 - name: Download all artefacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Generate release notes run: | From 18858867f1d6552d6efb32661c16ab1c4578ab31 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 18:06:36 +0100 Subject: [PATCH 08/38] try to use yaml anchor --- .github/workflows/build_and_run_test.yml | 73 ++++++------------------ 1 file changed, 19 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 0db6b19a2e..3753036ba4 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -19,16 +19,18 @@ jobs: check-cargo-fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - &checkout + uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Install toolchain + - &toolchain + name: Install toolchain uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: nightly - components: rustfmt + components: rustfmt, clippy target: wasm32-unknown-unknown default: true @@ -39,18 +41,8 @@ jobs: runs-on: ubuntu-latest needs: check-cargo-fmt steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - components: clippy - target: wasm32-unknown-unknown - default: true + - *checkout + - *toolchain - name: Run cargo clippy check run: make clippy @@ -64,10 +56,7 @@ jobs: runs-on: self-hosted needs: check-cargo-fmt steps: - - name: Checkout codes - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - *checkout - name: Build docker image timeout-minutes: 80 @@ -95,16 +84,15 @@ jobs: - litmus - litentry steps: - - name: Checkout codes - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - *checkout - - uses: actions/download-artifact@v3 + - &download-docker + uses: actions/download-artifact@v3 with: name: docker-artifact - - name: Load docker image + - &load-docker + name: Load docker image run: | docker load -i litentry-parachain.tar @@ -130,17 +118,8 @@ jobs: runs-on: ubuntu-latest needs: check-cargo-fmt steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - target: wasm32-unknown-unknown - default: true + - *checkout + - *toolchain - name: Run unittests run: cargo test --release -p pallet-* --lib @@ -154,17 +133,8 @@ jobs: - litmus - litentry steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - target: wasm32-unknown-unknown - default: true + - *checkout + - *toolchain - name: Run runtime integration tests run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib @@ -173,13 +143,8 @@ jobs: runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] steps: - - uses: actions/download-artifact@v3 - with: - name: docker-artifact - - - name: Load docker image - run: | - docker load -i litentry-parachain.tar + - *download-docker + - *load-docker - name: Dockerhub login uses: docker/login-action@v1 From d8ebed0ba2fff39da8ea3eef30635f1893357556 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 18:12:42 +0100 Subject: [PATCH 09/38] add feature branch trigger --- .github/workflows/build_and_run_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 3753036ba4..c38da808f6 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -4,6 +4,7 @@ on: push: branches: - dev + - 393-avoid-self-runner-in-CI paths-ignore: - '**/dependabot.yml' - '**/README.md' From 1453857af21105d28e199f1f6ecaef2e55f349b2 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 15 Mar 2022 18:37:30 +0100 Subject: [PATCH 10/38] revert anchor as its not supported --- .github/workflows/build_and_run_test.yml | 74 +++++++++++++++++------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index c38da808f6..0db6b19a2e 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -4,7 +4,6 @@ on: push: branches: - dev - - 393-avoid-self-runner-in-CI paths-ignore: - '**/dependabot.yml' - '**/README.md' @@ -20,18 +19,16 @@ jobs: check-cargo-fmt: runs-on: ubuntu-latest steps: - - &checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: fetch-depth: 0 - - &toolchain - name: Install toolchain + - name: Install toolchain uses: actions-rs/toolchain@v1 with: profile: minimal toolchain: nightly - components: rustfmt, clippy + components: rustfmt target: wasm32-unknown-unknown default: true @@ -42,8 +39,18 @@ jobs: runs-on: ubuntu-latest needs: check-cargo-fmt steps: - - *checkout - - *toolchain + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + components: clippy + target: wasm32-unknown-unknown + default: true - name: Run cargo clippy check run: make clippy @@ -57,7 +64,10 @@ jobs: runs-on: self-hosted needs: check-cargo-fmt steps: - - *checkout + - name: Checkout codes + uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Build docker image timeout-minutes: 80 @@ -85,15 +95,16 @@ jobs: - litmus - litentry steps: - - *checkout + - name: Checkout codes + uses: actions/checkout@v2 + with: + fetch-depth: 0 - - &download-docker - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v3 with: name: docker-artifact - - &load-docker - name: Load docker image + - name: Load docker image run: | docker load -i litentry-parachain.tar @@ -119,8 +130,17 @@ jobs: runs-on: ubuntu-latest needs: check-cargo-fmt steps: - - *checkout - - *toolchain + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + target: wasm32-unknown-unknown + default: true - name: Run unittests run: cargo test --release -p pallet-* --lib @@ -134,8 +154,17 @@ jobs: - litmus - litentry steps: - - *checkout - - *toolchain + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: nightly + target: wasm32-unknown-unknown + default: true - name: Run runtime integration tests run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib @@ -144,8 +173,13 @@ jobs: runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] steps: - - *download-docker - - *load-docker + - uses: actions/download-artifact@v3 + with: + name: docker-artifact + + - name: Load docker image + run: | + docker load -i litentry-parachain.tar - name: Dockerhub login uses: docker/login-action@v1 From d1e313f333014975cf7808d4e763ef184a6c11b3 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 00:42:05 +0100 Subject: [PATCH 11/38] try to use buildx cache --- .github/workflows/build_and_run_test.yml | 35 +++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 0db6b19a2e..7f0a0b56d4 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -61,7 +61,8 @@ jobs: cancel-in-progress: true # run the docker build on our self-hosted runner, which takes < 20min # on a standard github runner it takes ~1 hour - runs-on: self-hosted + # runs-on: self-hosted + runs-on: ubuntu-latest needs: check-cargo-fmt steps: - name: Checkout codes @@ -69,16 +70,30 @@ jobs: with: fetch-depth: 0 - - name: Build docker image - timeout-minutes: 80 - run: | - ./scripts/build-docker.sh - echo "=============================" - docker images + - name: Setup docker buildx + uses: docker/setup-buildx-action@v1 - - name: Save docker image - run: | - docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + # - name: Build docker image + # timeout-minutes: 80 + # run: | + # ./scripts/build-docker.sh + # echo "=============================" + # docker images + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + tags: litentry/litentry-parachain:latest + outputs: type=docker,dest=litentry-parachain.tar + push: false + file: docker/Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max + + # - name: Save docker image + # run: | + # docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image uses: actions/upload-artifact@v3 From 11b8afc1725acffec768111ca5b392843a4aad6b Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 01:42:43 +0100 Subject: [PATCH 12/38] add sccache and file-filters --- .github/workflows/build_and_run_test.yml | 75 ++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 7f0a0b56d4..f57f7d448d 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -16,6 +16,20 @@ env: CARGO_TERM_COLOR: always jobs: + check-file-change: + runs-on: ubuntu-latest + outputs: + ts: ${{ steps.filter.outputs.ts }} + steps: + # Checks to see if any files in the PR/commit match one of the listed file types. + # We can use this filter to decide whether or not to build docker images + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + ts: + - 'ts-tests/**' + check-cargo-fmt: runs-on: ubuntu-latest steps: @@ -63,7 +77,7 @@ jobs: # on a standard github runner it takes ~1 hour # runs-on: self-hosted runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] steps: - name: Checkout codes uses: actions/checkout@v2 @@ -80,7 +94,8 @@ jobs: # echo "=============================" # docker images - - name: Build and push Docker image + - name: Build docker image + if: needs.check-file-change.outputs.ts == 'false' uses: docker/build-push-action@v2 with: context: . @@ -91,6 +106,14 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Save docker image + # if the changed files are ts-tests/** files, + # do not build the docker image, just pull it instead + if: needs.check-file-change.outputs.ts == 'true' + run: | + docker pull litentry/litentry-parachain:latest + docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + # - name: Save docker image # run: | # docker save litentry/litentry-parachain:latest -o litentry-parachain.tar @@ -143,7 +166,8 @@ jobs: run-cargo-unit-tests: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.ts == 'false' steps: - uses: actions/checkout@v2 with: @@ -162,7 +186,14 @@ jobs: run-cargo-runtime-tests: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.ts == 'false' + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 10G + SCCACHE_DIR: /home/runner/.cache/sccache + CARGO_INCREMENTAL: 0 strategy: matrix: chain: @@ -181,9 +212,45 @@ jobs: target: wasm32-unknown-unknown default: true + # use sccache to accelerate binary compilation + # see https://www.infinyon.com/blog/2021/04/github-actions-best-practices/ + - name: Install sccache + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.2.15 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Cache cargo registry + uses: actions/cache@v2 + continue-on-error: false + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo- + + - name: Cache sccache + uses: actions/cache@v2 + continue-on-error: false + with: + path: /home/runner/.cache/sccache + key: sccache-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + sccache- + - name: Run runtime integration tests run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib + - name: Print sccache stats + run: sccache --show-stats + push-docker-image: runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] From 772861f9704a613336ae71dfac4d4d7f0adfa1dc Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 02:11:02 +0100 Subject: [PATCH 13/38] missing checkout --- .github/workflows/build_and_run_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index f57f7d448d..7cdbfe1f00 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -21,6 +21,10 @@ jobs: outputs: ts: ${{ steps.filter.outputs.ts }} steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + # Checks to see if any files in the PR/commit match one of the listed file types. # We can use this filter to decide whether or not to build docker images - uses: dorny/paths-filter@v2 From af60a9fea5268473b7e62d1ad5884d4087dd544d Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 02:14:50 +0100 Subject: [PATCH 14/38] fix permission --- .github/workflows/build_and_run_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 7cdbfe1f00..637aae9e21 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -227,6 +227,7 @@ jobs: mkdir -p $HOME/.local/bin curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Cache cargo registry From 99b636a3211fdbc50743bc62dd56027f2f8b4c35 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 10:53:57 +0100 Subject: [PATCH 15/38] move concurrency to workflow level --- .github/workflows/build_and_run_test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 637aae9e21..0835fd9c9b 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -15,6 +15,10 @@ on: env: CARGO_TERM_COLOR: always +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: check-file-change: runs-on: ubuntu-latest @@ -74,9 +78,6 @@ jobs: run: make clippy build-docker: - concurrency: - group: ${{ github.ref }} - cancel-in-progress: true # run the docker build on our self-hosted runner, which takes < 20min # on a standard github runner it takes ~1 hour # runs-on: self-hosted From 022cd771266886e02f931e4fb198e3f29ab47a2f Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 10:54:13 +0100 Subject: [PATCH 16/38] update ts-tests dep to check the file filter --- ts-tests/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts-tests/package.json b/ts-tests/package.json index a9f24a01b7..1fabcf5d20 100644 --- a/ts-tests/package.json +++ b/ts-tests/package.json @@ -25,7 +25,7 @@ }, "dependencies": { "@polkadot/api": "^7.12.1", - "@polkadot/types": "^7.11.1", + "@polkadot/types": "^7.12.1", "dotenv": "^16.0.0", "web3": "^1.7.1" }, From a107971e7a8182d524ff28c3faba83fc14030e44 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 11:44:58 +0100 Subject: [PATCH 17/38] unuse docker/build-push-action --- .github/workflows/build_and_run_test.yml | 40 ++++++++---------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 0835fd9c9b..d4dc849d39 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -59,7 +59,8 @@ jobs: check-cargo-clippy: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.ts == 'false' steps: - uses: actions/checkout@v2 with: @@ -80,8 +81,8 @@ jobs: build-docker: # run the docker build on our self-hosted runner, which takes < 20min # on a standard github runner it takes ~1 hour - # runs-on: self-hosted - runs-on: ubuntu-latest + # caching with docker/build-push-action doesn't seem to help + runs-on: self-hosted needs: [check-cargo-fmt, check-file-change] steps: - name: Checkout codes @@ -89,39 +90,24 @@ jobs: with: fetch-depth: 0 - - name: Setup docker buildx - uses: docker/setup-buildx-action@v1 - - # - name: Build docker image - # timeout-minutes: 80 - # run: | - # ./scripts/build-docker.sh - # echo "=============================" - # docker images - - name: Build docker image if: needs.check-file-change.outputs.ts == 'false' - uses: docker/build-push-action@v2 - with: - context: . - tags: litentry/litentry-parachain:latest - outputs: type=docker,dest=litentry-parachain.tar - push: false - file: docker/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max + timeout-minutes: 40 + run: | + ./scripts/build-docker.sh + echo "=============================" + docker images - - name: Save docker image + - name: Pull docker image optinally # if the changed files are ts-tests/** files, # do not build the docker image, just pull it instead if: needs.check-file-change.outputs.ts == 'true' run: | docker pull litentry/litentry-parachain:latest - docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - # - name: Save docker image - # run: | - # docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + - name: Save docker image + run: | + docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image uses: actions/upload-artifact@v3 From 5e13561b78899fd31e369ef1170d864c7482dcf2 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 11:48:19 +0100 Subject: [PATCH 18/38] cargo update --- Cargo.lock | 159 ++++++++++++++++++++++++++++------------------------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31c6865dba..0ee3225e1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd" +checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" [[package]] name = "approx" @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d306121baf53310a3fd342d88dc0824f6bbeace68347593658525565abee8" +checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" @@ -645,9 +645,9 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" [[package]] name = "blocking" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046e47d4b2d391b1f6f8b407b1deb8dee56c1852ccd868becf2710f601b5f427" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" dependencies = [ "async-channel", "async-task", @@ -1234,9 +1234,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "fdbfe11fe19ff083c48923cf179540e8cd0535903dc35e178a1fdeeb59aef51f" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -1255,10 +1255,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ + "autocfg", "cfg-if 1.0.0", "crossbeam-utils", "lazy_static", @@ -1268,9 +1269,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -1354,7 +1355,7 @@ dependencies = [ [[package]] name = "cumulus-client-cli" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "clap", "sc-cli", @@ -1364,7 +1365,7 @@ dependencies = [ [[package]] name = "cumulus-client-collator" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-client-consensus-common", "cumulus-client-network", @@ -1388,7 +1389,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-aura" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-client-consensus-common", @@ -1417,7 +1418,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-common" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-relay-chain-interface", @@ -1438,7 +1439,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-relay-chain" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-client-consensus-common", @@ -1462,7 +1463,7 @@ dependencies = [ [[package]] name = "cumulus-client-network" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-relay-chain-interface", @@ -1487,7 +1488,7 @@ dependencies = [ [[package]] name = "cumulus-client-pov-recovery" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "cumulus-relay-chain-interface", @@ -1511,7 +1512,7 @@ dependencies = [ [[package]] name = "cumulus-client-service" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-client-collator", "cumulus-client-consensus-common", @@ -1540,7 +1541,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-aura-ext" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-executive", "frame-support", @@ -1558,7 +1559,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-dmp-queue" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1576,7 +1577,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-parachain-system" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-pallet-parachain-system-proc-macro", "cumulus-primitives-core", @@ -1606,7 +1607,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-parachain-system-proc-macro" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", @@ -1617,7 +1618,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-session-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-benchmarking", "frame-support", @@ -1631,7 +1632,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-xcm" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1648,7 +1649,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-xcmp-queue" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1666,7 +1667,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-core" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-support", "parity-scale-codec", @@ -1682,7 +1683,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-parachain-inherent" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1705,7 +1706,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-timestamp" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "sp-inherents", @@ -1716,7 +1717,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-utility" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1733,7 +1734,7 @@ dependencies = [ [[package]] name = "cumulus-relay-chain-interface" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1754,7 +1755,7 @@ dependencies = [ [[package]] name = "cumulus-relay-chain-local" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1782,7 +1783,7 @@ dependencies = [ [[package]] name = "cumulus-test-relay-sproof-builder" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "parity-scale-codec", @@ -1982,9 +1983,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" +checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" [[package]] name = "ed25519" @@ -2706,9 +2707,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" +checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" dependencies = [ "bytes 1.1.0", "fnv", @@ -2725,9 +2726,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25546a65e5cf1f471f3438796fc634650b31d7fcde01d444c309aeb28b92e3a8" +checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ "log", "pest", @@ -3096,9 +3097,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" [[package]] name = "itertools" @@ -3607,9 +3608,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" [[package]] name = "libloading" @@ -4172,9 +4173,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" dependencies = [ "cc", "pkg-config", @@ -4479,9 +4480,9 @@ dependencies = [ [[package]] name = "lz4" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aac20ed6991e01bf6a2e68cc73df2b389707403662a8ba89f68511fb340f724c" +checksum = "4edcb94251b1c375c459e5abe9fb0168c1c826c3370172684844f8f3f8d1a885" dependencies = [ "libc", "lz4-sys", @@ -4489,9 +4490,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca79aa95d8b3226213ad454d328369853be3a1382d89532a854f4d69640acae" +checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17" dependencies = [ "cc", "libc", @@ -4627,9 +4628,9 @@ dependencies = [ [[package]] name = "mick-jaeger" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2c2cc134e57461f0898b0e921f0a7819b5e3f3a4335b9aa390ce81a5f36fb9" +checksum = "69672161530e8aeca1d1400fbf3f1a1747ff60ea604265a4e906c2442df20532" dependencies = [ "futures 0.3.21", "rand 0.8.5", @@ -4673,14 +4674,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "7ba42135c6a5917b9db9cd7b293e5409e1c6b041e6f9825e92e55a894c63b6f8" dependencies = [ "libc", "log", "miow 0.3.7", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -4879,13 +4881,12 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", ] [[package]] @@ -4983,9 +4984,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "opaque-debug" @@ -5321,7 +5322,7 @@ dependencies = [ [[package]] name = "pallet-collator-selection" version = "3.0.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-benchmarking", "frame-support", @@ -6010,7 +6011,7 @@ dependencies = [ [[package]] name = "parachain-info" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -6022,9 +6023,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aa6c5bb8070cf0456d9fc228b3022e900aae9092c48c9c45facf97422efc1d" +checksum = "865edee5b792f537356d9e55cbc138e7f4718dc881a7ea45a18b37bf61c21e3d" dependencies = [ "blake2-rfc", "crc32fast", @@ -7910,9 +7911,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ "aho-corasick", "memchr", @@ -10482,9 +10483,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "ebd69e719f31e88618baa1eaa6ee2de5c9a1c004f1e9ecdb58e8352a13f20a01" dependencies = [ "proc-macro2", "quote", @@ -10531,9 +10532,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] @@ -10658,7 +10659,7 @@ dependencies = [ "bytes 1.1.0", "libc", "memchr", - "mio 0.8.0", + "mio 0.8.1", "num_cpus", "once_cell", "pin-project-lite 0.2.8", @@ -10744,9 +10745,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f" +checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if 1.0.0", "pin-project-lite 0.2.8", @@ -10756,9 +10757,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8276d9a4a3a558d7b7ad5303ad50b53d58264641b82914b7ada36bd762e7a716" +checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" dependencies = [ "proc-macro2", "quote", @@ -10767,9 +10768,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" +checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" dependencies = [ "lazy_static", "valuable", @@ -11144,6 +11145,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.79" From 2349253d2d84fed906ad5562e08c526811e08d18 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 15:00:50 +0100 Subject: [PATCH 19/38] refine the file filter --- .github/workflows/build_and_run_test.yml | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index d4dc849d39..4e86dd3639 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -23,7 +23,7 @@ jobs: check-file-change: runs-on: ubuntu-latest outputs: - ts: ${{ steps.filter.outputs.ts }} + src: ${{ steps.filter.outputs.src }} steps: - uses: actions/checkout@v2 with: @@ -35,8 +35,15 @@ jobs: id: filter with: filters: | - ts: - - 'ts-tests/**' + src: + - 'node/**' + - 'pallets/**' + - 'primitives/**' + - 'runtime/**' + - 'scripts/build-docker.sh' + - '**/Cargo.lock' + - '**/Cargo.toml' + - '**/rust-toolchain.toml' check-cargo-fmt: runs-on: ubuntu-latest @@ -60,7 +67,7 @@ jobs: check-cargo-clippy: runs-on: ubuntu-latest needs: [check-cargo-fmt, check-file-change] - if: needs.check-file-change.outputs.ts == 'false' + if: needs.check-file-change.outputs.src == 'true' steps: - uses: actions/checkout@v2 with: @@ -91,7 +98,7 @@ jobs: fetch-depth: 0 - name: Build docker image - if: needs.check-file-change.outputs.ts == 'false' + if: needs.check-file-change.outputs.src == 'true' timeout-minutes: 40 run: | ./scripts/build-docker.sh @@ -99,9 +106,7 @@ jobs: docker images - name: Pull docker image optinally - # if the changed files are ts-tests/** files, - # do not build the docker image, just pull it instead - if: needs.check-file-change.outputs.ts == 'true' + if: needs.check-file-change.outputs.src == 'false' run: | docker pull litentry/litentry-parachain:latest @@ -158,7 +163,7 @@ jobs: run-cargo-unit-tests: runs-on: ubuntu-latest needs: [check-cargo-fmt, check-file-change] - if: needs.check-file-change.outputs.ts == 'false' + if: needs.check-file-change.outputs.src == 'true' steps: - uses: actions/checkout@v2 with: @@ -178,7 +183,7 @@ jobs: run-cargo-runtime-tests: runs-on: ubuntu-latest needs: [check-cargo-fmt, check-file-change] - if: needs.check-file-change.outputs.ts == 'false' + if: needs.check-file-change.outputs.src == 'true' env: RUST_BACKTRACE: full RUSTC_WRAPPER: sccache From af860727210e4504d1516673334bbdc07e3e794e Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 16:26:50 +0100 Subject: [PATCH 20/38] try cargo chef --- docker/Dockerfile | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 609dccd8b7..4ca82bdb8d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,17 +1,29 @@ +FROM rust:latest as chef +USER root +RUN cargo install cargo-chef +WORKDIR /app + +FROM chef AS planner +COPY . . +RUN rustup default nightly && \ + rustup target add wasm32-unknown-unknown --toolchain nightly +RUN cargo chef prepare --recipe-path recipe.json + # ========================== # stage 1: build # https://hub.docker.com/_/rust # our host is based on bullseye/sid => rust:latest # ========================== -FROM rust:latest as builder +FROM chef as builder WORKDIR /litentry -COPY . /litentry - -# always use the toolchain in base image -# we do not fix toolchain version as it's still evolving rapidly RUN rustup default nightly && \ rustup target add wasm32-unknown-unknown --toolchain nightly +COPY --from=planner /app/recipe.json /litentry/recipe.json +# Notice that we are specifying the --target flag! +RUN cargo chef cook --release --recipe-path recipe.json +COPY . /litentry + RUN apt-get update && \ apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev From 2bb509c90467a9c1deca8bc87308c8ac7dcba0cc Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 16:29:39 +0100 Subject: [PATCH 21/38] reuse buildx action --- .github/workflows/build_and_run_test.yml | 33 +++++++++++++++++------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 4e86dd3639..80aa94207f 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -89,7 +89,7 @@ jobs: # run the docker build on our self-hosted runner, which takes < 20min # on a standard github runner it takes ~1 hour # caching with docker/build-push-action doesn't seem to help - runs-on: self-hosted + runs-on: ubuntu-latest needs: [check-cargo-fmt, check-file-change] steps: - name: Checkout codes @@ -97,22 +97,37 @@ jobs: with: fetch-depth: 0 + # - name: Build docker image + # if: needs.check-file-change.outputs.src == 'true' + # timeout-minutes: 40 + # run: | + # ./scripts/build-docker.sh + # echo "=============================" + # docker images + + - name: Setup docker buildx + uses: docker/setup-buildx-action@v1 + - name: Build docker image if: needs.check-file-change.outputs.src == 'true' - timeout-minutes: 40 - run: | - ./scripts/build-docker.sh - echo "=============================" - docker images + uses: docker/build-push-action@v2 + with: + context: . + tags: litentry/litentry-parachain:latest + outputs: type=docker,dest=litentry-parachain.tar + push: false + file: docker/Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max - name: Pull docker image optinally if: needs.check-file-change.outputs.src == 'false' run: | docker pull litentry/litentry-parachain:latest - - name: Save docker image - run: | - docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + # - name: Save docker image + # run: | + # docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image uses: actions/upload-artifact@v3 From 3f501650ed624064d29d1bd95ec4a71d9fa28a64 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 17:06:53 +0100 Subject: [PATCH 22/38] fix recipe path --- .github/workflows/build_and_run_test.yml | 1 + docker/Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 80aa94207f..f757a451d9 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -41,6 +41,7 @@ jobs: - 'primitives/**' - 'runtime/**' - 'scripts/build-docker.sh' + - 'docker/Dockerfile' - '**/Cargo.lock' - '**/Cargo.toml' - '**/rust-toolchain.toml' diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ca82bdb8d..5fd589f8d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,9 +20,10 @@ RUN rustup default nightly && \ rustup target add wasm32-unknown-unknown --toolchain nightly COPY --from=planner /app/recipe.json /litentry/recipe.json +COPY . /litentry + # Notice that we are specifying the --target flag! RUN cargo chef cook --release --recipe-path recipe.json -COPY . /litentry RUN apt-get update && \ apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ From 2009d42c9c4bad9f293bce147796b42f811fc40e Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 17:28:56 +0100 Subject: [PATCH 23/38] prepend cargo build with apt --- docker/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5fd589f8d5..af1bf95f4a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,16 +19,15 @@ WORKDIR /litentry RUN rustup default nightly && \ rustup target add wasm32-unknown-unknown --toolchain nightly +RUN apt-get update && \ + apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ + apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev + COPY --from=planner /app/recipe.json /litentry/recipe.json COPY . /litentry -# Notice that we are specifying the --target flag! RUN cargo chef cook --release --recipe-path recipe.json -RUN apt-get update && \ - apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ - apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev - ARG BUILD_ARGS RUN cargo build --release $BUILD_ARGS From 13c8324be0f637a8965419d2417f52c290152172 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 17:36:33 +0100 Subject: [PATCH 24/38] fix order of copy --- docker/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index af1bf95f4a..e1d6eeaee0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,13 +23,14 @@ RUN apt-get update && \ apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev +# copy the recipe and toolchain file to make sure we are using consistent `cargo` COPY --from=planner /app/recipe.json /litentry/recipe.json -COPY . /litentry - +COPY rust-toolchain.toml /litentry/rust-toolchain.toml RUN cargo chef cook --release --recipe-path recipe.json +# copy and build the real src +COPY . /litentry ARG BUILD_ARGS - RUN cargo build --release $BUILD_ARGS # ========================== From a8b72b0d95a14e9b9a87245377709946644415c8 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 21:17:38 +0100 Subject: [PATCH 25/38] use self-hosted docker --- .github/workflows/build_and_run_test.yml | 23 ++++++++-------------- docker/Dockerfile | 25 +++++++++++++++--------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index f757a451d9..9a3ab1228a 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -87,10 +87,9 @@ jobs: run: make clippy build-docker: - # run the docker build on our self-hosted runner, which takes < 20min + # run the docker build on our self-hosted runner, which takes < 20min without cache # on a standard github runner it takes ~1 hour - # caching with docker/build-push-action doesn't seem to help - runs-on: ubuntu-latest + runs-on: self-hosted needs: [check-cargo-fmt, check-file-change] steps: - name: Checkout codes @@ -98,17 +97,10 @@ jobs: with: fetch-depth: 0 - # - name: Build docker image - # if: needs.check-file-change.outputs.src == 'true' - # timeout-minutes: 40 - # run: | - # ./scripts/build-docker.sh - # echo "=============================" - # docker images - - name: Setup docker buildx uses: docker/setup-buildx-action@v1 + # we try to cache docker layers between GHA with cargo-chef - name: Build docker image if: needs.check-file-change.outputs.src == 'true' uses: docker/build-push-action@v2 @@ -125,10 +117,7 @@ jobs: if: needs.check-file-change.outputs.src == 'false' run: | docker pull litentry/litentry-parachain:latest - - # - name: Save docker image - # run: | - # docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image uses: actions/upload-artifact@v3 @@ -136,6 +125,10 @@ jobs: name: docker-artifact path: litentry-parachain.tar + - name: Remove dangling docker images if any + run: | + [ -z "$(docker images --filter=dangling=true -q)" ] || docker rmi -f $(docker images --filter=dangling=true -q) + run-ts-tests: runs-on: ubuntu-latest needs: build-docker diff --git a/docker/Dockerfile b/docker/Dockerfile index e1d6eeaee0..efad9b37f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,19 +1,25 @@ +# =================================== +# stage 1: install cargo-chef +# https://hub.docker.com/_/rust +# our host is based on bullseye/sid => rust:latest +# =================================== FROM rust:latest as chef USER root RUN cargo install cargo-chef WORKDIR /app -FROM chef AS planner +# =================================== +# stage 2: generate cargo-chef recipe +# =================================== +FROM chef AS cacher COPY . . RUN rustup default nightly && \ rustup target add wasm32-unknown-unknown --toolchain nightly RUN cargo chef prepare --recipe-path recipe.json -# ========================== -# stage 1: build -# https://hub.docker.com/_/rust -# our host is based on bullseye/sid => rust:latest -# ========================== +# =================================== +# stage 3: build +# =================================== FROM chef as builder WORKDIR /litentry RUN rustup default nightly && \ @@ -24,7 +30,7 @@ RUN apt-get update && \ apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev # copy the recipe and toolchain file to make sure we are using consistent `cargo` -COPY --from=planner /app/recipe.json /litentry/recipe.json +COPY --from=cacher /app/recipe.json /litentry/recipe.json COPY rust-toolchain.toml /litentry/rust-toolchain.toml RUN cargo chef cook --release --recipe-path recipe.json @@ -32,10 +38,11 @@ RUN cargo chef cook --release --recipe-path recipe.json COPY . /litentry ARG BUILD_ARGS RUN cargo build --release $BUILD_ARGS +RUN strip target/release/litentry-collator -# ========================== +# =================================== # stage 2: packaging -# ========================== +# =================================== FROM ubuntu:20.04 LABEL maintainer="litentry-dev" From 497adc854921b730bc54e39cf6d718ed0feee543 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 23:15:12 +0100 Subject: [PATCH 26/38] remove buildx --- .github/workflows/build_and_run_test.yml | 21 ++++++++------------- scripts/build-docker.sh | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 9a3ab1228a..62fe11b493 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -97,26 +97,21 @@ jobs: with: fetch-depth: 0 - - name: Setup docker buildx - uses: docker/setup-buildx-action@v1 - - # we try to cache docker layers between GHA with cargo-chef - name: Build docker image if: needs.check-file-change.outputs.src == 'true' - uses: docker/build-push-action@v2 - with: - context: . - tags: litentry/litentry-parachain:latest - outputs: type=docker,dest=litentry-parachain.tar - push: false - file: docker/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max + timeout-minutes: 40 + run: | + ./scripts/build-docker.sh + echo "=============================" + docker images - name: Pull docker image optinally if: needs.check-file-change.outputs.src == 'false' run: | docker pull litentry/litentry-parachain:latest + + - name: Save docker image + run: | docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index aad487219e..d1bcbf630a 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -29,7 +29,7 @@ GITREPO=litentry-parachain # Build the image echo "------------------------------------------------------------" echo "Building ${GITUSER}/${GITREPO}:${TAG} docker image ..." -docker build --rm --no-cache --pull -f ./docker/Dockerfile \ +docker build --rm --pull -f ./docker/Dockerfile \ --build-arg BUILD_ARGS="$ARGS" \ -t ${GITUSER}/${GITREPO}:${TAG} . From a04e2c8b129ce9f584d1a653648701d27249877f Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Wed, 16 Mar 2022 23:54:56 +0100 Subject: [PATCH 27/38] update comment --- .dockerignore | 2 -- .gitignore | 1 - runtime/litentry/src/lib.rs | 7 ------- runtime/litmus/src/lib.rs | 7 ------- 4 files changed, 17 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9d8aa0ed0d..f88ea00741 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,2 @@ -**/polkadot -**/token-server **/target **/ts-tests diff --git a/.gitignore b/.gitignore index c0a0e1735b..04e7b5545a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ .cargo/config **/node_modules/ -**/para-* docker/yarn.lock docker/package.json diff --git a/runtime/litentry/src/lib.rs b/runtime/litentry/src/lib.rs index cce98b3346..d853b127fa 100644 --- a/runtime/litentry/src/lib.rs +++ b/runtime/litentry/src/lib.rs @@ -132,14 +132,7 @@ impl_opaque_keys! { /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { - // Note: // It's important to match `litentry-parachain-runtime`, which is runtime pkg name - // otherwise no extrinsic can be submitted. - // In logs it's shown: - // Failed to submit extrinsic: Extrinsic verification error: Runtime error: Execution failed: - // Other("Wasm execution trapped: wasm trap: unreachable ... - // - // However our CI passes (TODO) spec_name: create_runtime_str!("litentry-parachain"), impl_name: create_runtime_str!("litentry-parachain"), authoring_version: 1, diff --git a/runtime/litmus/src/lib.rs b/runtime/litmus/src/lib.rs index 463003ed9a..fa0b601247 100644 --- a/runtime/litmus/src/lib.rs +++ b/runtime/litmus/src/lib.rs @@ -132,14 +132,7 @@ impl_opaque_keys! { /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { - // Note: // It's important to match `litmus-parachain-runtime`, which is runtime pkg name - // otherwise no extrinsic can be submitted. - // In logs it's shown: - // Failed to submit extrinsic: Extrinsic verification error: Runtime error: Execution failed: - // Other("Wasm execution trapped: wasm trap: unreachable ... - // - // However our CI passes (TODO) spec_name: create_runtime_str!("litmus-parachain"), impl_name: create_runtime_str!("litmus-parachain"), authoring_version: 1, From d5cd48f83518d589b4120bbedfe40b30044a1536 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 00:16:34 +0100 Subject: [PATCH 28/38] retry buildx --- .github/workflows/build_and_run_test.yml | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 62fe11b493..6dd2c907a9 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -97,13 +97,28 @@ jobs: with: fetch-depth: 0 + - name: Setup docker buildx + uses: docker/setup-buildx-action@v1 + - name: Build docker image if: needs.check-file-change.outputs.src == 'true' - timeout-minutes: 40 - run: | - ./scripts/build-docker.sh - echo "=============================" - docker images + uses: docker/build-push-action@v2 + with: + context: . + tags: litentry/litentry-parachain:latest + outputs: type=docker,dest=litentry-parachain.tar + push: false + file: docker/Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max + + # - name: Build docker image + # if: needs.check-file-change.outputs.src == 'true' + # timeout-minutes: 40 + # run: | + # ./scripts/build-docker.sh + # echo "=============================" + # docker images - name: Pull docker image optinally if: needs.check-file-change.outputs.src == 'false' From 18d0c82b03ba73a98edc3f59b6c6a1d7a2c4970d Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 00:34:36 +0100 Subject: [PATCH 29/38] try local cache --- .github/workflows/build_and_run_test.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 6dd2c907a9..ca23e86671 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -100,6 +100,8 @@ jobs: - name: Setup docker buildx uses: docker/setup-buildx-action@v1 + # To sync GHA cache seems to be very slow, as we are building locally, + # simply use a local cache - name: Build docker image if: needs.check-file-change.outputs.src == 'true' uses: docker/build-push-action@v2 @@ -109,8 +111,16 @@ jobs: outputs: type=docker,dest=litentry-parachain.tar push: false file: docker/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache # - name: Build docker image # if: needs.check-file-change.outputs.src == 'true' From ed700c4b9433782e7ee96069fcc842a164b87abe Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 00:55:45 +0100 Subject: [PATCH 30/38] update comment to test cached builds --- node/src/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/command.rs b/node/src/command.rs index 3736d1e58c..ec622ed514 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -95,7 +95,7 @@ fn load_spec(id: &str) -> std::result::Result, St if chain_spec.is_litmus() { Box::new(chain_specs::litmus::ChainSpec::from_json_file(path.into())?) } else { - // By default litentry is used + // Fallback: Litentry chain spec Box::new(chain_spec) } }, From 95868a6994d542ba1a9fef961116b095db75d728 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 01:13:25 +0100 Subject: [PATCH 31/38] fix docker save --- .github/workflows/build_and_run_test.yml | 11 ++++------- scripts/build-docker.sh | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index ca23e86671..8c39df38c1 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -111,16 +111,16 @@ jobs: outputs: type=docker,dest=litentry-parachain.tar push: false file: docker/Dockerfile - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + cache-from: type=local,src=/tmp/parachain-buildx-cache + cache-to: type=local,dest=/tmp/parachain-buildx-cache-new,mode=max - name: Move cache # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + rm -rf /tmp/parachain-buildx-cache + mv /tmp/parachain-buildx-cache-new /tmp/parachain-buildx-cache # - name: Build docker image # if: needs.check-file-change.outputs.src == 'true' @@ -134,9 +134,6 @@ jobs: if: needs.check-file-change.outputs.src == 'false' run: | docker pull litentry/litentry-parachain:latest - - - name: Save docker image - run: | docker save litentry/litentry-parachain:latest -o litentry-parachain.tar - name: Upload docker image diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index d1bcbf630a..aad487219e 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -29,7 +29,7 @@ GITREPO=litentry-parachain # Build the image echo "------------------------------------------------------------" echo "Building ${GITUSER}/${GITREPO}:${TAG} docker image ..." -docker build --rm --pull -f ./docker/Dockerfile \ +docker build --rm --no-cache --pull -f ./docker/Dockerfile \ --build-arg BUILD_ARGS="$ARGS" \ -t ${GITUSER}/${GITREPO}:${TAG} . From 31e32761f2163e12127995fff0b0c8412260f515 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 08:30:48 +0100 Subject: [PATCH 32/38] modify comment to trigger GHA --- node/src/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/command.rs b/node/src/command.rs index ec622ed514..a70f3c2cad 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -95,7 +95,7 @@ fn load_spec(id: &str) -> std::result::Result, St if chain_spec.is_litmus() { Box::new(chain_specs::litmus::ChainSpec::from_json_file(path.into())?) } else { - // Fallback: Litentry chain spec + // Fallback: use Litentry chain spec Box::new(chain_spec) } }, From 38c44559431dd55baf01e37b94d919a89648e522 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 11:44:08 +0100 Subject: [PATCH 33/38] separate dev and prod Dockefile --- .../workflows/benchmark_runtime_weights.yml | 2 +- .github/workflows/build_and_run_test.yml | 19 ++------ .github/workflows/build_docker_with_args.yml | 8 +++- .github/workflows/create_release_draft.yml | 2 +- Makefile | 10 ++-- README.md | 6 ++- docker/{Dockerfile => Dockerfile.dev} | 1 - docker/Dockerfile.prod | 46 +++++++++++++++++++ scripts/build-docker.sh | 14 ++++-- 9 files changed, 83 insertions(+), 25 deletions(-) rename docker/{Dockerfile => Dockerfile.dev} (97%) create mode 100644 docker/Dockerfile.prod diff --git a/.github/workflows/benchmark_runtime_weights.yml b/.github/workflows/benchmark_runtime_weights.yml index 751d8186cb..12cafe5101 100644 --- a/.github/workflows/benchmark_runtime_weights.yml +++ b/.github/workflows/benchmark_runtime_weights.yml @@ -29,7 +29,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh runtime-benchmarks --features=runtime-benchmarks + ./scripts/build-docker.sh prod runtime-benchmarks --features=runtime-benchmarks - name: Push docker image run: | diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 8c39df38c1..6df103ceee 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -40,8 +40,7 @@ jobs: - 'pallets/**' - 'primitives/**' - 'runtime/**' - - 'scripts/build-docker.sh' - - 'docker/Dockerfile' + - 'docker/Dockerfile.dev' - '**/Cargo.lock' - '**/Cargo.toml' - '**/rust-toolchain.toml' @@ -100,8 +99,7 @@ jobs: - name: Setup docker buildx uses: docker/setup-buildx-action@v1 - # To sync GHA cache seems to be very slow, as we are building locally, - # simply use a local cache + # Syncing GHA cache seems to be very slow, we use local cache for a self-hosted runner - name: Build docker image if: needs.check-file-change.outputs.src == 'true' uses: docker/build-push-action@v2 @@ -110,26 +108,19 @@ jobs: tags: litentry/litentry-parachain:latest outputs: type=docker,dest=litentry-parachain.tar push: false - file: docker/Dockerfile + file: docker/Dockerfile.dev cache-from: type=local,src=/tmp/parachain-buildx-cache cache-to: type=local,dest=/tmp/parachain-buildx-cache-new,mode=max - name: Move cache - # Temp fix + if: needs.check-file-change.outputs.src == 'true' + # Temp fix for constantly growing cache, see # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 run: | rm -rf /tmp/parachain-buildx-cache mv /tmp/parachain-buildx-cache-new /tmp/parachain-buildx-cache - # - name: Build docker image - # if: needs.check-file-change.outputs.src == 'true' - # timeout-minutes: 40 - # run: | - # ./scripts/build-docker.sh - # echo "=============================" - # docker images - - name: Pull docker image optinally if: needs.check-file-change.outputs.src == 'false' run: | diff --git a/.github/workflows/build_docker_with_args.yml b/.github/workflows/build_docker_with_args.yml index d8c3825dd2..cc526cf4e5 100644 --- a/.github/workflows/build_docker_with_args.yml +++ b/.github/workflows/build_docker_with_args.yml @@ -3,6 +3,12 @@ name: Build docker with args on: workflow_dispatch: inputs: + dockerfile_type: + type: choice + description: The type of Dockerfile to use + options: + - dev + - prod docker_tag: description: The tag for the built docker image required: true @@ -24,7 +30,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}" + ./scripts/build-docker.sh ${{ github.event.inputs.dockerfile_type }} ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}" echo "=============================" docker images diff --git a/.github/workflows/create_release_draft.yml b/.github/workflows/create_release_draft.yml index b0b9a58faf..c1abadf842 100644 --- a/.github/workflows/create_release_draft.yml +++ b/.github/workflows/create_release_draft.yml @@ -85,7 +85,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh ${{ env.RELEASE_TAG }} + ./scripts/build-docker.sh prod ${{ env.RELEASE_TAG }} echo "=============================" docker images diff --git a/Makefile b/Makefile index 5771432488..cfecb30026 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,13 @@ srtool-build-wasm-litentry: srtool-build-wasm-litmus: @./scripts/build-wasm.sh litmus -.PHONY: build-docker ## Build docker image -build-docker: - @./scripts/build-docker.sh +.PHONY: build-docker-dev ## Build docker image using Dockerfile.dev +build-docker-dev: + @./scripts/build-docker.sh dev + +.PHONY: build-docker-prod ## Build docker image using Dockerfile.prod +build-docker-prod: + @./scripts/build-docker.sh prod .PHONY: build-node-benchmarks ## Build release node with `runtime-benchmarks` feature build-node-benchmarks: diff --git a/README.md b/README.md index e30bda9c72..3e0749c416 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,12 @@ make build-node To build the `litentry/litentry-parachain` docker image locally: ``` -make build-docker +make build-docker-de +or +make build-docker-prod ``` +depending on which Dockerfile should be used. The dev version has integrated `cargo-chef` for caching. +As of funtionalities they should be identical. To build the litentry-parachain runtime wasm manually: ``` diff --git a/docker/Dockerfile b/docker/Dockerfile.dev similarity index 97% rename from docker/Dockerfile rename to docker/Dockerfile.dev index efad9b37f3..793cc60a45 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile.dev @@ -38,7 +38,6 @@ RUN cargo chef cook --release --recipe-path recipe.json COPY . /litentry ARG BUILD_ARGS RUN cargo build --release $BUILD_ARGS -RUN strip target/release/litentry-collator # =================================== # stage 2: packaging diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod new file mode 100644 index 0000000000..53fb731308 --- /dev/null +++ b/docker/Dockerfile.prod @@ -0,0 +1,46 @@ +# ========================== +# stage 1: build +# https://hub.docker.com/_/rust +# our host is based on bullseye/sid => rust:latest +# ========================== +FROM rust:latest as builder +WORKDIR /litentry +COPY . /litentry + +# always use the toolchain in base image +# we do not fix toolchain version as it's still evolving rapidly +RUN rustup default nightly && \ + rustup target add wasm32-unknown-unknown --toolchain nightly + +RUN apt-get update && \ + apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ + apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev + +ARG BUILD_ARGS +RUN cargo build --release $BUILD_ARGS +RUN strip target/release/litentry-collator + +# ========================== +# stage 2: packaging +# ========================== + +FROM ubuntu:20.04 +LABEL maintainer="litentry-dev" + +COPY --from=builder /litentry/target/release/litentry-collator /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /litentry litentry && \ + mkdir -p /data /litentry/.local/share && \ + chown -R litentry:litentry /data && \ + ln -s /data /litentry/.local/share/litentry-collator && \ +# unclutter and minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ +# check if executable works in this container + /usr/local/bin/litentry-collator --version + +USER litentry +EXPOSE 30333 9933 9944 9615 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/local/bin/litentry-collator"] +CMD ["--help"] \ No newline at end of file diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index aad487219e..3a8def8d4b 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash set -eo pipefail +function usage() { + echo "Usage: $0 dev|prod [docker-tag] [build-args]" +} + +[ $# -lt 1 ] && (usage; exit 1) + ROOTDIR=$(git rev-parse --show-toplevel) cd "$ROOTDIR" -TAG="$1" -ARGS="$2" +TYPE="$1" +TAG="$2" +ARGS="$3" if [ -z "$TAG" ]; then TAG_COMMIT=`git rev-list --tags --max-count=1` @@ -20,6 +27,7 @@ if [ -z "$TAG" ]; then fi fi +echo "TYPE: $TYPE" echo "TAG: $TAG" echo "ARGS: $ARGS" @@ -29,7 +37,7 @@ GITREPO=litentry-parachain # Build the image echo "------------------------------------------------------------" echo "Building ${GITUSER}/${GITREPO}:${TAG} docker image ..." -docker build --rm --no-cache --pull -f ./docker/Dockerfile \ +docker build --rm --no-cache --pull -f ./docker/Dockerfile.${TYPE} \ --build-arg BUILD_ARGS="$ARGS" \ -t ${GITUSER}/${GITREPO}:${TAG} . From 4d2e801d80185827de4406a0a3b34010e39e9881 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 11:52:00 +0100 Subject: [PATCH 34/38] more comment and TYPE --- .github/workflows/build_and_run_test.yml | 6 ++++-- scripts/build-docker.sh | 10 +++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 6df103ceee..8f8f1e5b56 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -86,8 +86,10 @@ jobs: run: make clippy build-docker: - # run the docker build on our self-hosted runner, which takes < 20min without cache - # on a standard github runner it takes ~1 hour + # run the docker build on our self-hosted runner, which takes + # - 25min without cache, or + # - 15 min with cache + # on a standard github runner it takes 1 hour+ runs-on: self-hosted needs: [check-cargo-fmt, check-file-change] steps: diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index 3a8def8d4b..6eabf153f0 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -14,6 +14,14 @@ TYPE="$1" TAG="$2" ARGS="$3" +NOCACHE_FLAG= + +case "$TYPE" in + dev) ;; + prod) NOCACHE_FLAG="--no-cache" ;; + *) usage; exit 1 ;; +esac + if [ -z "$TAG" ]; then TAG_COMMIT=`git rev-list --tags --max-count=1` HEAD_COMMIT=`git rev-parse HEAD` @@ -37,7 +45,7 @@ GITREPO=litentry-parachain # Build the image echo "------------------------------------------------------------" echo "Building ${GITUSER}/${GITREPO}:${TAG} docker image ..." -docker build --rm --no-cache --pull -f ./docker/Dockerfile.${TYPE} \ +docker build --rm ${NOCACHE_FLAG} --pull -f ./docker/Dockerfile.${TYPE} \ --build-arg BUILD_ARGS="$ARGS" \ -t ${GITUSER}/${GITREPO}:${TAG} . From 77f8de957c8306eb987b8479ba76d012fe94cb7b Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 11:52:48 +0100 Subject: [PATCH 35/38] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 04e7b5545a..8a7534fbc2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ docker/generated-*/ ts-tests/package-lock.json tags.lock + +recipe.json From 91ea5badf2b0a79586882773b0336f961d43b4c8 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 11:55:44 +0100 Subject: [PATCH 36/38] more comment --- .github/workflows/build_and_run_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 8f8f1e5b56..38d2111cc7 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -90,6 +90,7 @@ jobs: # - 25min without cache, or # - 15 min with cache # on a standard github runner it takes 1 hour+ + # see also https://github.com/litentry/litentry-parachain/issues/259 runs-on: self-hosted needs: [check-cargo-fmt, check-file-change] steps: From c6edb4b01004e896c4aa14e983f33e903b822783 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Thu, 17 Mar 2022 13:34:44 +0100 Subject: [PATCH 37/38] update README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e0749c416..6b0117215f 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,12 @@ make build-node To build the `litentry/litentry-parachain` docker image locally: ``` -make build-docker-de +make build-docker-dev or make build-docker-prod ``` -depending on which Dockerfile should be used. The dev version has integrated `cargo-chef` for caching. +depending on which Dockerfile should be used: [Dockerfile.dev](./docker/Dockerfile.dev) or [Dockerfile.prod](./docker/Dockerfile.prod). +The dev version has integrated [`cargo-chef`](https://github.com/LukeMathWalker/cargo-chef) for caching. As of funtionalities they should be identical. To build the litentry-parachain runtime wasm manually: From 8a8fe2f002098506d5d3283a6c833ef7e7b8ac91 Mon Sep 17 00:00:00 2001 From: Kailai Wang Date: Tue, 22 Mar 2022 02:05:01 +0100 Subject: [PATCH 38/38] fix typo --- docker/Dockerfile.dev | 3 +-- docker/Dockerfile.prod | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 793cc60a45..2d5d6ca941 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -40,9 +40,8 @@ ARG BUILD_ARGS RUN cargo build --release $BUILD_ARGS # =================================== -# stage 2: packaging +# stage 4: packaging # =================================== - FROM ubuntu:20.04 LABEL maintainer="litentry-dev" diff --git a/docker/Dockerfile.prod b/docker/Dockerfile.prod index 53fb731308..d4e3795374 100644 --- a/docker/Dockerfile.prod +++ b/docker/Dockerfile.prod @@ -23,7 +23,6 @@ RUN strip target/release/litentry-collator # ========================== # stage 2: packaging # ========================== - FROM ubuntu:20.04 LABEL maintainer="litentry-dev"