Merge branch 'master' into ankan/fix-np-reward-deficit

.github/dependabot.yml

@@ -0,0 +1,27 @@
+version: 2
+updates:
+  # Update github actions:
+  - package-ecosystem: github-actions
+    directory: '/'
+    labels: ["A1-insubstantial", "R0-silent"]
+    schedule:
+      interval: daily
+  # Update Rust dependencies:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    labels: ["A1-insubstantial", "R0-silent"]
+    schedule:
+      interval: "daily"
+    groups:
+      # We assume these crates to be semver abiding and can therefore group them together.
+      known_good_semver:
+        patterns:
+        - "syn"
+        - "quote"
+        - "log"
+        - "paste"
+        - "*serde*"
+        - "clap"
+        update-types:
+        - "minor"
+        - "patch"

.github/workflows/check-licenses.yml

@@ -9,10 +9,9 @@ permissions:
 jobs:
   check-licenses:
     runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      matrix:
-        repo: [polkadot, substrate, cumulus]
+    env:
+      LICENSES: "'Apache-2.0' 'GPL-3.0-only' 'GPL-3.0-or-later WITH Classpath-exception-2.0'"
+      NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -21,16 +20,26 @@ jobs:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"
           scope: "@paritytech"
-      - name: Check the licenses for ${{ matrix.repo }}
+
+      - name: Check the licenses in Polkadot
         run: |
           shopt -s globstar
-          echo "install"
-          npm install -g @paritytech/[email protected]
-          echo "run for ${{ matrix.repo }}"
-          cd ${{ matrix.repo }}
           npx @paritytech/license-scanner scan \
-            --ensure-licenses=Apache-2.0 \
-            --ensure-licenses=GPL-3.0-only \
-            ./**/*.rs
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+            --ensure-licenses ${{ env.LICENSES }} \
+            -- ./polkadot/**/*.rs
+
+      - name: Check the licenses in Cumulus
+        run: |
+          shopt -s globstar
+          npx @paritytech/license-scanner scan \
+            --ensure-licenses ${{ env.LICENSES }} \
+            --exclude ./cumulus/parachain-template \
+            -- ./cumulus/**/*.rs
+
+      - name: Check the licenses in Substrate
+        run: |
+          shopt -s globstar
+          npx @paritytech/license-scanner scan \
+            --ensure-licenses ${{ env.LICENSES }} \
+            --exclude ./substrate/bin/node-template \
+            -- ./substrate/**/*.rs

.gitlab-ci.yml

@@ -22,7 +22,7 @@ workflow:
 
 variables:
   CI_IMAGE: "paritytech/ci-unified:bullseye-1.70.0-2023-05-23-v20230706"
-  BUILDAH_IMAGE: "quay.io/buildah/stable:v1.29"
+  # BUILDAH_IMAGE is defined in group variables
   BUILDAH_COMMAND: "buildah --storage-driver overlay2"
   RELENG_SCRIPTS_BRANCH: "master"
   RUSTY_CACHIER_SINGLE_BRANCH: master
@@ -333,3 +333,58 @@ cancel-pipeline-cargo-clippy:
   extends: .cancel-pipeline-template
   needs:
     - job: cargo-clippy
+
+cancel-pipeline-build-linux-stable:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: build-linux-stable
+
+cancel-pipeline-build-linux-stable-cumulus:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: build-linux-stable-cumulus
+
+cancel-pipeline-build-linux-substrate:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: build-linux-substrate
+
+cancel-pipeline-test-node-metrics:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: test-node-metrics
+
+cancel-pipeline-test-frame-ui:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: test-frame-ui
+
+cancel-pipeline-quick-benchmarks:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: quick-benchmarks
+
+cancel-pipeline-check-try-runtime:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: check-try-runtime
+
+cancel-pipeline-test-frame-examples-compile-to-wasm:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: test-frame-examples-compile-to-wasm
+
+cancel-pipeline-build-short-benchmark:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: build-short-benchmark
+
+cancel-pipeline-check-runtime-migration-rococo:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: check-runtime-migration-rococo
+
+cancel-pipeline-check-runtime-migration-westend:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: check-runtime-migration-westend

.gitlab/pipeline/build.yml

@@ -88,7 +88,7 @@ build-staking-miner:
     - job: build-malus
       artifacts: false
   script:
-    - time cargo build --locked --release --package staking-miner
+    - time cargo build -q --locked --release --package staging-staking-miner
     # # pack artifacts
     # - mkdir -p ./artifacts
     # - mv ./target/release/staking-miner ./artifacts/.
@@ -146,7 +146,7 @@ build-short-benchmark:
     - .run-immediately
     - .collect-artifacts
   script:
-    - cargo build --profile release --locked --features=runtime-benchmarks
+    - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot
     - mkdir -p artifacts
     - target/release/polkadot --version
     - cp ./target/release/polkadot ./artifacts/
@@ -265,6 +265,19 @@ build-runtime-testing:
     - job: build-runtime-starters
       artifacts: false
 
+build-short-benchmark-cumulus:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  script:
+    - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot-parachain
+    - mkdir -p artifacts
+    - target/release/polkadot-parachain --version
+    - cp ./target/release/polkadot-parachain ./artifacts/
+
 # substrate
 
 build-linux-substrate:
@@ -353,4 +366,3 @@ build-subkey-linux:
 #   after_script: [""]
 #   tags:
 #     - osx
-

.gitlab/pipeline/check.yml

@@ -121,30 +121,6 @@ test-rust-feature-propagation:
           --runtime ./target/release/wbuild/"$NETWORK"-runtime/target/wasm32-unknown-unknown/release/"$NETWORK"_runtime.wasm \
           on-runtime-upgrade --checks=pre-and-post live --uri wss://${NETWORK}-try-runtime-node.parity-chains.parity.io:443
 
-check-runtime-migration-polkadot:
-  stage: check
-  extends:
-    - .docker-env
-    - .test-pr-refs
-    - .check-runtime-migration
-  variables:
-    NETWORK: "polkadot"
-  allow_failure: true # FIXME https://github.com/paritytech/substrate/issues/13107
-
-check-runtime-migration-kusama:
-  stage: check
-  # DAG
-  needs:
-    - job: check-runtime-migration-polkadot
-      artifacts: false
-  extends:
-    - .docker-env
-    - .test-pr-refs
-    - .check-runtime-migration
-  variables:
-    NETWORK: "kusama"
-  allow_failure: true # FIXME https://github.com/paritytech/substrate/issues/13107
-
 check-runtime-migration-westend:
   stage: check
   extends:
@@ -153,7 +129,6 @@ check-runtime-migration-westend:
     - .check-runtime-migration
   variables:
     NETWORK: "westend"
-  allow_failure: true # FIXME https://github.com/paritytech/substrate/issues/13107
 
 check-runtime-migration-rococo:
   stage: check
@@ -167,7 +142,6 @@ check-runtime-migration-rococo:
     - .check-runtime-migration
   variables:
     NETWORK: "rococo"
-  allow_failure: true # FIXME https://github.com/paritytech/substrate/issues/13107
 
 find-fail-ci-phrase:
   stage: check

.gitlab/pipeline/short-benchmarks.yml

@@ -2,6 +2,9 @@
 # Here are all jobs that are executed during "short-benchmarks" stage
 
 # Run all pallet benchmarks only once to check if there are any errors
+
+# run short-benchmarks for relay chain runtimes from polkadot
+
 short-benchmark-polkadot: &short-bench
   stage: short-benchmarks
   extends:
@@ -12,6 +15,8 @@ short-benchmark-polkadot: &short-bench
       artifacts: true
   variables:
     RUNTIME: polkadot
+  tags:
+    - benchmark
   script:
     - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
 
@@ -24,3 +29,60 @@ short-benchmark-westend:
   <<: *short-bench
   variables:
     RUNTIME: westend
+
+# run short-benchmarks for system parachain runtimes from cumulus
+
+.short-benchmark-cumulus: &short-bench-cumulus
+  stage: short-benchmarks
+  extends:
+    - .common-refs
+    - .docker-env
+  needs:
+    - job: build-short-benchmark-cumulus
+      artifacts: true
+  variables:
+    RUNTIME_CHAIN: benchmarked-runtime-chain
+  tags:
+    - benchmark
+  script:
+    - ./artifacts/polkadot-parachain benchmark pallet --wasm-execution compiled --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+
+short-benchmark-asset-hub-polkadot:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: asset-hub-polkadot-dev
+
+short-benchmark-asset-hub-kusama:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: asset-hub-kusama-dev
+
+short-benchmark-asset-hub-westend:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: asset-hub-westend-dev
+
+short-benchmark-bridge-hub-polkadot:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: bridge-hub-polkadot-dev
+
+short-benchmark-bridge-hub-kusama:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: bridge-hub-kusama-dev
+
+short-benchmark-bridge-hub-rococo:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: bridge-hub-rococo-dev
+
+short-benchmark-collectives-polkadot:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: collectives-polkadot-dev
+
+short-benchmark-glutton-kusama:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: glutton-kusama-dev-1300

.gitlab/pipeline/test.yml

@@ -147,7 +147,10 @@ test-doc:
   extends:
     - .docker-env
     - .common-refs
-    - .run-immediately
+  # DAG
+  needs:
+    - job: test-rustdoc
+      artifacts: false
   variables:
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
@@ -160,10 +163,7 @@ test-rustdoc:
   extends:
     - .docker-env
     - .common-refs
-  # DAG
-  needs:
-    - job: test-doc
-      artifacts: false
+    - .run-immediately
   variables:
     SKIP_WASM_BUILD: 1
     RUSTDOCFLAGS: "-Dwarnings"
@@ -196,6 +196,8 @@ test-node-metrics:
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
   script:
+    # Build the required workers.
+    - cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
     - mkdir -p artifacts
     - time cargo test --profile testnet
       --locked
@@ -210,7 +212,7 @@ test-deterministic-wasm:
     - .common-refs
   # DAG
   needs:
-    - job: test-frame-support
+    - job: test-frame-ui
       artifacts: false
   script:
     - .gitlab/test_deterministic_wasm.sh
@@ -293,7 +295,7 @@ node-bench-regression-guard:
 
 # if this fails (especially after rust version upgrade) run
 # ./substrate/.maintain/update-rust-stable.sh <rust_version>
-test-frame-support:
+test-frame-ui:
   stage: test
   extends:
     - .docker-env
@@ -312,9 +314,8 @@ test-frame-support:
     # Ensure we run the UI tests.
     RUN_UI_TESTS: 1
   script:
-    - time cargo test --locked -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental --manifest-path ./substrate/frame/support/test/Cargo.toml
-    - time cargo test --locked -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental --manifest-path ./substrate/frame/support/test/Cargo.toml
-    - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --locked -- --ignored timeout
+    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
+    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
     - cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
 
 # This job runs all benchmarks defined in the `/bin/node/runtime` once to check that there are no errors.