LAst resort - dind - vm

.github/env

@@ -1 +1,2 @@
 IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
+RUST_NIGHTLY_VERSION="2024-09-11"

.github/scripts/cmd/cmd.py

@@ -6,6 +6,7 @@
 import argparse
 import _help
 import importlib.util
+import re
 
 _HelpAction = _help._HelpAction
 
@@ -40,20 +41,20 @@ def setup_logging():
 setup_logging()
 
 """
-BENCH 
+BENCH
 """
 
 bench_example = '''**Examples**:
- Runs all benchmarks 
+ Runs all benchmarks
  %(prog)s
 
  Runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets. **--quiet** makes it to output nothing to PR but reactions
  %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet
- 
+
  Runs bench for all pallets for westend runtime and fails fast on first failed benchmark
  %(prog)s --runtime westend --fail-fast
- 
- Does not output anything and cleans up the previous bot's & author command triggering comments in PR 
+
+ Does not output anything and cleans up the previous bot's & author command triggering comments in PR
  %(prog)s --runtime westend rococo --pallet pallet_balances pallet_multisig --quiet --clean
 '''
 
@@ -66,15 +67,44 @@ def setup_logging():
 parser_bench.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[])
 parser_bench.add_argument('--fail-fast', help='Fail fast on first failed benchmark', action='store_true')
 
+
 """
-FMT 
+BENCH OLD
+"""
+
+bench_example = '''**Examples**:
+ Runs all benchmarks 
+ %(prog)s
+
+ Runs benchmarks for pallet_balances and pallet_multisig for all runtimes which have these pallets. **--quiet** makes it to output nothing to PR but reactions
+ %(prog)s --pallet pallet_balances pallet_xcm_benchmarks::generic --quiet
+
+ Runs bench for all pallets for westend runtime and fails fast on first failed benchmark
+ %(prog)s --runtime westend --fail-fast
+
+ Does not output anything and cleans up the previous bot's & author command triggering comments in PR 
+ %(prog)s --runtime westend rococo --pallet pallet_balances pallet_multisig --quiet --clean
+'''
+
+parser_bench_old = subparsers.add_parser('bench-old', help='Runs benchmarks (Old CLI)', epilog=bench_example, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+for arg, config in common_args.items():
+    parser_bench_old.add_argument(arg, **config)
+
+parser_bench_old.add_argument('--runtime', help='Runtime(s) space separated', choices=runtimeNames, nargs='*', default=runtimeNames)
+parser_bench_old.add_argument('--pallet', help='Pallet(s) space separated', nargs='*', default=[])
+parser_bench_old.add_argument('--fail-fast', help='Fail fast on first failed benchmark', action='store_true')
+
+
+"""
+FMT
 """
 parser_fmt = subparsers.add_parser('fmt', help='Formats code (cargo +nightly-VERSION fmt) and configs (taplo format)')
 for arg, config in common_args.items():
     parser_fmt.add_argument(arg, **config)
 
 """
-Update UI 
+Update UI
 """
 parser_ui = subparsers.add_parser('update-ui', help='Updates UI tests')
 for arg, config in common_args.items():
@@ -102,7 +132,7 @@ def main():
         failed_benchmarks = {}
         successful_benchmarks = {}
 
-        profile = "release"
+        profile = "production"
 
         print(f'Provided runtimes: {args.runtime}')
         # convert to mapped dict
@@ -112,11 +142,22 @@ def main():
 
         # loop over remaining runtimes to collect available pallets
         for runtime in runtimesMatrix.values():
-            os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features={runtime['bench_features']}")
+            build_command = f"cargo build -p {runtime['package']} --profile {profile} --features={runtime['bench_features']}"
+            print(f'-- building "{runtime["name"]}" with `{build_command}`')
+            os.system(build_command)
             print(f'-- listing pallets for benchmark for {runtime["name"]}')
             wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
-            output = os.popen(
-                f"frame-omni-bencher v1 benchmark pallet --no-csv-header --no-storage-info --no-min-squares --no-median-slopes --all --list --runtime={wasm_file} {runtime['bench_flags']}").read()
+            list_command = f"frame-omni-bencher v1 benchmark pallet " \
+                f"--no-csv-header " \
+                f"--no-storage-info " \
+                f"--no-min-squares " \
+                f"--no-median-slopes " \
+                f"--all " \
+                f"--list " \
+                f"--runtime={wasm_file} " \
+                f"{runtime['bench_flags']}"
+            print(f'-- running: {list_command}')
+            output = os.popen(list_command).read()
             raw_pallets = output.strip().split('\n')
 
             all_pallets = set()
@@ -175,7 +216,15 @@ def main():
                     print(f'-- package_dir: {package_dir}')
                     print(f'-- manifest_path: {manifest_path}')
                     output_path = os.path.join(package_dir, "src", "weights.rs")
+                    # TODO: we can remove once all pallets in dev runtime are migrated to polkadot-sdk-frame
+                    try:
+                        uses_polkadot_sdk_frame = "true" in os.popen(f"cargo metadata --locked --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .dependencies | any(.name == \"polkadot-sdk-frame\")'").read()
+                    # Empty output from the previous os.popen command
+                    except StopIteration:
+                        uses_polkadot_sdk_frame = False
                     template = config['template']
+                    if uses_polkadot_sdk_frame and re.match(r"frame-(:?umbrella-)?weight-template\.hbs", os.path.normpath(template).split(os.path.sep)[-1]):
+                        template = "substrate/.maintain/frame-umbrella-weight-template.hbs"
                 else:
                     default_path = f"./{config['path']}/src/weights"
                     xcm_path = f"./{config['path']}/src/weights/xcm"
@@ -221,6 +270,149 @@ def main():
             print_and_log('✅ Successful benchmarks of runtimes/pallets:')
             for runtime, pallets in successful_benchmarks.items():
                 print_and_log(f'-- {runtime}: {pallets}')
+
+    if args.command == 'bench-old':
+        runtime_pallets_map = {}
+        failed_benchmarks = {}
+        successful_benchmarks = {}
+
+        profile = "release"
+
+        print(f'Provided runtimes: {args.runtime}')
+        # convert to mapped dict
+        runtimesMatrix = list(filter(lambda x: x['name'] in args.runtime, runtimesMatrix))
+        runtimesMatrix = {x['name']: x for x in runtimesMatrix}
+        print(f'Filtered out runtimes: {runtimesMatrix}')
+
+        # loop over remaining runtimes to collect available pallets
+        for runtime in runtimesMatrix.values():
+            build_command = f"cargo build -p {runtime['old_package']} --profile {profile} --features={runtime['bench_features']} --locked"
+            print(f'-- building {runtime["name"]} with `{build_command}`')
+            os.system(build_command)
+
+            chain = runtime['name'] if runtime == 'dev' else f"{runtime['name']}-dev"
+
+            machine_test = f"target/{profile}/{runtime['old_bin']} benchmark machine --chain={chain}"
+            print(f"Running machine test for `{machine_test}`")
+            os.system(machine_test)
+
+            print(f'-- listing pallets for benchmark for {chain}')
+            list_command = f"target/{profile}/{runtime['old_bin']} " \
+                f"benchmark pallet " \
+                f"--no-csv-header " \
+                f"--no-storage-info " \
+                f"--no-min-squares " \
+                f"--no-median-slopes " \
+                f"--all " \
+                f"--list " \
+                f"--chain={chain}"
+            print(f'-- running: {list_command}')
+            output = os.popen(list_command).read()
+            raw_pallets = output.strip().split('\n')
+
+            all_pallets = set()
+            for pallet in raw_pallets:
+                if pallet:
+                    all_pallets.add(pallet.split(',')[0].strip())
+
+            pallets = list(all_pallets)
+            print(f'Pallets in {runtime["name"]}: {pallets}')
+            runtime_pallets_map[runtime['name']] = pallets
+
+        print(f'\n')
+
+        # filter out only the specified pallets from collected runtimes/pallets
+        if args.pallet:
+            print(f'Pallets: {args.pallet}')
+            new_pallets_map = {}
+            # keep only specified pallets if they exist in the runtime
+            for runtime in runtime_pallets_map:
+                if set(args.pallet).issubset(set(runtime_pallets_map[runtime])):
+                    new_pallets_map[runtime] = args.pallet
+
+            runtime_pallets_map = new_pallets_map
+
+        print(f'Filtered out runtimes & pallets: {runtime_pallets_map}\n')
+
+        if not runtime_pallets_map:
+            if args.pallet and not args.runtime:
+                print(f"No pallets {args.pallet} found in any runtime")
+            elif args.runtime and not args.pallet:
+                print(f"{args.runtime} runtime does not have any pallets")
+            elif args.runtime and args.pallet:
+                print(f"No pallets {args.pallet} found in {args.runtime}")
+            else:
+                print('No runtimes found')
+            sys.exit(1)
+
+        for runtime in runtime_pallets_map:
+            for pallet in runtime_pallets_map[runtime]:
+                config = runtimesMatrix[runtime]
+                header_path = os.path.abspath(config['header'])
+                template = None
+
+                chain = runtime['name'] if runtime == 'dev' else f"{config['name']}-dev"
+
+                print(f'-- config: {config}')
+                if runtime == 'dev':
+                    # to support sub-modules (https://github.com/paritytech/command-bot/issues/275)
+                    search_manifest_path = f"cargo metadata --locked --format-version 1 --no-deps | jq -r '.packages[] | select(.name == \"{pallet.replace('_', '-')}\") | .manifest_path'"
+                    print(f'-- running: {search_manifest_path}')
+                    manifest_path = os.popen(search_manifest_path).read()
+                    if not manifest_path:
+                        print(f'-- pallet {pallet} not found in dev runtime')
+                        if args.fail_fast:
+                            print_and_log(f'Error: {pallet} not found in dev runtime')
+                            sys.exit(1)
+                    package_dir = os.path.dirname(manifest_path)
+                    print(f'-- package_dir: {package_dir}')
+                    print(f'-- manifest_path: {manifest_path}')
+                    output_path = os.path.join(package_dir, "src", "weights.rs")
+                    template = config['template']
+                else:
+                    default_path = f"./{config['path']}/src/weights"
+                    xcm_path = f"./{config['path']}/src/weights/xcm"
+                    output_path = default_path
+                    if pallet.startswith("pallet_xcm_benchmarks"):
+                        template = config['template']
+                        output_path = xcm_path
+
+                print(f'-- benchmarking {pallet} in {runtime} into {output_path}')
+                cmd = f"target/{profile}/{config['old_bin']} benchmark pallet " \
+                    f"--extrinsic=* " \
+                    f"--chain={chain} " \
+                    f"--pallet={pallet} " \
+                    f"--header={header_path} " \
+                    f"--output={output_path} " \
+                    f"--wasm-execution=compiled " \
+                    f"--steps=50 " \
+                    f"--repeat=20 " \
+                    f"--heap-pages=4096 " \
+                    f"{f'--template={template} ' if template else ''}" \
+                    f"--no-storage-info --no-min-squares --no-median-slopes "
+                print(f'-- Running: {cmd} \n')
+                status = os.system(cmd)
+
+                if status != 0 and args.fail_fast:
+                    print_and_log(f'❌ Failed to benchmark {pallet} in {runtime}')
+                    sys.exit(1)
+
+                # Otherwise collect failed benchmarks and print them at the end
+                # push failed pallets to failed_benchmarks
+                if status != 0:
+                    failed_benchmarks[f'{runtime}'] = failed_benchmarks.get(f'{runtime}', []) + [pallet]
+                else:
+                    successful_benchmarks[f'{runtime}'] = successful_benchmarks.get(f'{runtime}', []) + [pallet]
+
+        if failed_benchmarks:
+            print_and_log('❌ Failed benchmarks of runtimes/pallets:')
+            for runtime, pallets in failed_benchmarks.items():
+                print_and_log(f'-- {runtime}: {pallets}')
+
+        if successful_benchmarks:
+            print_and_log('✅ Successful benchmarks of runtimes/pallets:')
+            for runtime, pallets in successful_benchmarks.items():
+                print_and_log(f'-- {runtime}: {pallets}')
 
     elif args.command == 'fmt':
         command = f"cargo +nightly fmt"
@@ -251,4 +443,4 @@ def main():
     print('🚀 Done')
 
 if __name__ == '__main__':
-    main()
+    main()

.github/workflows/cmd.yml

@@ -231,7 +231,7 @@ jobs:
           fi
 
           if [[ $BODY == "/cmd bench"* ]]; then
-              echo "RUNNER=parity-weights" >> $GITHUB_OUTPUT
+              echo "RUNNER=cmd-bot-vm" >> $GITHUB_OUTPUT
           elif [[ $BODY == "/cmd update-ui"* ]]; then
               echo "RUNNER=parity-large" >> $GITHUB_OUTPUT
           else
@@ -284,9 +284,29 @@ jobs:
       JOB_NAME: "cmd"
     runs-on: ${{ needs.set-image.outputs.RUNNER }}
     timeout-minutes: 4320 # 72 hours -> 3 days; as it could take a long time to run all the runtimes/pallets
-    container:
-      image: ${{ needs.set-image.outputs.IMAGE }}
     steps:
+      - name: Install updates and protobuf-compiler
+        run: |
+          sudo apt update && sudo apt install --assume-yes \
+            openssl pkg-config g++ make cmake protobuf-compiler curl libssl-dev libclang-dev libudev-dev git jq python3 python3-pip
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ needs.get-pr-branch.outputs.repo }}
+          ref: ${{ needs.get-pr-branch.outputs.pr-branch }}
+
+      - name: Set rust version via common env file
+        run: |
+          cat .github/env >> $GITHUB_ENV
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          targets: "wasm32-unknown-unknown,x86_64-unknown-linux-musl"
+          components: "rust-src rustfmt clippy"
+          toolchain: "nightly-${{env.RUST_NIGHTLY_VERSION}}"
+
       - name: Get command
         uses: actions-ecosystem/action-regex-match@v2
         id: get-pr-comment
@@ -340,12 +360,12 @@ jobs:
               repo: context.repo.repo,
               body: `Command "${{ steps.get-pr-comment.outputs.group2 }}" has started 🚀 [See logs here](${job_url})`
             })
-
-      - name: Checkout
-        uses: actions/checkout@v4
+
+      - name: Cache Rust dependencies
+        if: startsWith(steps.get-pr-comment.outputs.group2, 'bench')
+        uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
         with:
-          repository: ${{ needs.get-pr-branch.outputs.repo }}
-          ref: ${{ needs.get-pr-branch.outputs.pr-branch }}
+          cache-on-failure: true
 
       - name: Install dependencies for bench
         if: startsWith(steps.get-pr-comment.outputs.group2, 'bench')
@@ -364,6 +384,7 @@ jobs:
           # Fixes "detected dubious ownership" error in the ci
           git config --global --add safe.directory '*'
           git remote -v
+          cat /proc/cpuinfo
           python3 -m pip install -r .github/scripts/generate-prdoc.requirements.txt
           python3 .github/scripts/cmd/cmd.py $CMD $PR_ARG
           git status