Merge remote-tracking branch 'origin/compatible' into georgeee/fix-16…

…397-mini

buildkite/scripts/bench/install.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+sudo apt-get update
+sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
+
+TESTNET_NAME="berkeley"
+
+git config --global --add safe.directory /workdir
+source buildkite/scripts/export-git-env-vars.sh
+
+source buildkite/scripts/debian/install.sh "mina-test-suite,mina-$TESTNET_NAME" 1
+
+pip3 install -r scripts/benchmarks/requirements.txt

buildkite/scripts/bench/snark_transaction_profiler.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -eo pipefail
+
+source buildkite/scripts/bench/install.sh
+
+K=1
+MAX_NUM_UPDATES=4
+MIN_NUM_UPDATES=2   
+
+echo "-- Run Snark Transaction Profiler with parameters: --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}"
+python3 ./scripts/benchmarks run --benchmark snark --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES} --outfile snark.out

buildkite/scripts/bench/zkapp_metrics.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eo pipefail
+
+source buildkite/scripts/bench/install.sh
+
+python3 ./scripts/benchmarks run --benchmark zkapp --outfile zakpp-out
+
+python3 ./scripts/benchmarks run --benchmark heap-usage --outfile heap-usage.out

buildkite/scripts/build-artifact.sh

@@ -23,7 +23,7 @@ make -C src/app/libp2p_helper
 MAINNET_TARGETS=""
 [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe"
 
-echo "--- Build all major tagets required for packaging"
+echo "--- Build all major targets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
 echo "Rust Version: $(rustc --version)"
 dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
@@ -44,4 +44,11 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
   src/app/rosetta/indexer_test/indexer_test.exe \
   src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe  \
-  src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
+  src/app/benchmarks/benchmarks.exe \
+  src/app/ledger_export_bench/ledger_export_benchmark.exe \
+  src/app/disk_caching_stats/disk_caching_stats.exe \
+  src/app/heap_usage/heap_usage.exe \
+  src/app/zkapp_limits/zkapp_limits.exe \
+  src/lib/snark_worker/standalone/run_snark_worker.exe \
+  src/test/command_line_tests/command_line_tests.exe \
+  src/test/archive/patch_archive_test/patch_archive_test.exe

buildkite/scripts/debian/install.sh

@@ -34,7 +34,7 @@ else
   debs=(${DEBS//,/ })
   for i in "${debs[@]}"; do
     case $i in
-      mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet)
+      mina-berkeley*|mina-devnet|mina-mainnet)
         # Downaload mina-logproc too
         source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER
       ;;

buildkite/scripts/debian/promote.sh

@@ -12,6 +12,9 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   -s|--from-component) FROM_COMPONENT="$2"; shift;;
   -t|--to-component) TO_COMPONENT="$2"; shift;;
   --new-name) NEW_NAME="$2"; shift;;
+  --new-repo) NEW_REPO="$2"; shift;;
+  -r|--repo) REPO="$2"; shift;;
+  --repo-key) REPO_KEY="$2"; shift;;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
 
@@ -28,6 +31,9 @@ function usage() {
   echo "  -s, --from-component   The source channel in which package currently resides"
   echo "  -t, --to-component     The target channel for package (unstable, alpha, beta etc.)"
   echo "  -c, --codename         The Debian codename (bullseye, focal etc.)"
+  echo "  -r, --repo             The Debian source repo"
+  echo "  --new-repo             The Debian target repo. By default equal to repo"
+  echo "  --repo-key         The Debian target repo key"
   echo ""
   echo "Example: $0 --package mina-archive --version 2.0.0-rc1-48efea4 --architecture amd64 --codename bullseye --from-component unstable --to-component nightly"
   exit 1
@@ -40,6 +46,9 @@ if [[ -z "$CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$NEW_NAME" ]]; then NEW_NAME=$PACKAGE; fi;
 if [[ -z "$FROM_COMPONENT" ]]; then usage "Source component is not set!"; fi;
 if [[ -z "$TO_COMPONENT" ]]; then usage "Target component is not set!"; fi;
+if [[ -z "$REPO" ]]; then usage "Repository is not set!"; fi;
+if [[ -z "$NEW_REPO" ]]; then NEW_REPO=$REPO; fi;
+if [[ -z "$REPO_KEY" ]]; then usage "Target repository key is not set!"; fi;
 
 # check for AWS Creds
 if [ -z "$AWS_ACCESS_KEY_ID" ]; then
@@ -65,5 +74,8 @@ else
     --new-version $NEW_VERSION \
     --suite $FROM_COMPONENT \
     --new-suite $TO_COMPONENT \
-    --new-name $NEW_NAME
+    --new-name $NEW_NAME \
+    --repo $REPO \
+    --new-repo $NEW_REPO
+    --sign $REPO_KEY
 fi

buildkite/scripts/debian/publish.sh

@@ -15,8 +15,18 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     exit 0
 fi
 
+if [ -z "${SIGN:-}" ]; then
+  SIGN_ARG=""
+else
+  sudo chown -R opam ~/.gnupg/
+  gpg --batch --yes --import /var/secrets/debian/key.gpg
+  SIGN_ARG="--sign $SIGN"
+fi
+
 source scripts/debian/publish.sh \
   --names "${DOWNLOAD_FOLDER}/mina-*.deb" \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \
-  --codename $MINA_DEB_CODENAME  
+  --codename $MINA_DEB_CODENAME \
+  --bucket $BUCKET \
+  $SIGN_ARG

buildkite/scripts/rosetta-integration-tests.sh

@@ -96,7 +96,7 @@ cat <<EOF >"$MINA_CONFIG_FILE"
   "daemon": { "network_id": "${MINA_NETWORK}" },
   "ledger": {
     "accounts": [
-      { "pk": "${BLOCK_PRODUCER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
+      { "pk": "${BLOCK_PRODUCER_PUB_KEY}", "balance": "600000000", "delegate": null, "sk": null },
       { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000000", "delegate": "${BLOCK_PRODUCER_PUB_KEY}", "sk": null },
       { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },
       { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "1000000", "delegate": null, "sk": null },

buildkite/scripts/tests/ledger_test_apply.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+sudo apt-get update
+sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 python3-pip wget
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/export-git-env-vars.sh
+
+source buildkite/scripts/debian/install.sh "mina-berkeley-instrumented" 1
+
+echo "removing magic config files"
+sudo rm -f /var/lib/coda/config_*
+
+./scripts/tests/ledger_test_apply.sh \
+    --mina-app mina \
+    --runtime-ledger-app mina-create-genesis

buildkite/src/Command/Base.dhall

@@ -131,6 +131,7 @@ let targetToAgent =
             , Integration = toMap { size = "integration" }
             , QA = toMap { size = "qa" }
             , Hardfork = toMap { size = "hardfork" }
+            , Perf = toMap { size = "perf" }
             , Multi = toMap { size = "generic-multi" }
             }
             target

buildkite/src/Command/Bench/Base.dhall

@@ -0,0 +1,96 @@
+let PipelineMode = ../../Pipeline/Mode.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let DebianVersions = ../../Constants/DebianVersions.dhall
+
+let RunInToolchain = ../../Command/RunInToolchain.dhall
+
+let Profiles = ../../Constants/Profiles.dhall
+
+let Command = ../../Command/Base.dhall
+
+let Docker = ../../Command/Docker/Type.dhall
+
+let Size = ../Size.dhall
+
+let Benchmarks = ../../Constants/Benchmarks.dhall
+
+let SelectFiles = ../../Lib/SelectFiles.dhall
+
+let Spec =
+      { Type =
+          { key : Text
+          , bench : Text
+          , label : Text
+          , size : Size
+          , name : Text
+          , path : Text
+          , mode : PipelineMode.Type
+          , dependsOn : List Command.TaggedKey.Type
+          , additionalDirtyWhen : List SelectFiles.Type
+          , yellowThreshold : Double
+          , redThreshold : Double
+          }
+      , default =
+          { mode = PipelineMode.Type.PullRequest
+          , size = Size.Medium
+          , dependsOn =
+              DebianVersions.dependsOn
+                DebianVersions.DebVersion.Bullseye
+                Profiles.Type.Standard
+          , additionalDirtyWhen = [] : List SelectFiles.Type
+          , yellowThreshold = 0.1
+          , redThreshold = 0.2
+          }
+      }
+
+let command
+    : Spec.Type -> Command.Type
+    =     \(spec : Spec.Type)
+      ->  Command.build
+            Command.Config::{
+            , commands =
+                RunInToolchain.runInToolchain
+                  (Benchmarks.toEnvList Benchmarks.Type::{=})
+                  "./buildkite/scripts/benchmarks.sh  ${spec.bench} --red-threshold ${Double/show
+                                                                                        spec.redThreshold} --yellow-threshold ${Double/show
+                                                                                                                                  spec.yellowThreshold}"
+            , label = "Perf: ${spec.label}"
+            , key = spec.key
+            , target = spec.size
+            , docker = None Docker.Type
+            , depends_on = spec.dependsOn
+            }
+
+let pipeline
+    : Spec.Type -> Pipeline.Config.Type
+    =     \(spec : Spec.Type)
+      ->  Pipeline.Config::{
+          , spec = JobSpec::{
+            , dirtyWhen =
+                  [ SelectFiles.strictlyStart (SelectFiles.contains "src")
+                  , SelectFiles.exactly
+                      "buildkite/src/Command/Bench/Base"
+                      "dhall"
+                  , SelectFiles.contains "scripts/benchmark"
+                  , SelectFiles.contains "buildkite/scripts/benchmark"
+                  ]
+                # spec.additionalDirtyWhen
+            , path = spec.path
+            , name = spec.name
+            , mode = spec.mode
+            , tags =
+              [ PipelineTag.Type.Long
+              , PipelineTag.Type.Test
+              , PipelineTag.Type.Stable
+              ]
+            }
+          , steps = [ command spec ]
+          }
+
+in  { command = command, pipeline = pipeline, Spec = Spec }

buildkite/src/Command/DockerImage.dhall

@@ -90,18 +90,23 @@ let generateStep =
                 ++  " --deb-build-flags ${BuildFlags.lowerName
                                             spec.build_flags}"
 
+          let remoteRepoCmds =
+                [ Cmd.run
+                    (     exportMinaDebCmd
+                      ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
+                      ++  " && "
+                      ++  buildDockerCmd
+                      ++  " && "
+                      ++  releaseDockerCmd
+                    )
+                ]
+
           let commands =
                 merge
-                  { PackagesO1Test =
-                    [ Cmd.run
-                        (     exportMinaDebCmd
-                          ++  " && source ./buildkite/scripts/export-git-env-vars.sh "
-                          ++  " && "
-                          ++  buildDockerCmd
-                          ++  " && "
-                          ++  releaseDockerCmd
-                        )
-                    ]
+                  { PackagesO1Test = remoteRepoCmds
+                  , Unstable = remoteRepoCmds
+                  , Nightly = remoteRepoCmds
+                  , Stable = remoteRepoCmds
                   , Local =
                     [ Cmd.run
                         (     exportMinaDebCmd

buildkite/src/Command/MinaArtifact.dhall

@@ -46,6 +46,7 @@ let MinaBuildSpec =
           , mode : PipelineMode.Type
           , tags : List PipelineTag.Type
           , channel : DebianChannel.Type
+          , debianRepo : DebianRepo.Type
           }
       , default =
           { prefix = "MinaArtifact"
@@ -58,6 +59,7 @@ let MinaBuildSpec =
           , mode = PipelineMode.Type.PullRequest
           , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
           , channel = DebianChannel.Type.Unstable
+          , debianRepo = DebianRepo.Type.PackagesO1Test
           }
       }
 
@@ -112,12 +114,16 @@ let publish_to_debian_repo =
                 Toolchain.select
                   spec.toolchainSelectMode
                   spec.debVersion
-                  [ "AWS_ACCESS_KEY_ID"
-                  , "AWS_SECRET_ACCESS_KEY"
-                  , "MINA_DEB_CODENAME=${DebianVersions.lowerName
-                                           spec.debVersion}"
-                  , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}"
-                  ]
+                  (   [ "AWS_ACCESS_KEY_ID"
+                      , "AWS_SECRET_ACCESS_KEY"
+                      , "MINA_DEB_CODENAME=${DebianVersions.lowerName
+                                               spec.debVersion}"
+                      , "MINA_DEB_RELEASE=${DebianChannel.lowerName
+                                              spec.channel}"
+                      ]
+                    # DebianRepo.keyIdEnvList spec.debianRepo
+                    # DebianRepo.bucketEnvList spec.debianRepo
+                  )
                   "./buildkite/scripts/debian/publish.sh"
             , label =
                 "Publish Mina for ${DebianVersions.capitalName
@@ -251,10 +257,10 @@ let docker_step
                     , deb_repo = DebianRepo.Type.Local
                     , deb_profile = spec.profile
                     , step_key =
-                        "test-suite-${DebianVersions.lowerName
-                                        spec.debVersion}${Profiles.toLabelSegment
-                                                            spec.profile}${BuildFlags.toLabelSegment
-                                                                             spec.buildFlags}--docker-image"
+                        "functional_test_suite-${DebianVersions.lowerName
+                                                   spec.debVersion}${Profiles.toLabelSegment
+                                                                       spec.profile}${BuildFlags.toLabelSegment
+                                                                                        spec.buildFlags}-docker-image"
                     , network = "berkeley"
                     }
                   ]