diff --git a/README-ci-failures.md b/README-ci-failures.md index 6e4f678c8b9..595b083f84b 100644 --- a/README-ci-failures.md +++ b/README-ci-failures.md @@ -46,7 +46,7 @@ In this case, you should: [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain) where they have been automatically uploaded. Look for your branch name and commit hash in the second column. You should find several images suffixed - with `-bullseye` and `-focal` (Debian versions). + with `-bullseye` or `-focal` (Debian versions). 3. For each such image, retrieve its full name and hash by hovering its link and clicking the `Copy full image name` tooltip that appears (or retrieve it on the image's page). diff --git a/README.md b/README.md index b848104e1d8..3e2459e5daa 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,18 @@ -### Build status + -| Develop | Berkeley | Compatible | -| ------- | -------- | ---------- | -| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - berkeley](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=berkeley)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) +

Mina

- - Mina logo - +
-# Mina + ![GitHub stars](https://img.shields.io/github/stars/minaprotocol/mina)  ![GitHub forks](https://img.shields.io/github/forks/minaprotocol/mina) + +![GitHub contributors](https://img.shields.io/github/contributors/minaprotocol/mina)  ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/minaprotocol/mina)  ![GitHub last commit](https://img.shields.io/github/last-commit/minaprotocol/mina) + +| Develop[^1] | Compatible[^2] | Master[^3] | +| ------- | ---------- | ---------- | +| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) + +
Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! @@ -60,3 +64,7 @@ The [Node Developers](https://docs.minaprotocol.com/node-developers) docs contai [Apache 2.0](LICENSE) Commits older than 2018-10-03 do not have a [LICENSE](LICENSE) file or this notice, but are distributed under the same terms. + +[^1]: Develop is a mainline branch containig code that may be not compatible with current mainnet and may require major upgrade (hardfork). +[^2]: Compatible is a mainline branch containig code which does not need hardfork in order to apply it to mainnet. +[^3]: Branch which contains current mainnet code. \ No newline at end of file diff --git a/automation/services/watchdog/Dockerfile b/automation/services/watchdog/Dockerfile index 43e6e00f825..05e207350f6 100644 --- a/automation/services/watchdog/Dockerfile +++ b/automation/services/watchdog/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.16-slim-buster +FROM python:3.9.16-slim-bullseye ARG GCLOUDSDK_DOWNLOAD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-296.0.1-linux-x86_64.tar.gz" ARG WATCHMAN_DEB_URL="http://ftp.us.debian.org/debian/pool/main/w/watchman/watchman_4.9.0-5+b1_amd64.deb" diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 263a258ef3c..7eb9c125203 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -24,7 +24,7 @@ make -C src/app/libp2p_helper MAINNET_TARGETS="" [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe" -echo "--- Build all major tagets required for packaging" +echo "--- Build all major targets required for packaging" echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}" echo "Rust Version: $(rustc --version)" dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \ @@ -45,4 +45,5 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \ src/app/rosetta/indexer_test/indexer_test.exe \ src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \ src/app/test_executive/test_executive.exe \ - src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log + src/test/command_line_tests/command_line_tests.exe \ + src/test/archive/patch_archive_test/patch_archive_test.exe diff --git a/buildkite/scripts/build-release.sh b/buildkite/scripts/build-release.sh index 68d794da5e1..dbf3bb490bc 100755 --- a/buildkite/scripts/build-release.sh +++ b/buildkite/scripts/build-release.sh @@ -9,7 +9,7 @@ source ~/.profile ./buildkite/scripts/build-artifact.sh echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" -echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" +echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/check-compatibility.sh b/buildkite/scripts/check-compatibility.sh index 6808c61e9e3..241de3c15d6 100755 --- a/buildkite/scripts/check-compatibility.sh +++ b/buildkite/scripts/check-compatibility.sh @@ -10,7 +10,7 @@ function get_shas { function image_tag { SHA=$1 - IMAGE_TAG="$SHA-bullseye-berkeley" + IMAGE_TAG="$SHA-bullseye-devnet" } function download-docker { @@ -22,17 +22,25 @@ function download-docker { function try_docker_shas { DOCKER_SHAS=$1 GOT_DOCKER=0 - for sha in $DOCKER_SHAS; do - download-docker $sha - if [ $? -eq 0 ] ; then - GOT_DOCKER=1 - image_tag $sha - break - else - echo "No docker available for SHA=$sha" - fi + + set +e + download-docker $sha + + if [ $? -eq 0 ] ; then + GOT_DOCKER=1 + image_tag $sha + break + else + echo "No docker available for SHA=$sha" + fi + set -e done + + if [[ $GOT_DOCKER == 0 ]]; then + echo "docker cannot be found for given shas: $DOCKER_SHAS" + exit 1 + fi } function image_id { diff --git a/buildkite/scripts/connect-to-berkeley.sh b/buildkite/scripts/connect-to-berkeley.sh deleted file mode 100755 index 1b2aa7cfe82..00000000000 --- a/buildkite/scripts/connect-to-berkeley.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - -# Don't prompt for answers during apt-get install -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl - -TESTNET_VERSION_NAME="berkeley" -TESTNET_NAME="testworld-2-0" - -git config --global --add safe.directory /workdir - -source buildkite/scripts/export-git-env-vars.sh - -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" - -# Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: - -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key -# Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ - -# Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background - -# Attempt to connect to the GraphQL client every 30s for up to 12 minutes -num_status_retries=24 -for ((i=1;i<=$num_status_retries;i++)); do - sleep 30s - set +e - mina client status - status_exit_code=$? - set -e - if [ $status_exit_code -eq 0 ]; then - break - elif [ $i -eq $num_status_retries ]; then - exit $status_exit_code - fi -done - -# Check that the daemon has connected to peers and is still up after 2 mins -sleep 2m -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi - diff --git a/buildkite/scripts/connect-to-testnet.sh b/buildkite/scripts/connect-to-testnet.sh index 3cfd7a4aa7b..cba27c93759 100755 --- a/buildkite/scripts/connect-to-testnet.sh +++ b/buildkite/scripts/connect-to-testnet.sh @@ -7,53 +7,46 @@ if [[ $# -ne 3 ]]; then exit 1 fi -TESTNET_VERSION_NAME="berkeley" TESTNET_NAME=$1 WAIT_BETWEEN_POLLING_GRAPHQL=$2 WAIT_AFTER_FINAL_CHECK=$3 -case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in - rampup|berkeley|release/2.0.0|develop) - ;; - *) - echo "Not pulling against rampup, not running the connect test" - exit 0 ;; -esac - # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive -apt-get update -apt-get install -y git apt-transport-https ca-certificates tzdata curl +sudo apt-get update +sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}" +source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1 # Remove lockfile if present -rm ~/.mina-config/.mina-lock ||: +sudo rm ~/.mina-config/.mina-lock ||: + +sudo mkdir -p /root/libp2p-keys/ -mkdir -p /root/libp2p-keys/ -# Pre-generated random password for this quick test -export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf -mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Set permissions on the keypair so the daemon doesn't complain -chmod -R 0700 /root/libp2p-keys/ +sudo chmod -R 0700 /root/libp2p-keys/ +# Pre-generated random password for this quick test +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key # Restart in the background -mina daemon \ - --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \ - --libp2p-keypair "/root/libp2p-keys/key" \ -& # -background +sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \ + TESTNET_NAME=$TESTNET_NAME \ + bash -c "mina daemon \ + --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \ + --libp2p-keypair \"/root/libp2p-keys/key\" \ + --seed &" # -background # Attempt to connect to the GraphQL client every 10s for up to 8 minutes num_status_retries=24 for ((i=1;i<=$num_status_retries;i++)); do sleep $WAIT_BETWEEN_POLLING_GRAPHQL set +e - mina client status + sudo mina client status status_exit_code=$? set -e if [ $status_exit_code -eq 0 ]; then @@ -63,13 +56,18 @@ for ((i=1;i<=$num_status_retries;i++)); do fi done -# Check that the daemon has connected to peers and is still up after 2 mins -sleep $WAIT_AFTER_FINAL_CHECK -mina client status -if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then - echo "Found some peers" -else - echo "No peers found" - exit 1 -fi +peer_retries=10 +for ((i=1;i<=$peer_retries;i++)); do + peer_count=$(sudo mina advanced get-peers | wc -l) + sudo mina client status + + if [ "$peer_count" -gt 0 ]; then + echo "Found some peers" + exit 0; + else + echo "No peers found" + fi + sleep $WAIT_AFTER_FINAL_CHECK +done +exit 1; \ No newline at end of file diff --git a/buildkite/scripts/debian/install.sh b/buildkite/scripts/debian/install.sh index 93313f4a2dd..bd9a863f8ef 100755 --- a/buildkite/scripts/debian/install.sh +++ b/buildkite/scripts/debian/install.sh @@ -34,7 +34,7 @@ else debs=(${DEBS//,/ }) for i in "${debs[@]}"; do case $i in - mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet) + mina-devnet|mina-mainnet|mina-devnet-lightnet) # Downaload mina-logproc too source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER ;; diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh index 57d3c2b2302..3855157e017 100755 --- a/buildkite/scripts/dump-mina-type-shapes.sh +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/rosetta-integration-tests.sh b/buildkite/scripts/rosetta-integration-tests.sh index c360dbcc3e7..555ece7ce28 100755 --- a/buildkite/scripts/rosetta-integration-tests.sh +++ b/buildkite/scripts/rosetta-integration-tests.sh @@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then send_zkapp_transactions & fi -next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) -sleep_time=$((($next_block_time - $curr_time) / 1000)) +mina client status --json + +next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13) +sleep_time=$((($next_block_time - $curr_time) / 10000000)) echo "Sleeping for ${sleep_time}s until next block is created..." -sleep ${sleep_time} +sleep $sleep_time # Mina Rosetta Checks (spec construction data perf) echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} ==============" diff --git a/buildkite/scripts/run-snark-transaction-profiler.sh b/buildkite/scripts/run-snark-transaction-profiler.sh index 802cd730632..70c20a92167 100755 --- a/buildkite/scripts/run-snark-transaction-profiler.sh +++ b/buildkite/scripts/run-snark-transaction-profiler.sh @@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive sudo apt-get update sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh diff --git a/buildkite/scripts/run-test-executive-cloud.sh b/buildkite/scripts/run-test-executive-cloud.sh index 884b8222cbc..8a3df280401 100755 --- a/buildkite/scripts/run-test-executive-cloud.sh +++ b/buildkite/scripts/run-test-executive-cloud.sh @@ -2,7 +2,7 @@ set -o pipefail -x TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then diff --git a/buildkite/scripts/run-test-executive-local.sh b/buildkite/scripts/run-test-executive-local.sh index 09c37c2b206..8512b93df0f 100755 --- a/buildkite/scripts/run-test-executive-local.sh +++ b/buildkite/scripts/run-test-executive-local.sh @@ -26,7 +26,7 @@ cleanup TEST_NAME="$1" -MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley" +MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet" ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG" if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then @@ -42,7 +42,7 @@ rm -f /etc/apt/sources.list.d/hashicorp.list apt-get update apt-get install -y git apt-transport-https ca-certificates tzdata curl -TESTNET_NAME="berkeley" +TESTNET_NAME="devnet" git config --global --add safe.directory /workdir diff --git a/buildkite/scripts/run_promote_build_job.sh b/buildkite/scripts/run_promote_build_job.sh index 191c24e197b..c7d02b1a5fa 100755 --- a/buildkite/scripts/run_promote_build_job.sh +++ b/buildkite/scripts/run_promote_build_job.sh @@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index da2a3789d3f..13e1f5f62cc 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)" if [[ -n "$DEBIANS" ]]; then if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi; if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi; - if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi; + if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi; if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi; if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi; if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi; diff --git a/buildkite/scripts/single-node-tests.sh b/buildkite/scripts/single-node-tests.sh index 93b224c5ba8..dcd6752ae23 100755 --- a/buildkite/scripts/single-node-tests.sh +++ b/buildkite/scripts/single-node-tests.sh @@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir source buildkite/scripts/export-git-env-vars.sh -source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1 +source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1 export MINA_LIBP2P_PASS="naughty blue worm" export MINA_PRIVKEY_PASS="naughty blue worm" diff --git a/buildkite/scripts/unit-test.sh b/buildkite/scripts/unit-test.sh index de885193086..b38ad7d94a8 100755 --- a/buildkite/scripts/unit-test.sh +++ b/buildkite/scripts/unit-test.sh @@ -21,10 +21,10 @@ export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go time make build echo "--- Build all targets" -dune build "${path}" --profile="${profile}" -j16 +dune build "${path}" --profile="${profile}" -j8 echo "--- Check for changes to verification keys" -time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j16 +time dune runtest "src/app/print_blockchain_snark_vk" --profile="${profile}" -j8 # Turn on the proof-cache assertion, so that CI will fail if the proofs need to # be updated. @@ -35,8 +35,8 @@ export ERROR_ON_PROOF=true # skip running all of the tests that have already succeeded, since dune will # only retry those tests that failed. echo "--- Run unit tests" -time dune runtest "${path}" --profile="${profile}" -j16 || \ +time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && \ echo "--- Retrying failed unit tests" && \ - time dune runtest "${path}" --profile="${profile}" -j16 || \ + time dune runtest "${path}" --profile="${profile}" -j8 || \ (./scripts/link-coredumps.sh && false)) diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index fb1ea8c9d89..8af413c2436 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,7 +7,7 @@ if [[ $# -ne 1 ]]; then exit 1 fi -TESTNET_NAME="${TESTNET_NAME:-berkeley}" +TESTNET_NAME="${TESTNET_NAME:-devnet}" # Don't prompt for answers during apt-get install export DEBIAN_FRONTEND=noninteractive diff --git a/buildkite/src/Command/ConnectToTestnet.dhall b/buildkite/src/Command/ConnectToTestnet.dhall index ba4bba1b974..91bf6ee8bea 100644 --- a/buildkite/src/Command/ConnectToTestnet.dhall +++ b/buildkite/src/Command/ConnectToTestnet.dhall @@ -4,9 +4,9 @@ let Command = ./Base.dhall let Size = ./Size.dhall -let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let RunInToolchain = ./RunInToolchain.dhall -let Cmd = ../Lib/Cmds.dhall +let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type in { step = \(dependsOn : List Command.TaggedKey.Type) @@ -17,12 +17,9 @@ in { step = -> Command.build Command.Config::{ , commands = - [ Cmd.runInDocker - Cmd.Docker::{ - , image = (../Constants/ContainerImages.dhall).ubuntu2004 - } + RunInToolchain.runInToolchain + ([] : List Text) "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}" - ] , label = "Connect to ${testnet}" , soft_fail = Some soft_fail , key = "connect-to-${testnet}" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 1199d461332..917a50fb621 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -10,8 +10,6 @@ let Size = ./Size.dhall let Profiles = ../Constants/Profiles.dhall -let Artifacts = ../Constants/Artifacts.dhall - let BuildFlags = ../Constants/BuildFlags.dhall let Cmd = ../Lib/Cmds.dhall @@ -20,57 +18,81 @@ let DockerLogin = ../Command/DockerLogin/Type.dhall let DebianRepo = ../Constants/DebianRepo.dhall +let DebianVersions = ../Constants/DebianVersions.dhall + +let Network = ../Constants/Network.dhall + +let Artifacts = ../Constants/Artifacts.dhall + let ReleaseSpec = { Type = { deps : List Command.TaggedKey.Type , network : Text - , service : Text + , service : Artifacts.Type , version : Text , branch : Text , repo : Text , no_cache : Bool - , deb_codename : Text + , deb_codename : DebianVersions.DebVersion , deb_release : Text , deb_version : Text , deb_profile : Profiles.Type , deb_repo : DebianRepo.Type , build_flags : BuildFlags.Type - , step_key : Text + , step_key_suffix : Text , if : Optional B/If } , default = { deps = [] : List Command.TaggedKey.Type - , network = "devnet" + , network = "${Network.lowerName Network.Type.Devnet}" , version = "\\\${MINA_DOCKER_TAG}" - , service = Artifacts.dockerName Artifacts.Type.Daemon + , service = Artifacts.Type.Daemon , branch = "\\\${BUILDKITE_BRANCH}" , repo = "\\\${BUILDKITE_REPO}" - , deb_codename = "bullseye" + , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" , deb_profile = Profiles.Type.Standard , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.PackagesO1Test , no_cache = False - , step_key = "daemon-standard-docker-image" + , step_key_suffix = "-docker-image" , if = None B/If } } +let stepKey = + \(spec : ReleaseSpec.Type) + -> "${Artifacts.lowerName + spec.service}${Profiles.toLabelSegment + spec.deb_profile}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" + +let stepLabel = + \(spec : ReleaseSpec.Type) + -> "Docker: ${Artifacts.capitalName + spec.service} ${spec.network} ${DebianVersions.capitalName + spec.deb_codename} ${Profiles.toSuffixUppercase + spec.deb_profile} ${BuildFlags.toSuffixUppercase + spec.build_flags}" + let generateStep = \(spec : ReleaseSpec.Type) - -> let exportMinaDebCmd = "export MINA_DEB_CODENAME=${spec.deb_codename}" + -> let exportMinaDebCmd = + "export MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.deb_codename}" let maybeCacheOption = if spec.no_cache then "--no-cache" else "" let buildDockerCmd = "./scripts/docker/build.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --network ${spec.network}" ++ " --version ${spec.version}" ++ " --branch ${spec.branch}" ++ " ${maybeCacheOption} " - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-repo ${DebianRepo.address spec.deb_repo}" ++ " --deb-release ${spec.deb_release}" ++ " --deb-version ${spec.deb_version}" @@ -81,10 +103,11 @@ let generateStep = let releaseDockerCmd = "./scripts/docker/release.sh" - ++ " --service ${spec.service}" + ++ " --service ${Artifacts.dockerName spec.service}" ++ " --version ${spec.version}" ++ " --network ${spec.network}" - ++ " --deb-codename ${spec.deb_codename}" + ++ " --deb-codename ${DebianVersions.lowerName + spec.deb_codename}" ++ " --deb-version ${spec.deb_version}" ++ " --deb-profile ${Profiles.lowerName spec.deb_profile}" ++ " --deb-build-flags ${BuildFlags.lowerName @@ -121,12 +144,16 @@ let generateStep = in Command.build Command.Config::{ , commands = commands - , label = "Docker: ${spec.step_key}" - , key = spec.step_key + , label = "${stepLabel spec}" + , key = "${stepKey spec}" , target = Size.XLarge , docker_login = Some DockerLogin::{=} , depends_on = spec.deps , if = spec.if } -in { generateStep = generateStep, ReleaseSpec = ReleaseSpec } +in { generateStep = generateStep + , ReleaseSpec = ReleaseSpec + , stepKey = stepKey + , stepLabel = stepLabel + } diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 183826d81b4..9e9af392e52 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -30,14 +30,14 @@ let DebianVersions = ../Constants/DebianVersions.dhall let DebianRepo = ../Constants/DebianRepo.dhall -let Artifacts = ../Constants/Artifacts.dhall - let Profiles = ../Constants/Profiles.dhall let Toolchain = ../Constants/Toolchain.dhall let Network = ../Constants/Network.dhall +let Artifacts = ../Constants/Artifacts.dhall + let Spec = { Type = { codename : DebianVersions.DebVersion @@ -98,6 +98,17 @@ let pipeline "gcr.io/o1labs-192920/mina-daemon:\${BUILDKITE_COMMIT:0:7}-${DebianVersions.lowerName debVersion}-${network_name}" + let dockerSpec = + DockerImage.ReleaseSpec::{ + , deps = + [ { name = pipelineName, key = generateLedgersJobKey } ] + , service = Artifacts.Type.Daemon + , network = network_name + , deb_codename = debVersion + , deb_profile = profile + , deb_repo = DebianRepo.Type.Local + } + in Pipeline.Config::{ , spec = JobSpec::{ , dirtyWhen = [ S.everything ] @@ -155,20 +166,7 @@ let pipeline , key = "publish-hardfork-deb-pkg" , target = Size.Small } - , DockerImage.generateStep - DockerImage.ReleaseSpec::{ - , deps = - [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Daemon - , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" - , deb_profile = profile - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" - } + , DockerImage.generateStep dockerSpec , Command.build Command.Config::{ , commands = @@ -182,10 +180,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -202,10 +197,7 @@ let pipeline , target = Size.XLarge , depends_on = [ { name = pipelineName - , key = - "daemon-berkeley-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , key = DockerImage.stepKey dockerSpec } ] , if = None B/If @@ -214,28 +206,20 @@ let pipeline DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Archive + , service = Artifacts.Type.Archive , network = network_name - , deb_codename = "${DebianVersions.lowerName debVersion}" + , deb_codename = debVersion , deb_profile = profile , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" } , DockerImage.generateStep DockerImage.ReleaseSpec::{ , deps = [ { name = pipelineName, key = generateLedgersJobKey } ] - , service = Artifacts.dockerName Artifacts.Type.Rosetta + , service = Artifacts.Type.Rosetta , network = network_name , deb_repo = DebianRepo.Type.Local - , deb_codename = "${DebianVersions.lowerName debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - debVersion}${Profiles.toLabelSegment - profile}-docker-image" + , deb_codename = debVersion } ] } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 28793fb3d22..1af0da3f865 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -42,7 +42,7 @@ let MinaBuildSpec = , artifacts : List Artifacts.Type , debVersion : DebianVersions.DebVersion , profile : Profiles.Type - , networks : List Network.Type + , network : Network.Type , buildFlags : BuildFlags.Type , toolchainSelectMode : Toolchain.SelectionMode , mode : PipelineMode.Type @@ -55,7 +55,7 @@ let MinaBuildSpec = , debVersion = DebianVersions.DebVersion.Bullseye , profile = Profiles.Type.Standard , buildFlags = BuildFlags.Type.None - , networks = [ Network.Type.Berkeley ] + , network = Network.Type.Devnet , toolchainSelectMode = Toolchain.SelectionMode.ByDebian , mode = PipelineMode.Type.PullRequest , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ] @@ -63,6 +63,30 @@ let MinaBuildSpec = } } +let labelSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion} ${Network.capitalName + spec.network} ${Profiles.toSuffixUppercase + spec.profile} ${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let nameSuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${DebianVersions.capitalName + spec.debVersion}${Network.capitalName + spec.network}${Profiles.toSuffixUppercase + spec.profile}${BuildFlags.toSuffixUppercase + spec.buildFlags}" + +let keySuffix + : MinaBuildSpec.Type -> Text + = \(spec : MinaBuildSpec.Type) + -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment + spec.buildFlags}" + let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -79,22 +103,18 @@ let build_artifacts , "MINA_COMMIT_SHA1=\$BUILDKITE_COMMIT" , "MINA_DEB_CODENAME=${DebianVersions.lowerName spec.debVersion}" - , Network.foldMinaBuildMainnetEnv spec.networks + , Network.buildMainnetEnv spec.network ] # BuildFlags.buildEnvs spec.buildFlags ) "./buildkite/scripts/build-release.sh ${Artifacts.toDebianNames spec.artifacts - spec.networks}" + spec.network}" # [ Cmd.run "./buildkite/scripts/debian/upload-to-gs.sh ${DebianVersions.lowerName spec.debVersion}" ] - , label = - "Build Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile} ${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , label = "Debian: Build ${labelSuffix spec}" , key = "build-deb-pkg" , target = Size.XLarge , retries = @@ -105,36 +125,6 @@ let build_artifacts ] } -let publish_to_debian_repo = - \(spec : MinaBuildSpec.Type) - -> Command.build - Command.Config::{ - , commands = - Toolchain.select - spec.toolchainSelectMode - spec.debVersion - [ "AWS_ACCESS_KEY_ID" - , "AWS_SECRET_ACCESS_KEY" - , "MINA_DEB_CODENAME=${DebianVersions.lowerName - spec.debVersion}" - , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" - ] - "./buildkite/scripts/debian/publish.sh" - , label = - "Publish Mina for ${DebianVersions.capitalName - spec.debVersion} ${Profiles.toSuffixUppercase - spec.profile}" - , key = "publish-deb-pkg" - , depends_on = - DebianVersions.dependsOnStep - (Some spec.prefix) - spec.debVersion - spec.profile - spec.buildFlags - "build" - , target = Size.Small - } - let docker_step : Artifacts.Type -> MinaBuildSpec.Type -> List DockerImage.ReleaseSpec.Type = \(artifact : Artifacts.Type) @@ -145,116 +135,97 @@ let docker_step DebianVersions.dependsOnStep (Some spec.prefix) spec.debVersion + spec.network spec.profile spec.buildFlags step_dep_name in merge { Daemon = - Prelude.List.map - Network.Type - DockerImage.ReleaseSpec.Type - ( \(n : Network.Type) - -> DockerImage.ReleaseSpec::{ - , deps = deps - , service = - Artifacts.dockerName Artifacts.Type.Daemon - , network = Network.lowerName n - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , deb_profile = spec.profile - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local - , step_key = - "daemon-${Network.lowerName - n}-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" - } - ) - spec.networks + [ DockerImage.ReleaseSpec::{ + , deps = deps + , service = Artifacts.Type.Daemon + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] , TestExecutive = [] : List DockerImage.ReleaseSpec.Type , LogProc = [] : List DockerImage.ReleaseSpec.Type + , Toolchain = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Toolchain + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion + , deb_profile = spec.profile + , build_flags = spec.buildFlags + , deb_repo = DebianRepo.Type.Local + } + ] + , ItnOrchestrator = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.ItnOrchestrator + , network = Network.lowerName Network.Type.Devnet + , deb_repo = DebianRepo.Type.Local + } + ] + , Leaderboard = + [ DockerImage.ReleaseSpec::{ + , service = Artifacts.Type.Leaderboard + } + ] , BatchTxn = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-batch-txn" - , network = "berkeley" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.BatchTxn + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "batch-txn-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" } ] , Archive = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-archive" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.Archive + , deb_codename = spec.debVersion , deb_profile = spec.profile , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local - , step_key = - "archive-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" } ] , Rosetta = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-rosetta" - , network = "berkeley" - , build_flags = spec.buildFlags - , deb_repo = DebianRepo.Type.Local + , service = Artifacts.Type.Rosetta + , network = Network.lowerName spec.network + , deb_codename = spec.debVersion , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "rosetta-${DebianVersions.lowerName - spec.debVersion}${BuildFlags.toLabelSegment - spec.buildFlags}-docker-image" + , deb_repo = DebianRepo.Type.Local } ] , ZkappTestTransaction = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-zkapp-test-transaction" + , service = Artifacts.Type.ZkappTestTransaction , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" - , step_key = - "zkapp-test-transaction-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" + , deb_codename = spec.debVersion } ] , FunctionalTestSuite = [ DockerImage.ReleaseSpec::{ , deps = deps - , service = "mina-test-suite" - , deb_codename = - "${DebianVersions.lowerName spec.debVersion}" + , service = Artifacts.Type.FunctionalTestSuite + , network = Network.lowerName Network.Type.Devnet + , deb_codename = spec.debVersion , build_flags = spec.buildFlags , deb_repo = DebianRepo.Type.Local , deb_profile = spec.profile - , step_key = - "test-suite-${DebianVersions.lowerName - spec.debVersion}${Profiles.toLabelSegment - spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}--docker-image" - , network = "berkeley" } ] } @@ -289,6 +260,34 @@ let docker_commands ) flattened_docker_steps +let publish_to_debian_repo = + \(spec : MinaBuildSpec.Type) + -> Command.build + Command.Config::{ + , commands = + Toolchain.select + spec.toolchainSelectMode + spec.debVersion + [ "AWS_ACCESS_KEY_ID" + , "AWS_SECRET_ACCESS_KEY" + , "MINA_DEB_CODENAME=${DebianVersions.lowerName + spec.debVersion}" + , "MINA_DEB_RELEASE=${DebianChannel.lowerName spec.channel}" + ] + "./buildkite/scripts/debian/publish.sh" + , label = "Debian: Publish ${labelSuffix spec}" + , key = "publish-deb-pkg" + , depends_on = + DebianVersions.dependsOnStep + (Some spec.prefix) + spec.debVersion + spec.network + spec.profile + spec.buildFlags + "build" + , target = Size.Small + } + let pipelineBuilder : MinaBuildSpec.Type -> List Command.Type -> Pipeline.Config.Type = \(spec : MinaBuildSpec.Type) @@ -297,11 +296,7 @@ let pipelineBuilder , spec = JobSpec::{ , dirtyWhen = DebianVersions.dirtyWhen spec.debVersion , path = "Release" - , name = - "${spec.prefix}${DebianVersions.capitalName - spec.debVersion}${Profiles.toSuffixUppercase - spec.profile}${BuildFlags.toSuffixUppercase - spec.buildFlags}" + , name = "${spec.prefix}${nameSuffix spec}" , tags = spec.tags , mode = spec.mode } @@ -333,4 +328,6 @@ let pipeline in { pipeline = pipeline , onlyDebianPipeline = onlyDebianPipeline , MinaBuildSpec = MinaBuildSpec + , labelSuffix = labelSuffix + , keySuffix = keySuffix } diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall new file mode 100644 index 00000000000..aff17f8519b --- /dev/null +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -0,0 +1,30 @@ +let Artifacts = ../Constants/Artifacts.dhall + +let Command = ./Base.dhall + +let Size = ./Size.dhall + +let Network = ../Constants/Network.dhall + +let RunWithPostgres = ./RunWithPostgres.dhall + +in { step = + \(dependsOn : List Command.TaggedKey.Type) + -> Command.build + Command.Config::{ + , commands = + [ RunWithPostgres.runInDockerWithPostgresConn + [ "PATCH_ARCHIVE_TEST_APP=mina-patch-archive-test" + , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" + ] + "./src/test/archive/sample_db/archive_db.sql" + Artifacts.Type.FunctionalTestSuite + (None Network.Type) + "./scripts/patch-archive-test.sh" + ] + , label = "Archive: Patch Archive test" + , key = "patch-archive-test" + , target = Size.Large + , depends_on = dependsOn + } + } diff --git a/buildkite/src/Command/PromotePackage.dhall b/buildkite/src/Command/PromotePackage.dhall new file mode 100644 index 00000000000..4989004d394 --- /dev/null +++ b/buildkite/src/Command/PromotePackage.dhall @@ -0,0 +1,305 @@ +let B = ../External/Buildkite.dhall + +let B/If = B.definitions/commandStep/properties/if/Type + +let Prelude = ../External/Prelude.dhall + +let List/map = Prelude.List.map + +let Package = ../Constants/DebianPackage.dhall + +let Network = ../Constants/Network.dhall + +let PipelineMode = ../Pipeline/Mode.dhall + +let PipelineTag = ../Pipeline/Tag.dhall + +let Pipeline = ../Pipeline/Dsl.dhall + +let JobSpec = ../Pipeline/JobSpec.dhall + +let DebianChannel = ../Constants/DebianChannel.dhall + +let Profiles = ../Constants/Profiles.dhall + +let Artifact = ../Constants/Artifacts.dhall + +let DebianVersions = ../Constants/DebianVersions.dhall + +let Toolchain = ../Constants/Toolchain.dhall + +let Command = ./Base.dhall + +let Size = ./Size.dhall + +let Cmd = ../Lib/Cmds.dhall + +let PromoteDebianSpec = + { Type = + { deps : List Command.TaggedKey.Type + , package : Package.Type + , version : Text + , new_version : Text + , architecture : Text + , network : Network.Type + , codename : DebianVersions.DebVersion + , from_channel : DebianChannel.Type + , to_channel : DebianChannel.Type + , profile : Profiles.Type + , remove_profile_from_name : Bool + , step_key : Text + , if : Optional B/If + } + , default = + { deps = [] : List Command.TaggedKey.Type + , package = Package.Type.LogProc + , version = "" + , new_version = "" + , architecture = "amd64" + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , from_channel = DebianChannel.Type.Unstable + , to_channel = DebianChannel.Type.NightlyDevelop + , profile = Profiles.Type.Standard + , remove_profile_from_name = False + , step_key = "promote-debian-package" + , if = None B/If + } + } + +let PromoteDockerSpec = + { Type = + { deps : List Command.TaggedKey.Type + , name : Artifact.Type + , version : Text + , profile : Profiles.Type + , codename : DebianVersions.DebVersion + , new_tag : Text + , network : Network.Type + , step_key : Text + , if : Optional B/If + , publish : Bool + , remove_profile_from_name : Bool + } + , default = + { deps = [] : List Command.TaggedKey.Type + , name = Artifact.Type.Daemon + , version = "" + , new_tag = "" + , step_key = "promote-docker" + , profile = Profiles.Type.Standard + , network = Network.Type.Devnet + , codename = DebianVersions.DebVersion.Bullseye + , if = None B/If + , publish = False + , remove_profile_from_name = False + } + } + +let promoteDebianStep = + \(spec : PromoteDebianSpec.Type) + -> let package_name + : Text + = Package.debianName spec.package spec.profile spec.network + + let new_name = + if spec.remove_profile_from_name + + then "--new-name ${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else "" + + in Command.build + Command.Config::{ + , commands = + Toolchain.runner + DebianVersions.DebVersion.Bullseye + [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" ] + "./buildkite/scripts/promote-deb.sh --package ${package_name} --version ${spec.version} --new-version ${spec.new_version} --architecture ${spec.architecture} --codename ${DebianVersions.lowerName + spec.codename} --from-component ${DebianChannel.lowerName + spec.from_channel} --to-component ${DebianChannel.lowerName + spec.to_channel} ${new_name}" + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDebianVerificationStep = + \(spec : PromoteDebianSpec.Type) + -> let name = + if spec.remove_profile_from_name + + then "${Package.debianName + spec.package + Profiles.Type.Standard + spec.network}" + + else Package.debianName spec.package spec.profile spec.network + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./scripts/debian/verify.sh --package ${name} --version ${spec.new_version} --codename ${DebianVersions.lowerName + spec.codename} --channel ${DebianChannel.lowerName + spec.to_channel}" + ] + , label = "Debian: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerStep = + \(spec : PromoteDockerSpec.Type) + -> let old_tag = + Artifact.dockerTag + spec.name + spec.version + spec.codename + spec.profile + spec.network + False + + let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let publish = if spec.publish then "-p" else "" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "./buildkite/scripts/promote-docker.sh --name ${Artifact.dockerName + spec.name} --version ${old_tag} --tag ${new_tag} ${publish}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.XLarge + , depends_on = spec.deps + , if = spec.if + } + +let promoteDockerVerificationStep = + \(spec : PromoteDockerSpec.Type) + -> let new_tag = + Artifact.dockerTag + spec.name + spec.new_tag + spec.codename + spec.profile + spec.network + spec.remove_profile_from_name + + let repo = + if spec.publish + + then "docker.io/minaprotocol" + + else "gcr.io/o1labs-192920" + + in Command.build + Command.Config::{ + , commands = + [ Cmd.run + "docker pull ${repo}/${Artifact.dockerName + spec.name}:${new_tag}" + ] + , label = "Docker: ${spec.step_key}" + , key = spec.step_key + , target = Size.Small + , depends_on = spec.deps + , if = spec.if + } + +let promotePipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + (\(spec : PromoteDebianSpec.Type) -> promoteDebianStep spec) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + (\(spec : PromoteDockerSpec.Type) -> promoteDockerStep spec) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "PromotePackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +let verifyPipeline + : List PromoteDebianSpec.Type + -> List PromoteDockerSpec.Type + -> DebianVersions.DebVersion + -> PipelineMode.Type + -> Pipeline.Config.Type + = \(debians_spec : List PromoteDebianSpec.Type) + -> \(dockers_spec : List PromoteDockerSpec.Type) + -> \(debVersion : DebianVersions.DebVersion) + -> \(mode : PipelineMode.Type) + -> let steps = + List/map + PromoteDebianSpec.Type + Command.Type + ( \(spec : PromoteDebianSpec.Type) + -> promoteDebianVerificationStep spec + ) + debians_spec + # List/map + PromoteDockerSpec.Type + Command.Type + ( \(spec : PromoteDockerSpec.Type) + -> promoteDockerVerificationStep spec + ) + dockers_spec + + in Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = DebianVersions.dirtyWhen debVersion + , path = "Release" + , name = "VerifyPackage" + , tags = [] : List PipelineTag.Type + , mode = mode + } + , steps = steps + } + +in { promoteDebianStep = promoteDebianStep + , promoteDockerStep = promoteDockerStep + , promoteDebianVerificationStep = promoteDebianVerificationStep + , promoteDockerVerificationStep = promoteDockerVerificationStep + , promotePipeline = promotePipeline + , verifyPipeline = verifyPipeline + , PromoteDebianSpec = PromoteDebianSpec + , PromoteDockerSpec = PromoteDockerSpec + } diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index a5437e6a6c1..cc31e92dcbe 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -40,7 +40,7 @@ let PromoteDebianSpec = , version = "\\\\\$MINA_DEB_VERSION" , new_version = "\\\\\$MINA_DEB_VERSION" , architecture = "amd64" - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , from_channel = DebianChannel.Type.Unstable , to_channel = DebianChannel.Type.NightlyCompatible diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index cbc60627dc8..7e56225d5ef 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -41,7 +41,7 @@ let PromoteDockerSpec = , new_tags = [] : List Text , step_key = "promote-docker" , profile = Profiles.Type.Standard - , network = Network.Type.Berkeley + , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If , publish = False diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a23..1c10cae985e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -4,12 +4,12 @@ let P = Prelude let Text/concatMap = P.Text.concatMap +let Cmd = ../Lib/Cmds.dhall + let Optional/map = Prelude.Optional.map let Optional/default = Prelude.Optional.default -let Cmd = ../Lib/Cmds.dhall - let ContainerImages = ../Constants/ContainerImages.dhall let Artifacts = ../Constants/Artifacts.dhall diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index b41f52cf28c..673b14c3808 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -18,6 +18,9 @@ let Artifact | Rosetta | ZkappTestTransaction | FunctionalTestSuite + | Toolchain + | ItnOrchestrator + | Leaderboard > let AllButTests = @@ -33,7 +36,13 @@ let AllButTests = let Main = [ Artifact.Daemon, Artifact.LogProc, Artifact.Archive, Artifact.Rosetta ] -let All = AllButTests # [ Artifact.FunctionalTestSuite ] +let All = + AllButTests + # [ Artifact.FunctionalTestSuite + , Artifact.Toolchain + , Artifact.ItnOrchestrator + , Artifact.Leaderboard + ] let capitalName = \(artifact : Artifact) @@ -46,6 +55,9 @@ let capitalName = , Rosetta = "Rosetta" , ZkappTestTransaction = "ZkappTestTransaction" , FunctionalTestSuite = "FunctionalTestSuite" + , Toolchain = "Toolchain" + , ItnOrchestrator = "ItnOrchestrator" + , Leaderboard = "Leaderboard" } artifact @@ -60,6 +72,9 @@ let lowerName = , Rosetta = "rosetta" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "toolchain" + , ItnOrchestrator = "itnOrchestrator" + , Leaderboard = "leaderboard" } artifact @@ -74,6 +89,9 @@ let dockerName = , Rosetta = "mina-rosetta" , ZkappTestTransaction = "mina-zkapp-test-transaction" , FunctionalTestSuite = "mina-test-suite" + , Toolchain = "mina-toolchain" + , ItnOrchestrator = "itn-orchestrator" + , Leaderboard = "leaderboard" } artifact @@ -89,38 +107,32 @@ let toDebianName = , Rosetta = "rosetta_${Network.lowerName network}" , ZkappTestTransaction = "zkapp_test_transaction" , FunctionalTestSuite = "functional_test_suite" + , Toolchain = "" + , ItnOrchestrator = "" + , Leaderboard = "" } artifact let toDebianNames = \(artifacts : List Artifact) - -> \(networks : List Network.Type) + -> \(network : Network.Type) -> let list_of_list_of_debians = Prelude.List.map Artifact (List Text) ( \(a : Artifact) -> merge - { Daemon = - Prelude.List.map - Network.Type - Text - (\(n : Network.Type) -> toDebianName a n) - networks + { Daemon = [ toDebianName a network ] , Archive = [ "archive" ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] - , Rosetta = - Prelude.List.map - Network.Type - Text - ( \(n : Network.Type) - -> "rosetta_${Network.lowerName n}" - ) - networks + , Rosetta = [ toDebianName a network ] , ZkappTestTransaction = [ "zkapp_test_transaction" ] , FunctionalTestSuite = [ "functional_test_suite" ] + , Toolchain = [] : List Text + , ItnOrchestrator = [] : List Text + , Leaderboard = [] : List Text } a ) @@ -165,6 +177,9 @@ let dockerTag = "${version_and_codename}-${Network.lowerName network}" , ZkappTestTransaction = "${version_and_codename}" , FunctionalTestSuite = "${version_and_codename}" + , Toolchain = "${version_and_codename}" + , ItnOrchestrator = "${version_and_codename}" + , Leaderboard = "${version_and_codename}" } artifact diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index f3042f5f067..af12f14134d 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -4,6 +4,8 @@ let Optional/default = Prelude.Optional.default let Profiles = ./Profiles.dhall +let Network = ./Network.dhall + let BuildFlags = ./BuildFlags.dhall let S = ../Lib/SelectFiles.dhall @@ -33,6 +35,7 @@ let lowerName = let dependsOnStep = \(prefix : Optional Text) -> \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(buildFlag : BuildFlags.Type) -> \(step : Text) @@ -40,47 +43,28 @@ let dependsOnStep = let prefix = Optional/default Text "MinaArtifact" prefix + let name = + "${prefix}${capitalName + debVersion}${Network.capitalName + network}${profileSuffix}${BuildFlags.toSuffixUppercase + buildFlag}" + in merge - { Bookworm = - [ { name = - "${prefix}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Bullseye = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Jammy = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] - , Focal = - [ { name = - "${prefix}${capitalName - debVersion}${profileSuffix}${BuildFlags.toSuffixUppercase - buildFlag}" - , key = "${step}-deb-pkg" - } - ] + { Bookworm = [ { name = name, key = "${step}-deb-pkg" } ] + , Bullseye = [ { name = name, key = "${step}-deb-pkg" } ] + , Jammy = [ { name = name, key = "${step}-deb-pkg" } ] + , Focal = [ { name = name, key = "${step}-deb-pkg" } ] } debVersion let dependsOn = \(debVersion : DebVersion) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> dependsOnStep (None Text) debVersion + network profile BuildFlags.Type.None "build" @@ -92,10 +76,8 @@ let minimalDirtyWhen = , S.exactly "buildkite/src/Command/MinaArtifact" "dhall" , S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaArtifact") , S.strictlyStart (S.contains "dockerfiles/stages") - , S.exactly "scripts/debian/build" "sh" - , S.exactly "scripts/debian/builder-helpers" "sh" - , S.exactly "scripts/docker/release" "sh" - , S.exactly "scripts/docker/build" "sh" + , S.strictlyStart (S.contains "scripts/debian") + , S.strictlyStart (S.contains "scripts/docker") , S.exactly "buildkite/scripts/build-artifact" "sh" , S.exactly "buildkite/scripts/build-hardfork-package" "sh" , S.exactly "buildkite/scripts/check-compatibility" "sh" diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index 7db6179132c..926a077b77d 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -1,9 +1,3 @@ -let Prelude = ../External/Prelude.dhall - -let Optional/map = Prelude.Optional.map - -let Optional/default = Prelude.Optional.default - let Profiles = ./Profiles.dhall let Artifacts = ./Artifacts.dhall @@ -36,46 +30,47 @@ let lowerName = let dependsOn = \(docker : Docker) - -> \(network : Optional Network.Type) + -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> let profileSuffix = Profiles.toSuffixUppercase profile + -> let network = "${Network.capitalName network}" + + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let prefix = "MinaArtifact" let suffix = "docker-image" - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - - let key = - "${Artifacts.lowerName - binary}${networkOrDefault}-${lowerName docker}-${suffix}" + let key = "${Artifacts.lowerName binary}-${suffix}" in merge { Bookworm = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Bullseye = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${profileSuffix}" , key = key } ] , Jammy = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] , Focal = - [ { name = "${prefix}${capitalName docker}${profileSuffix}" + [ { name = + "${prefix}${capitalName + docker}${network}${capitalName + docker}${profileSuffix}" , key = key } ] diff --git a/buildkite/src/Constants/Network.dhall b/buildkite/src/Constants/Network.dhall index 92462c87ae3..86094577017 100644 --- a/buildkite/src/Constants/Network.dhall +++ b/buildkite/src/Constants/Network.dhall @@ -4,23 +4,18 @@ let List/any = Prelude.List.any let Network : Type - = < Devnet | Mainnet | Berkeley > + = < Devnet | Mainnet > let capitalName = \(network : Network) - -> merge - { Devnet = "Devnet", Mainnet = "Mainnet", Berkeley = "Berkeley" } - network + -> merge { Devnet = "Devnet", Mainnet = "Mainnet" } network let lowerName = \(network : Network) - -> merge - { Devnet = "devnet", Mainnet = "mainnet", Berkeley = "berkeley" } - network + -> merge { Devnet = "devnet", Mainnet = "mainnet" } network let requiresMainnetBuild = - \(network : Network) - -> merge { Devnet = True, Mainnet = True, Berkeley = False } network + \(network : Network) -> merge { Devnet = False, Mainnet = True } network let foldMinaBuildMainnetEnv = \(networks : List Network) @@ -30,9 +25,28 @@ let foldMinaBuildMainnetEnv = else "MINA_BUILD_MAINNET=false" +let buildMainnetEnv = + \(network : Network) + -> if requiresMainnetBuild network + + then "MINA_BUILD_MAINNET=true" + + else "MINA_BUILD_MAINNET=false" + +let foldNames = + \(networks : List Network) + -> Prelude.List.fold + Network + networks + Text + (\(x : Network) -> \(y : Text) -> "${capitalName x}" ++ y) + "" + in { Type = Network , capitalName = capitalName , lowerName = lowerName , requiresMainnetBuild = requiresMainnetBuild , foldMinaBuildMainnetEnv = foldMinaBuildMainnetEnv + , buildMainnetEnv = buildMainnetEnv + , foldNames = foldNames } diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall index 57c2ecc0099..9d7bc0acc73 100644 --- a/buildkite/src/Jobs/Lint/Merge.dhall +++ b/buildkite/src/Jobs/Lint/Merge.dhall @@ -73,19 +73,6 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } - , Command.build - Command.Config::{ - , commands = - [ Cmd.run "scripts/merged-to-proof-systems.sh berkeley" ] - , label = - "[proof-systems] Check merges cleanly into proof-systems berkeley branch" - , key = "merged-to-proof-systems-berkeley" - , soft_fail = Some (B/SoftFail.Boolean True) - , target = Size.Small - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } - } , Command.build Command.Config::{ , commands = diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall deleted file mode 100644 index 5c88037f8d1..00000000000 --- a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall +++ /dev/null @@ -1,46 +0,0 @@ -let S = ../../Lib/SelectFiles.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let JobSpec = ../../Pipeline/JobSpec.dhall - -let DockerImage = ../../Command/DockerImage.dhall - -let Profiles = ../../Constants/Profiles.dhall - -let DebianVersions = ../../Constants/DebianVersions.dhall - -let DebianRepo = ../../Constants/DebianRepo.dhall - -let spec = - DockerImage.ReleaseSpec::{ - , service = "itn-orchestrator" - , step_key = "itn-orchestrator-docker-image" - , network = "berkeley" - , deb_repo = DebianRepo.Type.Local - , deps = - DebianVersions.dependsOn - DebianVersions.DebVersion.Bullseye - Profiles.Type.Standard - } - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart - (S.contains "buildkite/src/Jobs/Release/ItnOrchestratorArtifact") - , S.strictlyStart (S.contains "src/app/itn_orchestrator") - ] - , path = "Release" - , name = "ItnOrchestratorArtifact" - , tags = - [ PipelineTag.Type.Long - , PipelineTag.Type.Release - , PipelineTag.Type.Stable - ] - } - , steps = [ DockerImage.generateStep spec ] - } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetLightnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall similarity index 88% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index aa4037d8626..18633217d3c 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -21,13 +21,12 @@ in Pipeline.build , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction ] - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall similarity index 89% rename from buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 82d9d49a532..76829c42836 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactMainnetFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -24,13 +24,12 @@ in Pipeline.build , Artifacts.Type.ZkappTestTransaction ] , debVersion = DebianVersions.DebVersion.Focal - , networks = [ Network.Type.Devnet, Network.Type.Mainnet ] + , network = Network.Type.Mainnet , tags = [ PipelineTag.Type.Long , PipelineTag.Type.Release , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable - , prefix = "MinaArtifactMainnet" } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseye.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall index 86e483e1e31..357da774d05 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -27,10 +31,9 @@ in Pipeline.build , steps = [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "bullseye" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Bullseye , no_cache = True - , step_key = "toolchain-bullseye-docker-image" } in DockerImage.generateStep toolchainBullseyeSpec diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall index ab0030f99f4..2ffe0472892 100644 --- a/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall +++ b/buildkite/src/Jobs/Release/MinaToolchainArtifactFocal.dhall @@ -8,6 +8,10 @@ let JobSpec = ../../Pipeline/JobSpec.dhall let DockerImage = ../../Command/DockerImage.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -25,14 +29,13 @@ in Pipeline.build , tags = [ PipelineTag.Type.Toolchain ] } , steps = - [ let toolchainSpec = + [ let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{ - , service = "mina-toolchain" - , deb_codename = "focal" + , service = Artifacts.Type.Toolchain + , deb_codename = DebianVersions.DebVersion.Focal , no_cache = True - , step_key = "toolchain-focal-docker-image" } - in DockerImage.generateStep toolchainSpec + in DockerImage.generateStep toolchainBullseyeSpec ] } diff --git a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall b/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall deleted file mode 100644 index b44d2b6ea96..00000000000 --- a/buildkite/src/Jobs/Test/BerkeleyCompatibility.dhall +++ /dev/null @@ -1,48 +0,0 @@ -let JobSpec = ../../Pipeline/JobSpec.dhall - -let Pipeline = ../../Pipeline/Dsl.dhall - -let PipelineTag = ../../Pipeline/Tag.dhall - -let Cmd = ../../Lib/Cmds.dhall - -let S = ../../Lib/SelectFiles.dhall - -let Command = ../../Command/Base.dhall - -let Docker = ../../Command/Docker/Type.dhall - -let Size = ../../Command/Size.dhall - -let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] - -in Pipeline.build - Pipeline.Config::{ - , spec = JobSpec::{ - , dirtyWhen = - [ S.strictlyStart (S.contains "src") - , S.exactly "buildkite/scripts/check-compatibility" "sh" - , S.exactly "buildkite/src/Jobs/Test/BerkeleyCompatibility" "dhall" - ] - , path = "Test" - , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] - , name = "BerkeleyCompatibility" - } - , steps = - [ Command.build - Command.Config::{ - , commands = - [ Cmd.run "buildkite/scripts/check-compatibility.sh berkeley" ] - , label = "Test: berkeley compatibilty test" - , key = "berkeley-compatibilty-test" - , target = Size.XLarge - , docker = None Docker.Type - , depends_on = dependsOn - , timeout_in_minutes = Some +60 - } - ] - } diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index dee29949cd5..bef4ebb1e47 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -12,9 +12,12 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall similarity index 74% rename from buildkite/src/Jobs/Test/ConnectToBerkeley.dhall rename to buildkite/src/Jobs/Test/ConnectToDevnet.dhall index 312e63f9435..526ee022143 100644 --- a/buildkite/src/Jobs/Test/ConnectToBerkeley.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -14,18 +14,15 @@ let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall let Network = ../../Constants/Network.dhall -let Dockers = ../../Constants/DockerVersions.dhall - let dependsOn = - Dockers.dependsOn - Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard - Artifacts.Type.Daemon in Pipeline.build Pipeline.Config::{ @@ -33,17 +30,17 @@ in Pipeline.build , dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/scripts/connect-to-testnet" "sh" - , S.exactly "buildkite/src/Jobs/Test/ConnectToBerkeley" "dhall" + , S.exactly "buildkite/src/Jobs/Test/ConnectToDevnet" "dhall" , S.exactly "buildkite/src/Command/ConnectToTestnet" "dhall" ] , path = "Test" - , name = "ConnectToBerkeley" + , name = "ConnectToDevnet" , tags = [ PipelineTag.Type.Long, PipelineTag.Type.Test ] } , steps = [ ConnectToTestnet.step dependsOn - "berkeley" + "${Network.lowerName Network.Type.Devnet}" "40s" "2m" (B/SoftFail.Boolean True) diff --git a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall index 99132224a5b..8bcecf6c27f 100644 --- a/buildkite/src/Jobs/Test/DevelopCompatibility.dhall +++ b/buildkite/src/Jobs/Test/DevelopCompatibility.dhall @@ -14,11 +14,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + let dependsOn = - [ { name = "MinaArtifactBullseye" - , key = "daemon-berkeley-bullseye-docker-image" - } - ] + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Lightnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall new file mode 100644 index 00000000000..8bbf02c72ef --- /dev/null +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -0,0 +1,44 @@ +let S = ../../Lib/SelectFiles.dhall + +let Pipeline = ../../Pipeline/Dsl.dhall + +let PipelineTag = ../../Pipeline/Tag.dhall + +let JobSpec = ../../Pipeline/JobSpec.dhall + +let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let Dockers = ../../Constants/DockerVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Artifacts = ../../Constants/Artifacts.dhall + +let dependsOn = + Dockers.dependsOn + Dockers.Type.Bullseye + Network.Type.Devnet + Profiles.Type.Standard + Artifacts.Type.FunctionalTestSuite + +in Pipeline.build + Pipeline.Config::{ + , spec = JobSpec::{ + , dirtyWhen = + [ S.strictlyStart (S.contains "src") + , S.exactly "scripts/path-archive-test" "sh" + , S.exactly "buildkite/src/Jobs/Test/PatchArchiveTest" "dhall" + , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall" + ] + , path = "Test" + , name = "PatchArchiveTest" + , tags = + [ PipelineTag.Type.Long + , PipelineTag.Type.Test + , PipelineTag.Type.Stable + ] + } + , steps = [ PatchArchiveTest.step dependsOn ] + } diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 0e3d665e2ce..24ce7acf7e1 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -19,7 +19,7 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 4d7ae4b9c3b..21bfdcb7498 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -18,14 +18,16 @@ let Profiles = ../../Constants/Profiles.dhall let Dockers = ../../Constants/DockerVersions.dhall -let Artifacts = ../../Constants/Artifacts.dhall +let RunWithPostgres = ../../Command/RunWithPostgres.dhall let Network = ../../Constants/Network.dhall -let RunWithPostgres = ../../Command/RunWithPostgres.dhall +let Artifacts = ../../Constants/Artifacts.dhall let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,12 +57,13 @@ in Pipeline.build ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" Artifacts.Type.Rosetta - (Some Network.Type.Berkeley) + (Some Network.Type.Devnet) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-fast.sh" ] @@ -71,7 +74,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 10ff09fea4e..ec77336a12c 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -20,12 +20,14 @@ let Network = ../../Constants/Network.dhall let Profiles = ../../Constants/Profiles.dhall -let Artifacts = ../../Constants/Artifacts.dhall - let Dockers = ../../Constants/DockerVersions.dhall +let Artifacts = ../../Constants/Artifacts.dhall + let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type +let network = Network.Type.Devnet + let dirtyWhen = [ S.strictlyStart (S.contains "src") , S.exactly "buildkite/src/Jobs/Test/RosettaIntegrationTests" "dhall" @@ -55,7 +57,8 @@ in Pipeline.build , Cmd.runInDocker Cmd.Docker::{ , image = - "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-berkeley" + "gcr.io/o1labs-192920/mina-rosetta:\\\${MINA_DOCKER_TAG}-${Network.lowerName + network}" } "buildkite/scripts/rosetta-integration-tests-full.sh" ] @@ -66,7 +69,7 @@ in Pipeline.build , depends_on = Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + network Profiles.Type.Standard Artifacts.Type.Rosetta } diff --git a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall index 0523f1e1e74..accae521318 100644 --- a/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall +++ b/buildkite/src/Jobs/Test/RunSnarkProfiler.dhall @@ -16,11 +16,14 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall +let Network = ../../Constants/Network.dhall + let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 74959dd325a..e5bbae08182 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -14,6 +14,8 @@ let DebianVersions = ../../Constants/DebianVersions.dhall let Profiles = ../../Constants/Profiles.dhall +let Network = ../../Constants/Network.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -21,9 +23,11 @@ let Size = ../../Command/Size.dhall let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Lightnet # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye + Network.Type.Devnet Profiles.Type.Standard let buildTestCmd diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index 933a21bae6e..a388d2e0220 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index c02eb9c5262..85a8b614ee2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -21,12 +21,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye - (Some Network.Type.Berkeley) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye - (None Network.Type) + Network.Type.Devnet Profiles.Type.Standard Artifacts.Type.Archive diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index bc47db104ad..971dd12d44e 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -20,7 +20,17 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let dependsOn = [ { name = "MinaArtifactBullseye", key = "build-deb-pkg" } ] +let DebianVersions = ../../Constants/DebianVersions.dhall + +let Network = ../../Constants/Network.dhall + +let Profiles = ../../Constants/Profiles.dhall + +let dependsOn = + DebianVersions.dependsOn + DebianVersions.DebVersion.Bullseye + Network.Type.Devnet + Profiles.Type.Standard let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type diff --git a/scripts/debian/aptly.sh b/scripts/debian/aptly.sh index 8eae74c3544..408bda8f105 100755 --- a/scripts/debian/aptly.sh +++ b/scripts/debian/aptly.sh @@ -12,7 +12,6 @@ CLEAR='\033[0m' RED='\033[0;31m' # global variables -declare CLI_VERSION='1.0.0'; declare CLI_NAME='aptly.sh'; declare PS4='debug($LINENO) ${FUNCNAME[0]:+${FUNCNAME[0]}}(): '; @@ -34,28 +33,28 @@ function start_aptly() { local __background=$3 local __clean=$4 local __component=$5 - local __repo="$__distribution-$__component" + local __repo="${__distribution}"-"${__component}" local __port=$6 - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi - - aptly repo create -component $__component -distribution $__distribution $__repo - aptly repo add $__repo $__debs + aptly repo create -component "${__component}" -distribution "${__distribution}" "${__repo}" - aptly snapshot create $__component from repo $__repo + aptly repo add "${__repo}" "${__debs}" - aptly publish snapshot -distribution=$__distribution -skip-signing $__component + aptly snapshot create "${__component}" from repo "${__repo}" - if [ $__background = 1 ]; then - aptly serve -listen localhost:$__port & - else - aptly serve -listen localhost:$__port + aptly publish snapshot -distribution="${__distribution}" -skip-signing "${__component}" + + if [ "${__background}" = 1 ]; then + aptly serve -listen localhost:"${__port}" & + else + aptly serve -listen localhost:"${__port}" fi - + } @@ -68,7 +67,7 @@ function start_help(){ echo "" echo "Parameters:" echo "" - echo " -b, --background The Docker name (mina-berkeley, mina-archive etc.)" + echo " -b, --background The Docker name (mina, mina-archive etc.)" echo " -c, --codename The Codename for debian repository" echo " -d, --debians The Debian(s) to be available in aptly. Supports regular expression" echo " -m, --component The Component for debian repository. For example: unstable" @@ -91,7 +90,7 @@ function start(){ local __clean=0 local __component="unstable" local __port=$PORT - + while [ ${#} -gt 0 ]; do error_message="Error: a value is needed for '$1'"; @@ -99,7 +98,7 @@ function start(){ -h | --help ) start_help; ;; - -b | --background ) + -b | --background ) __background=1 shift; ;; @@ -130,13 +129,13 @@ function start(){ ;; esac done - - start_aptly $__distribution \ - $__debs \ - $__background \ - $__clean \ - $__component \ - $__port + + start_aptly "${__distribution}" \ + "${__debs}" \ + "${__background}" \ + "${__clean}" \ + "${__component}" \ + "${__port}" } @@ -156,9 +155,9 @@ function stop_help(){ } function stop(){ - + local __clean=0 - + while [ ${#} -gt 0 ]; do case $1 in -h | --help ) @@ -175,9 +174,9 @@ function stop(){ ;; esac done - + pkill aptly - if [ $__clean = 1 ]; then + if [ "${__clean}" = 1 ]; then rm -rf ~/.aptly fi } @@ -215,4 +214,4 @@ function main(){ esac } -main "$@"; \ No newline at end of file +main "$@"; diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index 1df887ce9d9..610e2e41b5a 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -8,16 +8,16 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # In case of running this script on detached head, script has difficulties in finding out # what is the current branch. -if [[ -n "$BRANCH_NAME" ]]; then - BRANCH_NAME="$BRANCH_NAME" source ${SCRIPTPATH}/../export-git-env-vars.sh +if [[ -n "${BRANCH_NAME}" ]]; then + BRANCH_NAME="${BRANCH_NAME}" source "${SCRIPTPATH}"/../export-git-env-vars.sh else - source ${SCRIPTPATH}/../export-git-env-vars.sh -fi + source "${SCRIPTPATH}"/../export-git-env-vars.sh +fi echo "after export" -source ${SCRIPTPATH}/builder-helpers.sh - +source "${SCRIPTPATH}"/builder-helpers.sh + if [ $# -eq 0 ] then echo "No arguments supplied. Building all known debian packages" @@ -25,22 +25,21 @@ if [ $# -eq 0 ] build_keypair_deb build_archive_deb build_batch_txn_deb - build_daemon_berkeley_deb build_daemon_mainnet_deb build_daemon_devnet_deb build_test_executive_deb build_functional_test_suite_deb build_zkapp_test_transaction_deb - else + else for i in "$@"; do - if [[ $(type -t "build_${i}_deb") == function ]] - then + if [[ $(type -t "build_${i}_deb") == function ]] + then echo "Building $i debian package" "build_${i}_deb" - else + else echo "invalid debian package name '$i'" exit 1 fi done -fi \ No newline at end of file +fi diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 0e34178731d..96539eb5b9d 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -10,7 +10,7 @@ MINA_DEB_RELEASE=${MINA_DEB_RELEASE:-"unstable"} # Helper script to include when building deb archives. -echo "--- Setting up the envrionment to build debian packages..." +echo "--- Setting up the environment to build debian packages..." SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" cd "${SCRIPTPATH}/../../_build" @@ -46,13 +46,13 @@ esac # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" + MINA_DEB_NAME="mina-devnet" DEB_SUFFIX="" ;; *) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" + MINA_DEB_NAME="mina-devnet-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac @@ -63,7 +63,7 @@ if [[ -v DUNE_INSTRUMENT_WITH ]]; then INSTRUMENTED_SUFFIX=instrumented MINA_DEB_NAME="${MINA_DEB_NAME}-${INSTRUMENTED_SUFFIX}" DEB_SUFFIX="${DEB_SUFFIX}-${INSTRUMENTED_SUFFIX}" -fi +fi BUILDDIR="deb_build" @@ -123,9 +123,9 @@ build_deb() { # Build the package echo "------------------------------------------------------------" - fakeroot dpkg-deb --build "${BUILDDIR}" ${1}_${MINA_DEB_VERSION}.deb + fakeroot dpkg-deb --build "${BUILDDIR}" "${1}"_"${MINA_DEB_VERSION}".deb echo "build_deb outputs:" - ls -lh ${1}_*.deb + ls -lh "${1}"_*.deb echo "deleting BUILDDIR ${BUILDDIR}" rm -rf "${BUILDDIR}" @@ -137,9 +137,9 @@ copy_common_daemon_configs() { echo "------------------------------------------------------------" echo "copy_common_daemon_configs inputs:" - echo "Network Name: ${1} (like mainnet, devnet, berkeley)" + echo "Network Name: ${1} (like mainnet, devnet)" echo "Signature Type: ${2} (mainnet or testnet)" - echo "Seed List URL path: ${3} (like seed-lists/berkeley_seeds.txt)" + echo "Seed List URL path: ${3} (like seed-lists/devnet_seeds.txt)" # Copy shared binaries cp ../src/app/libp2p_helper/result/bin/libp2p_helper "${BUILDDIR}/usr/local/bin/coda-libp2p_helper" @@ -148,7 +148,7 @@ copy_common_daemon_configs() { cp ./default/src/app/validate_keypair/validate_keypair.exe "${BUILDDIR}/usr/local/bin/mina-validate-keypair" # Copy signature-based Binaries (based on signature type $2 passed into the function) - cp ./default/src/app/cli/src/mina_${2}_signatures.exe "${BUILDDIR}/usr/local/bin/mina" + cp ./default/src/app/cli/src/mina_"${2}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina" # Copy over Build Configs (based on $2) mkdir -p "${BUILDDIR}/etc/coda/build_config" @@ -161,9 +161,8 @@ copy_common_daemon_configs() { # Include all useful genesis ledgers cp ../genesis_ledgers/mainnet.json "${BUILDDIR}/var/lib/coda/mainnet.json" cp ../genesis_ledgers/devnet.json "${BUILDDIR}/var/lib/coda/devnet.json" - cp ../genesis_ledgers/berkeley.json "${BUILDDIR}/var/lib/coda/berkeley.json" # Set the default configuration based on Network name ($1) - cp ../genesis_ledgers/${1}.json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" + cp ../genesis_ledgers/"${1}".json "${BUILDDIR}/var/lib/coda/config_${GITHASH_CONFIG}.json" cp ../scripts/hardfork/create_runtime_config.sh "${BUILDDIR}/usr/local/bin/mina-hf-create-runtime-config" cp ../scripts/mina-verify-packaged-fork-config "${BUILDDIR}/usr/local/bin/mina-verify-packaged-fork-config" # Update the mina.service with a new default PEERS_URL based on Seed List URL $3 @@ -239,20 +238,25 @@ build_batch_txn_deb() { build_functional_test_suite_deb() { create_control_file mina-test-suite "${SHARED_DEPS}" 'Test suite apps for mina.' + mkdir -p "${BUILDDIR}/etc/mina/test/archive" + + cp -r ../src/test/archive/* "${BUILDDIR}"/etc/mina/test/archive/ + # Binaries cp ./default/src/test/command_line_tests/command_line_tests.exe "${BUILDDIR}/usr/local/bin/mina-command-line-tests" - + cp ./default/src/test/archive/patch_archive_test/patch_archive_test.exe "${BUILDDIR}/usr/local/bin/mina-patch-archive-test" + build_deb mina-test-suite } ##################################### END TEST SUITE PACKAGE ####################################### function copy_common_rosetta_configs () { - - # Copy rosetta-based Binaries - cp ./default/src/app/rosetta/rosetta_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" - cp ./default/src/app/rosetta/ocaml-signer/signer_${1}_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" - + + # Copy rosetta-based Binaries + cp ./default/src/app/rosetta/rosetta_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-rosetta" + cp ./default/src/app/rosetta/ocaml-signer/signer_"${1}"_signatures.exe "${BUILDDIR}/usr/local/bin/mina-ocaml-signer" + mkdir -p "${BUILDDIR}/etc/mina/rosetta" mkdir -p "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" mkdir -p "${BUILDDIR}/etc/mina/rosetta/scripts" @@ -262,52 +266,38 @@ function copy_common_rosetta_configs () { cp ../src/app/rosetta/rosetta-cli-config/*.json "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ../src/app/rosetta/rosetta-cli-config/*.ros "${BUILDDIR}/etc/mina/rosetta/rosetta-cli-config" cp ./default/src/app/rosetta/indexer_test/indexer_test.exe "${BUILDDIR}/usr/local/bin/mina-rosetta-indexer-test" - + } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet rosetta deb" create_control_file mina-rosetta-mainnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "mainnet" - + build_deb mina-rosetta-mainnet } ##################################### ROSETTA MAINNET PACKAGE ####################################### build_rosetta_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building devnet rosetta deb" create_control_file mina-rosetta-devnet "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" copy_common_rosetta_configs "testnet" - - build_deb mina-rosetta-devnet -} - -##################################### ROSETTA BERKELEY PACKAGE ####################################### -build_rosetta_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building rosetta berkeley deb" - - create_control_file mina-rosetta-berkeley "${SHARED_DEPS}" 'Mina Protocol Rosetta Client' "${SUGGESTED_DEPS}" - copy_common_rosetta_configs "testnet" - - build_deb mina-rosetta-berkeley + build_deb mina-rosetta-devnet } - ##################################### MAINNET PACKAGE ####################################### build_daemon_mainnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building mainnet deb without keys:" @@ -321,32 +311,17 @@ build_daemon_mainnet_deb() { ##################################### DEVNET PACKAGE ####################################### build_daemon_devnet_deb() { - + echo "------------------------------------------------------------" echo "--- Building testnet signatures deb without keys:" - create_control_file mina-devnet "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' "${SUGGESTED_DEPS}" + create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon for the Devnet Network' copy_common_daemon_configs devnet testnet 'seed-lists/devnet_seeds.txt' - build_deb mina-devnet -} -##################################### END DEVNET PACKAGE ####################################### - -##################################### BERKELEY PACKAGE ####################################### -build_daemon_berkeley_deb() { - - echo "------------------------------------------------------------" - echo "--- Building Mina Berkeley testnet signatures deb without keys:" - - create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon' - - copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt' - build_deb "${MINA_DEB_NAME}" - } -##################################### END BERKELEY PACKAGE ####################################### +##################################### END DEVNET PACKAGE ####################################### ##################################### ARCHIVE PACKAGE ########################################## build_archive_deb () { @@ -362,13 +337,13 @@ build_archive_deb () { cp ./default/src/app/archive/archive.exe "${BUILDDIR}/usr/local/bin/mina-archive" cp ./default/src/app/archive_blocks/archive_blocks.exe "${BUILDDIR}/usr/local/bin/mina-archive-blocks" cp ./default/src/app/extract_blocks/extract_blocks.exe "${BUILDDIR}/usr/local/bin/mina-extract-blocks" - + mkdir -p "${BUILDDIR}/etc/mina/archive" - cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/etc/mina/archive" - + cp ../scripts/archive/missing-blocks-guardian.sh "${BUILDDIR}/usr/local/bin/mina-missing-blocks-guardian" + cp ./default/src/app/missing_blocks_auditor/missing_blocks_auditor.exe "${BUILDDIR}/usr/local/bin/mina-missing-blocks-auditor" cp ./default/src/app/replayer/replayer.exe "${BUILDDIR}/usr/local/bin/mina-replayer" - + cp ../src/app/archive/create_schema.sql "${BUILDDIR}/etc/mina/archive" cp ../src/app/archive/drop_tables.sql "${BUILDDIR}/etc/mina/archive" @@ -386,6 +361,6 @@ build_zkapp_test_transaction_deb () { # Binaries cp ./default/src/app/zkapp_test_transaction/zkapp_test_transaction.exe "${BUILDDIR}/usr/local/bin/mina-zkapp-test-transaction" - build_deb mina-zkapp-test-transaction + build_deb mina-zkapp-test-transaction } ##################################### END ZKAPP TEST TXN PACKAGE ####################################### diff --git a/scripts/debian/publish.sh b/scripts/debian/publish.sh index f20d9b53d33..fe36fb111e2 100755 --- a/scripts/debian/publish.sh +++ b/scripts/debian/publish.sh @@ -52,33 +52,44 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \ echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}" # Upload the deb files to s3. # If this fails, attempt to remove the lockfile and retry. -for i in {1..10}; do ( +for _i in {1..10}; do ( ${DEBS3_UPLOAD} \ --component "${DEB_RELEASE}" \ --codename "${DEB_CODENAME}" \ "${DEB_NAMES}" ) && break || scripts/debian/clear-s3-lockfile.sh; done -# Verify integrity of debs on remote repo -function verify_o1test_repo_has_package { - sudo apt-get update - ${DEBS3_SHOW} ${1} ${DEB_VERSION} $ARCH -c $DEB_CODENAME -m $DEB_RELEASE - return $? -} - for deb in $DEB_NAMES do - echo "Adding packages.o1test.net $DEB_CODENAME $DEB_RELEASE" - sudo echo "deb [trusted=yes] http://packages.o1test.net $DEB_CODENAME $DEB_RELEASE" | sudo tee /etc/apt/sources.list.d/mina.list + echo "Adding packages.o1test.net ${DEB_CODENAME} ${DEB_RELEASE}" + sudo echo "deb [trusted=yes] http://packages.o1test.net ${DEB_CODENAME} ${DEB_RELEASE}" | sudo tee /etc/apt/sources.list.d/mina.list DEBS3_SHOW="deb-s3 show $BUCKET_ARG $S3_REGION_ARG" - deb_split=(${deb//_/ }) - deb="${deb_split[0]}" - deb=$(basename $deb) + # extracting name from debian package path. E.g: + # _build/mina-archive_3.0.1-develop-a2a872a.deb -> mina-archive + deb=$(basename "$deb") + deb="${deb%_*}" - for i in {1..10}; do (verify_o1test_repo_has_package $deb) && break || sleep 60; done - -done + for i in {1..10}; do + + sudo apt-get update + ${DEBS3_SHOW} "$deb" "${DEB_VERSION}" "${ARCH}" -c "${DEB_CODENAME}" -m "${DEB_RELEASE}" + LAST_VERIFY_STATUS=$? + + if [[ $LAST_VERIFY_STATUS == 0 ]]; then + echo "succesfully validated that package is uploaded to deb-s3" + break + fi + + sleep 60 + i=$((i+1)) + done + if [[ $LAST_VERIFY_STATUS != 0 ]]; then + echo "Cannot locate '$deb' in debian repo. failing job..." + echo "You may still try to rerun job as debian repository is known from imperfect performance" + exit 1 + fi +done diff --git a/scripts/debian/reversion.sh b/scripts/debian/reversion.sh index 0aa2c16faf3..b17bbbab95b 100755 --- a/scripts/debian/reversion.sh +++ b/scripts/debian/reversion.sh @@ -49,8 +49,8 @@ if [[ -z "$SUITE" ]]; then NEW_SUITE=$SUITE; fi; function rebuild_deb() { rm -f "${DEB}_${VERSION}.deb" rm -rf "${NEW_NAME}_${NEW_VERSION}" - - wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/${CODENAME}/m/mi/${DEB}_${VERSION}.deb + + wget https://s3.us-west-2.amazonaws.com/packages.o1test.net/pool/"${CODENAME}"/m/mi/"${DEB}"_"${VERSION}".deb dpkg-deb -R "${DEB}_${VERSION}.deb" "${NEW_NAME}_${NEW_VERSION}" sed -i 's/Version: '"${VERSION}"'/Version: '"${NEW_VERSION}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" sed -i 's/Package: '"${DEB}"'/Package: '"${NEW_NAME}"'/g' "${NEW_NAME}_${NEW_VERSION}/DEBIAN/control" @@ -60,4 +60,4 @@ function rebuild_deb() { rebuild_deb -source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version ${NEW_VERSION} --codename ${CODENAME} --release ${NEW_RELEASE} +source scripts/debian/publish.sh --names "${NEW_NAME}_${NEW_VERSION}.deb" --version "${NEW_VERSION}" --codename "${CODENAME}" --release "${NEW_RELEASE}" diff --git a/scripts/debian/verify.sh b/scripts/debian/verify.sh index c61d3636a7e..25877aa3f67 100755 --- a/scripts/debian/verify.sh +++ b/scripts/debian/verify.sh @@ -13,7 +13,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -if [ -z $PACKAGE ]; then +if [ -z "${PACKAGE}" ]; then echo "No package defined. exiting.."; exit 1; fi diff --git a/scripts/patch-archive-test.sh b/scripts/patch-archive-test.sh new file mode 100755 index 00000000000..48e1c61861a --- /dev/null +++ b/scripts/patch-archive-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -x +# test replayer on known archive db + +NETWORK_DATA_FOLDER=src/test/archive/sample_db +PATCH_ARCHIVE_TEST_APP=${PATCH_ARCHIVE_TEST_APP:-_build/default/src/test/archive/patch_archive_test/patch_archive_test.exe} +PG_PORT=${PG_PORT:-5432} +POSTGRES_USER=${POSTGRES_USER:-postgres} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} + +CONN=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${PG_PORT} + + +echo "Running patch archive test" +$PATCH_ARCHIVE_TEST_APP --source-uri $CONN \ + --network-data-folder $NETWORK_DATA_FOLDER diff --git a/scripts/replayer-test.sh b/scripts/replayer-test.sh index b75c32f4b08..8b6f511295b 100755 --- a/scripts/replayer-test.sh +++ b/scripts/replayer-test.sh @@ -5,7 +5,7 @@ set -x INPUT_FILE=src/test/archive/sample_db/replayer_input_file.json REPLAYER_APP=_build/default/src/app/replayer/replayer.exe -PG_CONN=postgres://postgres:postgres@localhost:5433/archive +PG_CONN=postgres://postgres:postgres@localhost:5432/archive while [[ "$#" -gt 0 ]]; do case $1 in -i|--input-file) INPUT_FILE="$2"; shift;; diff --git a/src/app/archive/cli/dune b/src/app/archive/cli/dune index 69f4c7aa20c..2a815358764 100644 --- a/src/app/archive/cli/dune +++ b/src/app/archive/cli/dune @@ -2,22 +2,23 @@ (name archive_cli) (public_name archive.cli) (libraries - ;;opam libraries - async_kernel - caqti - archive_lib - core - async - caqti-async - core_kernel - async.async_command - ;;local libraries - genesis_constants - mina_runtime_config - cli_lib - logger - block_time - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_mina))) + ;;opam libraries + async_kernel + caqti + archive_lib + core + async + caqti-async + core_kernel + async.async_command + ;;local libraries + genesis_constants + mina_runtime_config + cli_lib + logger + block_time + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_mina))) diff --git a/src/app/archive/dune b/src/app/archive/dune index 27c2526bfc1..29b24b628ab 100644 --- a/src/app/archive/dune +++ b/src/app/archive/dune @@ -4,9 +4,18 @@ (public_name archive) (modules archive) (modes native) - (libraries archive_cli async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -14,9 +23,19 @@ (public_name archive-testnet) (modules archive_testnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.testnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.testnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package archive) @@ -24,6 +43,16 @@ (public_name archive-mainnet) (modules archive_mainnet_signatures) (modes native) - (libraries archive_cli mina_signature_kind.mainnet async async_unix core_kernel base mina_version bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + archive_cli + mina_signature_kind.mainnet + async + async_unix + core_kernel + base + mina_version + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/archive/lib/dune b/src/app/archive/lib/dune index b24273b9de8..3752fa03a60 100644 --- a/src/app/archive/lib/dune +++ b/src/app/archive/lib/dune @@ -2,80 +2,88 @@ (name archive_lib) (public_name archive_lib) (libraries - ;;opam libraries - sexplib0 - async_unix - base64 - async_rpc_kernel - base.caml - base.base_internalhash_types - bin_prot.shape - async_kernel - caqti-driver-postgresql - core - async - core_kernel - caqti - caqti-async - async.async_rpc - integers - ppx_inline_test.config - uri - ;;local libraries - bounded_types - mina_wire_types - kimchi_backend - child_processes - precomputed_values - coda_genesis_ledger - consensus.vrf - mina_runtime_config - sgn - mina_base.util - kimchi_pasta - kimchi_pasta.basic - pickles - random_oracle - pipe_lib - block_time - currency - staged_ledger - data_hash_lib - transition_frontier_base - genesis_ledger_helper - one_or_two - mina_caqti - mina_generators - zkapp_command_builder - mina_base - mina_ledger - mina_block - transition_frontier - logger - mina_metrics - signature_lib - mina_numbers - unsigned_extended - mina_state - pickles_types - pickles.backend - with_hash - mina_base.import - consensus - genesis_constants - genesis_ledger_helper.lib - verifier - quickcheck_lib - snark_params - mina_transaction - protocol_version - staged_ledger_diff - error_json - ppx_deriving_yojson.runtime - ppx_version.runtime - o1trace - ) - (inline_tests (flags -verbose -show-counts)) + ;;opam libraries + sexplib0 + async_unix + base64 + async_rpc_kernel + base.caml + base.base_internalhash_types + bin_prot.shape + async_kernel + caqti-driver-postgresql + core + async + core_kernel + caqti + caqti-async + async.async_rpc + integers + ppx_inline_test.config + uri + ;;local libraries + bounded_types + mina_wire_types + kimchi_backend + child_processes + precomputed_values + coda_genesis_ledger + consensus.vrf + mina_runtime_config + sgn + mina_base.util + kimchi_pasta + kimchi_pasta.basic + pickles + random_oracle + pipe_lib + block_time + currency + staged_ledger + data_hash_lib + transition_frontier_base + genesis_ledger_helper + one_or_two + mina_caqti + mina_generators + zkapp_command_builder + mina_base + mina_ledger + mina_block + transition_frontier + logger + mina_metrics + signature_lib + mina_numbers + unsigned_extended + mina_state + pickles_types + pickles.backend + with_hash + mina_base.import + consensus + genesis_constants + genesis_ledger_helper.lib + verifier + quickcheck_lib + snark_params + mina_transaction + protocol_version + staged_ledger_diff + error_json + ppx_deriving_yojson.runtime + ppx_version.runtime + o1trace) + (inline_tests + (flags -verbose -show-counts)) (modes native) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf ppx_deriving_yojson h_list.ppx))) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_jane + ppx_custom_printf + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/archive_blocks/dune b/src/app/archive_blocks/dune index 48f0db5aabd..152986ce769 100644 --- a/src/app/archive_blocks/dune +++ b/src/app/archive_blocks/dune @@ -3,27 +3,28 @@ (name archive_blocks) (public_name archive_blocks) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - uri - stdio - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - base.caml - async.async_command - ;; local libraries - logger - mina_block - bounded_types - genesis_constants - archive_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + uri + stdio + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + base.caml + async.async_command + ;; local libraries + logger + mina_block + bounded_types + genesis_constants + archive_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/batch_txn_tool/dune b/src/app/batch_txn_tool/dune index f55ebfdf875..8b957922d9f 100644 --- a/src/app/batch_txn_tool/dune +++ b/src/app/batch_txn_tool/dune @@ -1,37 +1,34 @@ (executable (name batch_txn_tool) (libraries - ;; opam libraries - async.async_command - integers - base - uri - core_kernel - core - async - async_kernel - yojson - async_unix - ;; local libraries - currency - mina_base.import - graphql_lib - signature_lib - mina_base - secrets - mina_numbers - snark_params - unsigned_extended - mina_compile_config - mina_wire_types - integration_test_lib - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async.async_command + integers + base + uri + core_kernel + core + async + async_kernel + yojson + async_unix + ;; local libraries + currency + mina_base.import + graphql_lib + signature_lib + mina_base + secrets + mina_numbers + snark_params + unsigned_extended + mina_compile_config + mina_wire_types + integration_test_lib + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) (preprocessor_deps ../../../graphql_schema.json) - (preprocess (pps - ppx_version - ppx_let - ppx_mina -))) + (preprocess + (pps ppx_version ppx_let ppx_mina))) diff --git a/src/app/benchmarks/benchmarks.ml b/src/app/benchmarks/benchmarks.ml index bc09c3c7b8f..0396bee99c6 100644 --- a/src/app/benchmarks/benchmarks.ml +++ b/src/app/benchmarks/benchmarks.ml @@ -3,11 +3,11 @@ (* you can control which libraries are benchmarked using the environment variable BENCHMARK_LIBRARIES, which is either "all" or a comma-delimited list of libraries; if the variable is not present, or empty, run benchmarks for all libraries - *) +*) open Core_kernel -let available_libraries = ["vrf_lib_tests"; "mina_base"] +let available_libraries = [ "vrf_lib_tests"; "mina_base" ] let run_benchmarks_in_lib libname = printf "Running inline tests in library \"%s\"\n%!" libname ; diff --git a/src/app/benchmarks/dune b/src/app/benchmarks/dune index 23918c24e7c..4052d497da6 100644 --- a/src/app/benchmarks/dune +++ b/src/app/benchmarks/dune @@ -3,8 +3,17 @@ (executable (name benchmarks) (public_name main) - (libraries core_bench.inline_benchmarks vrf_lib_tests mina_base core_kernel core base bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (libraries + core_bench.inline_benchmarks + vrf_lib_tests + mina_base + core_kernel + core + base + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (link_flags -linkall) (modes native)) diff --git a/src/app/best_tip_merger/dune b/src/app/best_tip_merger/dune index 2b8ab0c9e03..93c2ff1087b 100644 --- a/src/app/best_tip_merger/dune +++ b/src/app/best_tip_merger/dune @@ -1,50 +1,50 @@ (executable - (package best_tip_merger) - (name best_tip_merger) - (public_name best_tip_merger) - (modes native) - (libraries - ;; opam libraries - async.async_command - async_kernel - stdio - base.caml - yojson - lib - core_kernel - async - ppx_deriving_yojson.runtime - core - sexplib0 - result - base.base_internalhash_types - async_unix - ;; local libraries - visualization - mina_numbers - mina_state - transition_frontier_extensions - mina_base - logger.file_system - logger - cli_lib - transition_frontier - data_hash_lib - rose_tree - consensus - mina_wire_types - pickles.backend - snark_params - pickles - kimchi_pasta - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_jane - ppx_version - ppx_compare)) - ) + (package best_tip_merger) + (name best_tip_merger) + (public_name best_tip_merger) + (modes native) + (libraries + ;; opam libraries + async.async_command + async_kernel + stdio + base.caml + yojson + lib + core_kernel + async + ppx_deriving_yojson.runtime + core + sexplib0 + result + base.base_internalhash_types + async_unix + ;; local libraries + visualization + mina_numbers + mina_state + transition_frontier_extensions + mina_base + logger.file_system + logger + cli_lib + transition_frontier + data_hash_lib + rose_tree + consensus + mina_wire_types + pickles.backend + snark_params + pickles + kimchi_pasta + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_jane + ppx_version + ppx_compare))) diff --git a/src/app/cli/src/cli_entrypoint/dune b/src/app/cli/src/cli_entrypoint/dune index 9c321cf212c..cefa0dc0d61 100644 --- a/src/app/cli/src/cli_entrypoint/dune +++ b/src/app/cli/src/cli_entrypoint/dune @@ -5,77 +5,84 @@ (public_name cli.mina_cli_entrypoint) (modes native) (libraries - ;;opam libraries - stdio - async_kernel - async_unix - base - core_kernel - core - bin_prot - bin_prot.shape - init - async - core.daemon - base.caml - uri - async.async_command - result - sexplib0 - ;;local libraries - verifier - ledger_proof - snark_worker - mina_block - pipe_lib - gossip_net - data_hash_lib - mina_ledger - precomputed_values - mina_networking - cache_dir - parallel - secrets - logger - mina_lib - currency - signature_lib - mina_base - mina_base.import - error_json - genesis_ledger_helper - consensus - child_processes - node_addrs_and_ports - mina_plugins - logger.file_system - cli_lib - genesis_constants - blake2 - mina_metrics - transaction_witness - snark_work_lib - transaction_snark - mina_compile_config - node_error_service - mina_user_error - file_system - mina_version - mina_net2 - block_time - mina_runtime_config - trust_system - protocol_version - coda_genesis_proof - with_hash - block_producer - prover - blockchain_snark - snarky.backendless - o1trace - ppx_version.runtime - internal_tracing - itn_logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_here ppx_let ppx_sexp_conv ppx_deriving_yojson))) + ;;opam libraries + stdio + async_kernel + async_unix + base + core_kernel + core + bin_prot + bin_prot.shape + init + async + core.daemon + base.caml + uri + async.async_command + result + sexplib0 + ;;local libraries + verifier + ledger_proof + snark_worker + mina_block + pipe_lib + gossip_net + data_hash_lib + mina_ledger + precomputed_values + mina_networking + cache_dir + parallel + secrets + logger + mina_lib + currency + signature_lib + mina_base + mina_base.import + error_json + genesis_ledger_helper + consensus + child_processes + node_addrs_and_ports + mina_plugins + logger.file_system + cli_lib + genesis_constants + blake2 + mina_metrics + transaction_witness + snark_work_lib + transaction_snark + mina_compile_config + node_error_service + mina_user_error + file_system + mina_version + mina_net2 + block_time + mina_runtime_config + trust_system + protocol_version + coda_genesis_proof + with_hash + block_producer + prover + blockchain_snark + snarky.backendless + o1trace + ppx_version.runtime + internal_tracing + itn_logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_here + ppx_let + ppx_sexp_conv + ppx_deriving_yojson))) diff --git a/src/app/cli/src/dune b/src/app/cli/src/dune index ce6717a5cb4..d88c44698a4 100644 --- a/src/app/cli/src/dune +++ b/src/app/cli/src/dune @@ -7,8 +7,10 @@ (modules mina) (modes native) (libraries mina_cli_entrypoint) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -18,8 +20,10 @@ (modules mina_testnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.testnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) (executable @@ -29,6 +33,8 @@ (modules mina_mainnet_signatures) (modes native) (libraries mina_cli_entrypoint mina_signature_kind.mainnet) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60-66)) diff --git a/src/app/cli/src/init/dune b/src/app/cli/src/init/dune index aaa7df52dba..00a4dee6b0a 100644 --- a/src/app/cli/src/init/dune +++ b/src/app/cli/src/init/dune @@ -1,135 +1,158 @@ (rule (targets assets.ml) - (deps (source_tree assets)) - (action (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) + (deps + (source_tree assets)) + (action + (run %{bin:ocaml-crunch} -m plain assets -o assets.ml))) (library (name init) (public_name init) (libraries - ;;opam libraries - stdio - integers - astring - async_ssl - sexplib0 - async.async_rpc - result - graphql - async_kernel - async_unix - uri - core_kernel - core_kernel.uuid - core - core.uuid - base - async - cohttp - graphql_parser - async.async_command - async_rpc_kernel - base.caml - base.base_internalhash_types - graphql-cohttp - cohttp-async - graphql-async - mirage-crypto-ec - base_quickcheck - ;;local libraries - bounded_types - snark_profiler_lib - archive_lib - mina_wire_types - perf_histograms - one_or_two - block_time - child_processes - mina_net2 - snark_worker - allocation_functor - blockchain_snark - genesis_ledger_helper.lib - transaction_inclusion_status - error_json - network_pool - user_command_input - network_peer - logger - itn_logger - coda_genesis_proof - data_hash_lib - currency - genesis_constants - bash_colors - graphql_lib - o1trace_webkit_event - generated_graphql_queries - itn_crypto - mina_transaction - mina_generators - mina_user_error - mina_ledger - mina_block - mina_lib - mina_base - cli_lib - mina_graphql - mina_state - consensus - mina_version - o1trace - protocol_version - node_error_service - genesis_ledger_helper - signature_lib - mina_base.import - mina_numbers - precomputed_values - with_hash - transaction_snark - transaction_snark_tests - transaction_protocol_state - test_util - file_system - daemon_rpcs - trust_system - secrets - participating_state - staged_ledger - mina_commands - mina_runtime_config - coda_genesis_ledger - mina_networking - sgn - unsigned_extended - mina_metrics - group_map - snark_work_lib - mina_compile_config - mina_transaction_logic - verifier - parallel - random_oracle - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pickles_types - pickles - pickles.backend - snark_params - mina_signature_kind - string_sign - zkapp_command_builder - internal_tracing - transaction_snark_scan_state - ) - (instrumentation (backend bisect_ppx)) - (preprocessor_deps ../../../../../graphql_schema.json - ../../../../graphql-ppx-config.inc) - (preprocess (pps ppx_deriving_yojson ppx_mina ppx_version - ppx_base ppx_assert ppx_bench ppx_bin_prot ppx_custom_printf - ppx_fields_conv ppx_fixed_literal ppx_here ppx_inline_test ppx_let - ppx_module_timer ppx_optional ppx_pipebang ppx_sexp_message - ppx_sexp_value ppx_string ppx_typerep_conv ppx_variants_conv - graphql_ppx -- %{read-lines:../../../../graphql-ppx-config.inc}))) + ;;opam libraries + stdio + integers + astring + async_ssl + sexplib0 + async.async_rpc + result + graphql + async_kernel + async_unix + uri + core_kernel + core_kernel.uuid + core + core.uuid + base + async + cohttp + graphql_parser + async.async_command + async_rpc_kernel + base.caml + base.base_internalhash_types + graphql-cohttp + cohttp-async + graphql-async + mirage-crypto-ec + base_quickcheck + ;;local libraries + bounded_types + snark_profiler_lib + archive_lib + mina_wire_types + perf_histograms + one_or_two + block_time + child_processes + mina_net2 + snark_worker + allocation_functor + blockchain_snark + genesis_ledger_helper.lib + transaction_inclusion_status + error_json + network_pool + user_command_input + network_peer + logger + itn_logger + coda_genesis_proof + data_hash_lib + currency + genesis_constants + bash_colors + graphql_lib + o1trace_webkit_event + generated_graphql_queries + itn_crypto + mina_transaction + mina_generators + mina_user_error + mina_ledger + mina_block + mina_lib + mina_base + cli_lib + mina_graphql + mina_state + consensus + mina_version + o1trace + protocol_version + node_error_service + genesis_ledger_helper + signature_lib + mina_base.import + mina_numbers + precomputed_values + with_hash + transaction_snark + transaction_snark_tests + transaction_protocol_state + test_util + file_system + daemon_rpcs + trust_system + secrets + participating_state + staged_ledger + mina_commands + mina_runtime_config + coda_genesis_ledger + mina_networking + sgn + unsigned_extended + mina_metrics + group_map + snark_work_lib + mina_compile_config + mina_transaction_logic + verifier + parallel + random_oracle + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pickles_types + pickles + pickles.backend + snark_params + mina_signature_kind + string_sign + zkapp_command_builder + internal_tracing + transaction_snark_scan_state) + (instrumentation + (backend bisect_ppx)) + (preprocessor_deps + ../../../../../graphql_schema.json + ../../../../graphql-ppx-config.inc) + (preprocess + (pps + ppx_deriving_yojson + ppx_mina + ppx_version + ppx_base + ppx_assert + ppx_bench + ppx_bin_prot + ppx_custom_printf + ppx_fields_conv + ppx_fixed_literal + ppx_here + ppx_inline_test + ppx_let + ppx_module_timer + ppx_optional + ppx_pipebang + ppx_sexp_message + ppx_sexp_value + ppx_string + ppx_typerep_conv + ppx_variants_conv + graphql_ppx + -- + %{read-lines:../../../../graphql-ppx-config.inc}))) diff --git a/src/app/delegation_verify/dune b/src/app/delegation_verify/dune index af2363a72c9..90b52ac1f3a 100644 --- a/src/app/delegation_verify/dune +++ b/src/app/delegation_verify/dune @@ -1,49 +1,50 @@ (executable (name delegation_verify) (libraries - core_kernel - async - async_kernel - async_unix - core - stdio - base - base.caml - ppx_deriving_yojson.runtime - yojson - base64 - integers - async.async_command - sexplib0 - sexplib - hex - ; mina libs - signature_lib - mina_block - transaction_snark - blockchain_snark - mina_base - genesis_constants - uptime_service - currency - ledger_proof - mina_base.import - mina_state - mina_wire_types - consensus - data_hash_lib - mina_numbers - snark_params - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - genesis_ledger_helper - mina_runtime_config - precomputed_values - logger - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) + core_kernel + async + async_kernel + async_unix + core + stdio + base + base.caml + ppx_deriving_yojson.runtime + yojson + base64 + integers + async.async_command + sexplib0 + sexplib + hex + ; mina libs + signature_lib + mina_block + transaction_snark + blockchain_snark + mina_base + genesis_constants + uptime_service + currency + ledger_proof + mina_base.import + mina_state + mina_wire_types + consensus + data_hash_lib + mina_numbers + snark_params + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + genesis_ledger_helper + mina_runtime_config + precomputed_values + logger) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_jane ppx_custom_printf h_list.ppx))) diff --git a/src/app/disk_caching_stats/dune b/src/app/disk_caching_stats/dune index 57292875db8..6a8db8e8632 100644 --- a/src/app/disk_caching_stats/dune +++ b/src/app/disk_caching_stats/dune @@ -1,11 +1,45 @@ (executable (name disk_caching_stats) - (libraries pickles pickles_types pickles.backend snark_params crypto_params network_pool mina_base - signature_lib one_or_two currency ledger_proof mina_state mina_base.import mina_wire_types - mina_numbers random_oracle random_oracle_input kimchi_pasta.basic data_hash_lib with_hash - kimchi_pasta mina_ledger transaction_snark_scan_state mina_transaction_logic transaction_snark - sgn snark_profiler_lib genesis_constants digestif bigarray-compat - ; -- - core core_kernel base base.caml bin_prot splittable_random sexplib0) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_jane ppx_version))) + (libraries + pickles + pickles_types + pickles.backend + snark_params + crypto_params + network_pool + mina_base + signature_lib + one_or_two + currency + ledger_proof + mina_state + mina_base.import + mina_wire_types + mina_numbers + random_oracle + random_oracle_input + kimchi_pasta.basic + data_hash_lib + with_hash + kimchi_pasta + mina_ledger + transaction_snark_scan_state + mina_transaction_logic + transaction_snark + sgn + snark_profiler_lib + genesis_constants + digestif + bigarray-compat + ; -- + core + core_kernel + base + base.caml + bin_prot + splittable_random + sexplib0) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_jane ppx_version))) diff --git a/src/app/dump_blocks/dump_blocks.ml b/src/app/dump_blocks/dump_blocks.ml index f1ca08be9d4..8ecc364d923 100644 --- a/src/app/dump_blocks/dump_blocks.ml +++ b/src/app/dump_blocks/dump_blocks.ml @@ -8,10 +8,7 @@ open Full_frontier.For_tests Choice of the actual codec module depends on more than one CLI argument, which is why it must be delayed until all the CLI arguments are parsed. This choice is made by mk_io function below. *) -type 'a io = - { encoding : 'a - ; filename : string - } +type 'a io = { encoding : 'a; filename : string } type 'a codec = (module Encoding.S with type t = 'a) @@ -41,15 +38,20 @@ let encoded_block = Command.Arg_type.create (fun s -> match String.split ~on:':' s with | [ encoding; filename ] -> - { encoding= + { encoding = ( match String.lowercase encoding with - | "sexp" -> Encoding.Sexp - | "json" -> Json - | "bin" | "binary" -> Binary - | _ -> failwith "invalid encoding" ) + | "sexp" -> + Encoding.Sexp + | "json" -> + Json + | "bin" | "binary" -> + Binary + | _ -> + failwith "invalid encoding" ) ; filename } - | _ -> failwith "invalid format" ) + | _ -> + failwith "invalid format" ) let () = let open Core_kernel in @@ -62,18 +64,14 @@ let precomputed_values = Lazy.force Precomputed_values.for_unit_tests let constraint_constants = precomputed_values.constraint_constants -let output_block : type a. a -> a codec io -> unit - = fun content { encoding; filename } -> +let output_block : type a. a -> a codec io -> unit = + fun content { encoding; filename } -> let module Enc : Encoding.S with type t = a = (val encoding) in let channel = - match filename with - | "-" -> Out_channel.stdout - | s -> Out_channel.create s + match filename with "-" -> Out_channel.stdout | s -> Out_channel.create s in - fprintf channel "%s" (Enc.to_string content); - match filename with - | "-" -> () - | _ -> Out_channel.close channel + fprintf channel "%s" (Enc.to_string content) ; + match filename with "-" -> () | _ -> Out_channel.close channel (* This executable outputs random block to stderr in sexp and json The output is useful for src/lib/mina_block tests when the sexp/json representation changes. *) @@ -91,75 +89,67 @@ let f (type a) ?parent (outputs : a codec io list) make_breadcrumb = List.iter outputs ~f:(fun output -> let module Enc = (val output.encoding) in let content = - Enc.of_breadcrumb - ?with_parent_statehash:parent - breadcrumb + Enc.of_breadcrumb ?with_parent_statehash:parent breadcrumb in - eprintf !"Randomly generated block, %s: %s\n" + eprintf + !"Randomly generated block, %s: %s\n" Enc.name (match output.filename with "-" -> "" | s -> s) ; - output_block content output;); + output_block content output ) ; clean_up_persistent_root ~frontier ) let default_outputs = - [ { encoding= Encoding.Sexp; filename= "-" } - ; { encoding= Encoding.Json; filename= "-" } + [ { encoding = Encoding.Sexp; filename = "-" } + ; { encoding = Encoding.Json; filename = "-" } ] let command = Command.basic ~summary:"Transcribe between block formats or generate a random block" (let%map_open.Command outputs = - flag - "-o" - (listed encoded_block) - ~doc:"output :; formats are: sexp / json / bin; a dash (-) denotes stdout (example: -o json:-)" + flag "-o" (listed encoded_block) + ~doc: + "output :; formats are: sexp / json / bin; a dash (-) \ + denotes stdout (example: -o json:-)" and input = - flag - "-i" - (optional encoded_block) - ~doc:"input :; formats are: sexp / json / bin; a dash (-) denotes stdin (example: -i json:-)" + flag "-i" (optional encoded_block) + ~doc: + "input :; formats are: sexp / json / bin; a dash (-) \ + denotes stdin (example: -i json:-)" and full = - flag - "--full" - no_arg - ~doc:"use full blocks, not just precomputed values" + flag "--full" no_arg ~doc:"use full blocks, not just precomputed values" and parent = - flag - "--parent" - (optional string) - ~aliases:[ "--parent-statehash" ] + flag "--parent" (optional string) ~aliases:[ "--parent-statehash" ] ~doc:"statehash of the parent block" in - let outs = match outputs with - | [] -> default_outputs - | _ -> outputs - in + let outs = match outputs with [] -> default_outputs | _ -> outputs in let conf = if full then - Conf ( Option.map input ~f:(mk_io Block) - , List.map ~f:(mk_io Encoding.Block) outs) + Conf + ( Option.map input ~f:(mk_io Block) + , List.map ~f:(mk_io Encoding.Block) outs ) else - Conf ( Option.map input ~f:(mk_io Precomputed) - , List.map ~f:(mk_io Encoding.Precomputed) outs) + Conf + ( Option.map input ~f:(mk_io Precomputed) + , List.map ~f:(mk_io Encoding.Precomputed) outs ) in fun () -> - match conf with - | Conf (None, outs) -> - let verifier = verifier () in - Core_kernel.Quickcheck.test (gen_breadcrumb ~verifier ()) ~trials:1 ~f:(f ?parent outs) - | Conf (Some { encoding; filename }, outs)-> - let module Enc = (val encoding) in - let input = - match filename with - | "-" -> In_channel.stdin - | s -> In_channel.create s - in - let contents = - In_channel.input_all input - |> Enc.of_string - in - List.iter outs ~f:(output_block contents)) + match conf with + | Conf (None, outs) -> + let verifier = verifier () in + Core_kernel.Quickcheck.test + (gen_breadcrumb ~verifier ()) + ~trials:1 ~f:(f ?parent outs) + | Conf (Some { encoding; filename }, outs) -> + let module Enc = (val encoding) in + let input = + match filename with + | "-" -> + In_channel.stdin + | s -> + In_channel.create s + in + let contents = In_channel.input_all input |> Enc.of_string in + List.iter outs ~f:(output_block contents) ) let () = Command_unix.run command - diff --git a/src/app/dump_blocks/dune b/src/app/dump_blocks/dune index 1e499797f02..4c403c5a92d 100644 --- a/src/app/dump_blocks/dune +++ b/src/app/dump_blocks/dune @@ -2,27 +2,29 @@ (name dump_blocks) (public_name dump_blocks) (libraries - async - async_kernel - async_unix - base - block_time - bounded_types - core - core_kernel - core_unix.command_unix - logger - mina_base - mina_block - mina_state - mina_wire_types - ppx_inline_test.runner.lib - precomputed_values - sexplib0 - transition_frontier - transition_frontier_base - transition_frontier_full_frontier - yojson) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_custom_printf)) + async + async_kernel + async_unix + base + block_time + bounded_types + core + core_kernel + core_unix.command_unix + logger + mina_base + mina_block + mina_state + mina_wire_types + ppx_inline_test.runner.lib + precomputed_values + sexplib0 + transition_frontier + transition_frontier_base + transition_frontier_full_frontier + yojson) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_custom_printf)) (modes native)) diff --git a/src/app/dump_blocks/encoding.ml b/src/app/dump_blocks/encoding.ml index 1a246a1a2b9..dba55a5b52a 100644 --- a/src/app/dump_blocks/encoding.ml +++ b/src/app/dump_blocks/encoding.ml @@ -12,52 +12,49 @@ let block_of_breadcrumb ?with_parent_statehash breadcrumb = let open Mina_block in let block = Frontier_base.Breadcrumb.block breadcrumb in match with_parent_statehash with - | None -> - block - | Some hash -> - let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in - let h = header block in - let state = Header.protocol_state h in - let header = - Header.create - ~protocol_state:{ state with previous_state_hash } - ~protocol_state_proof:(Header.protocol_state_proof h) - ~delta_block_chain_proof:(Header.delta_block_chain_proof h) - ?proposed_protocol_version_opt:(Header.proposed_protocol_version_opt h) - ~current_protocol_version:(Header.current_protocol_version h) - () - in - Mina_block.create ~header ~body:(body block) + | None -> + block + | Some hash -> + let previous_state_hash = Mina_base.State_hash.of_base58_check_exn hash in + let h = header block in + let state = Header.protocol_state h in + let header = + Header.create + ~protocol_state:{ state with previous_state_hash } + ~protocol_state_proof:(Header.protocol_state_proof h) + ~delta_block_chain_proof:(Header.delta_block_chain_proof h) + ?proposed_protocol_version_opt: + (Header.proposed_protocol_version_opt h) + ~current_protocol_version:(Header.current_protocol_version h) + () + in + Mina_block.create ~header ~body:(body block) module type S = sig type t val name : string - val of_breadcrumb : ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t + + val of_breadcrumb : + ?with_parent_statehash:string -> Frontier_base.Breadcrumb.t -> t val to_string : t -> string + val of_string : string -> t end -module Sexp_block : S with type t = Mina_block.t = struct +module Sexp_block : S with type t = Mina_block.t = struct type t = Mina_block.t let name = "sexp" let of_breadcrumb = block_of_breadcrumb + let to_string b = Mina_block.sexp_of_t b |> Sexp.to_string |> append_newline - let to_string b = - Mina_block.sexp_of_t b - |> Sexp.to_string - |> append_newline - - let of_string s = - Sexp.of_string s - |> Mina_block.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.t_of_sexp end - module Binary_block : S with type t = Mina_block.t = struct type t = Mina_block.t @@ -84,16 +81,14 @@ let constraint_constants = precomputed_values.constraint_constants let precomputed_of_breadcrumb ?with_parent_statehash breadcrumb = let open Frontier_base in let block = block_of_breadcrumb ?with_parent_statehash breadcrumb in - let staged_ledger = - Transition_frontier.Breadcrumb.staged_ledger breadcrumb - in + let staged_ledger = Transition_frontier.Breadcrumb.staged_ledger breadcrumb in let scheduled_time = Mina_block.(Header.protocol_state @@ header block) |> Mina_state.Protocol_state.blockchain_state |> Mina_state.Blockchain_state.timestamp in - Mina_block.Precomputed.of_block ~logger ~constraint_constants - ~staged_ledger ~scheduled_time + Mina_block.Precomputed.of_block ~logger ~constraint_constants ~staged_ledger + ~scheduled_time (Breadcrumb.block_with_hash breadcrumb) module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct @@ -104,16 +99,12 @@ module Sexp_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb let to_string b = - Mina_block.Precomputed.sexp_of_t b - |> Sexp.to_string - |> append_newline + Mina_block.Precomputed.sexp_of_t b |> Sexp.to_string |> append_newline - let of_string s = - Sexp.of_string s - |> Mina_block.Precomputed.t_of_sexp + let of_string s = Sexp.of_string s |> Mina_block.Precomputed.t_of_sexp end -module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct +module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct type t = Mina_block.Precomputed.t let name = "json" @@ -122,12 +113,10 @@ module Json_precomputed : S with type t = Mina_block.Precomputed.t = struct let to_string b = Mina_block.Precomputed.to_yojson b - |> Yojson.Safe.to_string - |> append_newline + |> Yojson.Safe.to_string |> append_newline let of_string s = - Yojson.Safe.from_string s - |> Mina_block.Precomputed.of_yojson + Yojson.Safe.from_string s |> Mina_block.Precomputed.of_yojson |> Result.ok_or_failwith end @@ -138,9 +127,7 @@ module Binary_precomputed : S with type t = Mina_block.Precomputed.t = struct let of_breadcrumb = precomputed_of_breadcrumb - let to_string = - Binable.to_string (module Mina_block.Precomputed.Stable.Latest) + let to_string = Binable.to_string (module Mina_block.Precomputed.Stable.Latest) - let of_string = - Binable.of_string (module Mina_block.Precomputed.Stable.Latest) + let of_string = Binable.of_string (module Mina_block.Precomputed.Stable.Latest) end diff --git a/src/app/extract_blocks/dune b/src/app/extract_blocks/dune index a41ea72fc19..1b31d5f0471 100644 --- a/src/app/extract_blocks/dune +++ b/src/app/extract_blocks/dune @@ -3,47 +3,55 @@ (name extract_blocks) (public_name extract_blocks) (libraries - ;; opam libraries - async_unix - core - base - base64 - integers - caqti-async - caqti-driver-postgresql - async - core_kernel - caqti - async_kernel - uri - async.async_command - ;; local libraries - consensus.vrf - mina_wire_types - bounded_types - mina_base - mina_base.import - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - pickles - pickles_types - pickles.backend - archive_lib - data_hash_lib - signature_lib - logger - mina_caqti - block_time - mina_numbers - mina_transaction - protocol_version - snark_params - currency - unsigned_extended - sgn - with_hash - error_json - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx))) + ;; opam libraries + async_unix + core + base + base64 + integers + caqti-async + caqti-driver-postgresql + async + core_kernel + caqti + async_kernel + uri + async.async_command + ;; local libraries + consensus.vrf + mina_wire_types + bounded_types + mina_base + mina_base.import + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + pickles + pickles_types + pickles.backend + archive_lib + data_hash_lib + signature_lib + logger + mina_caqti + block_time + mina_numbers + mina_transaction + protocol_version + snark_params + currency + unsigned_extended + sgn + with_hash + error_json) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv + h_list.ppx))) diff --git a/src/app/generate_keypair/dune b/src/app/generate_keypair/dune index 1a987e63659..6ef13957692 100644 --- a/src/app/generate_keypair/dune +++ b/src/app/generate_keypair/dune @@ -4,6 +4,8 @@ (public_name generate_keypair) (modes native) (libraries cli_lib async core_kernel mina_version async_unix bounded_types) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/graphql_schema_dump/dune b/src/app/graphql_schema_dump/dune index 7116105997b..6294dcf3ae4 100644 --- a/src/app/graphql_schema_dump/dune +++ b/src/app/graphql_schema_dump/dune @@ -1,5 +1,16 @@ (executable (name graphql_schema_dump) - (libraries graphql_parser mina_version mina_graphql async async_unix graphql-async yojson bounded_types genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries + graphql_parser + mina_version + mina_graphql + async + async_unix + graphql-async + yojson + bounded_types + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/heap_usage/dune b/src/app/heap_usage/dune index a74b285b81f..e9b41204cb6 100644 --- a/src/app/heap_usage/dune +++ b/src/app/heap_usage/dune @@ -3,57 +3,58 @@ (name heap_usage) (public_name heap_usage) (libraries - ;; opam libraries - async_unix - core - result - async_kernel - stdio - async - core_kernel - base - base.caml - base.base_internalhash_types - async.async_command - yojson - sexplib0 - ;; local libraries - mina_block ; temp - mina_base - mina_base.import - mina_numbers - currency - bounded_types - data_hash_lib - signature_lib - merkle_ledger - mina_compile_config - mina_ledger - mina_state - mina_transaction_logic - genesis_constants - staged_ledger_diff - parallel_scan - pickles - pickles.backend - pickles_types - transaction_snark - transaction_snark_scan_state - ledger_proof - blake2 - mina_generators - with_hash - random_oracle - snark_params - crypto_params - kimchi_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - pasta_bindings - zkapp_command_builder - transaction_snark_tests - snark_profiler_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + async_unix + core + result + async_kernel + stdio + async + core_kernel + base + base.caml + base.base_internalhash_types + async.async_command + yojson + sexplib0 + ;; local libraries + mina_block ; temp + mina_base + mina_base.import + mina_numbers + currency + bounded_types + data_hash_lib + signature_lib + merkle_ledger + mina_compile_config + mina_ledger + mina_state + mina_transaction_logic + genesis_constants + staged_ledger_diff + parallel_scan + pickles + pickles.backend + pickles_types + transaction_snark + transaction_snark_scan_state + ledger_proof + blake2 + mina_generators + with_hash + random_oracle + snark_params + crypto_params + kimchi_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + pasta_bindings + zkapp_command_builder + transaction_snark_tests + snark_profiler_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv))) diff --git a/src/app/ledger_export_bench/dune b/src/app/ledger_export_bench/dune index 2352ffd4130..6ee4c2a05c7 100644 --- a/src/app/ledger_export_bench/dune +++ b/src/app/ledger_export_bench/dune @@ -1,15 +1,17 @@ (executable (name ledger_export_benchmark) (libraries - base - core - core_bench - core_kernel - yojson - ; mina libraries - mina_runtime_config) + base + core + core_bench + core_kernel + yojson + ; mina libraries + mina_runtime_config) ; the -w list here should be the same as in src/dune (flags -short-paths -g -w @a-4-29-40-41-42-44-45-48-58-59-60) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane)) (modes native)) diff --git a/src/app/logproc/dune b/src/app/logproc/dune index 4d38e6278e4..77ac3b6d14e 100644 --- a/src/app/logproc/dune +++ b/src/app/logproc/dune @@ -3,20 +3,21 @@ (public_name logproc) (modules logproc) (libraries - ;;opam libraries - result - stdio - core - yojson - cmdliner - core_kernel - ppx_deriving_yojson.runtime - ;;local libraries - logger - logproc_lib - bounded_types - bash_colors - interpolator_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_jane ppx_deriving.std))) + ;;opam libraries + result + stdio + core + yojson + cmdliner + core_kernel + ppx_deriving_yojson.runtime + ;;local libraries + logger + logproc_lib + bounded_types + bash_colors + interpolator_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_jane ppx_deriving.std))) diff --git a/src/app/logproc/logproc.ml b/src/app/logproc/logproc.ml index 8c519b962f1..44a1f7d3fa9 100644 --- a/src/app/logproc/logproc.ml +++ b/src/app/logproc/logproc.ml @@ -81,8 +81,9 @@ let process_line ~timezone ~interpolation_config ~filter line = type parser_state = | Start_of_line - | Processable_line of {line_start_pos: int; previous_elements: string list} - | Unprocessable_line of {line_start_pos: int} + | Processable_line of + { line_start_pos : int; previous_elements : string list } + | Unprocessable_line of { line_start_pos : int } (* Iterates over lines prefixed by `prefix`, invoking `on_hit` for each * line. If a line does not start with `prefix`, `on_miss` is invoked, @@ -105,11 +106,11 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> failwith "cannot extract start of line state" - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos i in let elements = List.rev (this_element :: previous_elements) in on_hit (String.concat elements ~sep:"") - | Unprocessable_line {line_start_pos} -> + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos i ; Out_channel.output_char output_channel '\n' in @@ -117,14 +118,15 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit match state with | Start_of_line -> Start_of_line - | Processable_line {line_start_pos; previous_elements} -> + | Processable_line { line_start_pos; previous_elements } -> let this_element = extract_input_string line_start_pos buf_len in Processable_line - { line_start_pos= 0 - ; previous_elements= this_element :: previous_elements } - | Unprocessable_line {line_start_pos} -> + { line_start_pos = 0 + ; previous_elements = this_element :: previous_elements + } + | Unprocessable_line { line_start_pos } -> forward_input line_start_pos buf_len ; - Unprocessable_line {line_start_pos= 0} + Unprocessable_line { line_start_pos = 0 } in let parse_char ~buf_len i state = if i >= buf_len then `Stop (carryover_state ~buf_len state) @@ -133,10 +135,10 @@ let iter_lines_prefixed_with input_channel output_channel ~prefix ~on_hit | Start_of_line -> let next_state = if Char.equal (Bytes.unsafe_get input_buffer i) prefix then - Processable_line {line_start_pos= i; previous_elements= []} + Processable_line { line_start_pos = i; previous_elements = [] } else ( on_miss () ; - Unprocessable_line {line_start_pos= i} ) + Unprocessable_line { line_start_pos = i } ) in `Continue next_state | Processable_line _ | Unprocessable_line _ -> @@ -209,9 +211,9 @@ let () = Arg.( value & opt - (enum [("hidden", Hidden); ("inline", Inline); ("after", After)]) + (enum [ ("hidden", Hidden); ("inline", Inline); ("after", After) ]) Inline - & info ["i"; "interpolation-mode"] ~docv:"MODE" ~doc) + & info [ "i"; "interpolation-mode" ] ~docv:"MODE" ~doc) in let max_interpolation_length = let doc = @@ -221,16 +223,17 @@ let () = Arg.( value & opt int 25 & info - ["m"; "max-interpolation-length"] + [ "m"; "max-interpolation-length" ] ~docv:"MAX_INTERPOLATION_LENGTH" ~doc) in let pretty_print = let doc = "Pretty print json values." in - Arg.(value & flag & info ["p"; "pretty-print"] ~docv:"PRETTY_PRINT" ~doc) + Arg.( + value & flag & info [ "p"; "pretty-print" ] ~docv:"PRETTY_PRINT" ~doc) in let lift_interpolation_config mode max_interpolation_length pretty_print = let open Interpolator in - {mode; max_interpolation_length; pretty_print} + { mode; max_interpolation_length; pretty_print } in Term.( const lift_interpolation_config @@ -240,23 +243,23 @@ let () = let doc = "Timezone to display timestamps in. Defaults to the system's timezone." in - Arg.(value & opt string "" & info ["z"; "zone"] ~docv:"TIMEZONE" ~doc) + Arg.(value & opt string "" & info [ "z"; "zone" ] ~docv:"TIMEZONE" ~doc) in let filter = let doc = "Filter displayed log lines. The filter language has similar syntax to \ - javascript, with a few notable differences. Similar to \"jq\", doing \ - an anymous access like \"[\"a\"]\" or \".a\" will refer to that key on \ - the javascript object being processed, which in the context of \ - logproc, is the json log entry itself. Basic literals (such as strings \ - and integers) are supported, as is structural equality (\"==\") and \ - boolean operations (\"!\", \"&&\", \"||\"). There is also support for \ - the \"in\" operator for checking existence in arrays (works like \ + javascript, with a few notable differences. Similar to \"jq\", doing an \ + anymous access like \"[\"a\"]\" or \".a\" will refer to that key on the \ + javascript object being processed, which in the context of logproc, is \ + the json log entry itself. Basic literals (such as strings and \ + integers) are supported, as is structural equality (\"==\") and boolean \ + operations (\"!\", \"&&\", \"||\"). There is also support for the \ + \"in\" operator for checking existence in arrays (works like \ javascript, not like jq). Regexes can also be expressed \ (\"/some_regex/\") and can be matched on using a special \"match\" \ operator. See examples for more information." in - Arg.(value & opt string "" & info ["f"; "filter"] ~docv:"FILTER" ~doc) + Arg.(value & opt string "" & info [ "f"; "filter" ] ~docv:"FILTER" ~doc) in let main_term = Term.(const main $ timezone $ interpolation_config $ filter) @@ -289,7 +292,8 @@ let () = ; `I ( "a complex filter:" , {|logproc -f '.message match /broadcast/ && .metadata.peer.host == "182.9.63.3" || .metadata.peer.discover_port == 8302|} - ) ] + ) + ] in Term.info ~version:"0.1" ~doc ~exits:Term.default_exits ~man "logproc" in diff --git a/src/app/missing_blocks_auditor/dune b/src/app/missing_blocks_auditor/dune index 49e9020bc1c..a08801fafa2 100644 --- a/src/app/missing_blocks_auditor/dune +++ b/src/app/missing_blocks_auditor/dune @@ -3,21 +3,22 @@ (name missing_blocks_auditor) (public_name missing_blocks_auditor) (libraries - ;; opam libraries - core - async_kernel - uri - caqti-driver-postgresql - caqti - async - core_kernel - caqti-async - base - async_unix - async.async_command - ;; local libraries - logger - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let))) + ;; opam libraries + core + async_kernel + uri + caqti-driver-postgresql + caqti + async + core_kernel + caqti-async + base + async_unix + async.async_command + ;; local libraries + logger + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let))) diff --git a/src/app/print_blockchain_snark_vk/dune b/src/app/print_blockchain_snark_vk/dune index 28872e1df1e..ad5287fc4fd 100644 --- a/src/app/print_blockchain_snark_vk/dune +++ b/src/app/print_blockchain_snark_vk/dune @@ -1,19 +1,23 @@ (executable (name print_blockchain_snark_vk) - (libraries - blockchain_snark genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version))) + (libraries blockchain_snark genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (rule (deps print_blockchain_snark_vk.exe) (targets blockchain_snark_vk.json.computed) (action - (with-stdout-to %{targets} + (with-stdout-to + %{targets} (run %{deps})))) (rule - (deps (:orig blockchain_snark_vk.json) (:computed blockchain_snark_vk.json.computed)) + (deps + (:orig blockchain_snark_vk.json) + (:computed blockchain_snark_vk.json.computed)) (alias runtest) (action (diff %{orig} %{computed}))) diff --git a/src/app/reformat/dune b/src/app/reformat/dune index e069e49a5da..c0fbc0fa245 100644 --- a/src/app/reformat/dune +++ b/src/app/reformat/dune @@ -1,7 +1,14 @@ (executable (name reformat) (public_name reformat) - (libraries core async async_kernel async_unix core_kernel sexplib0 async.async_command) + (libraries + core + async + async_kernel + async_unix + core_kernel + sexplib0 + async.async_command) (preprocess (pps ppx_jane)) ; the -w list here should be the same as in src/dune diff --git a/src/app/reformat/reformat.ml b/src/app/reformat/reformat.ml index 0ea294650e1..e71dad30c28 100644 --- a/src/app/reformat/reformat.ml +++ b/src/app/reformat/reformat.ml @@ -19,7 +19,8 @@ let dirs_trustlist = ; "snarky" ; "_opam" ; "opam_switches" - ; ".direnv" ] + ; ".direnv" + ] let rec fold_over_files ~path ~process_path ~init ~f = let%bind all = Sys.ls_dir path in @@ -42,11 +43,11 @@ let main dry_run check path = | `Dir -> not (List.exists dirs_trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path )) + String.is_suffix ~suffix:s path ) ) | `File -> (not (List.exists trustlist ~f:(fun s -> - String.is_suffix ~suffix:s path ))) + String.is_suffix ~suffix:s path ) ) ) && ( String.is_suffix ~suffix:".ml" path || String.is_suffix ~suffix:".mli" path ) ) ~f:(fun () file -> @@ -55,7 +56,7 @@ let main dry_run check path = return () in if check then - let prog, args = ("ocamlformat", ["--doc-comments=before"; file]) in + let prog, args = ("ocamlformat", [ "--doc-comments=before"; file ]) in let%bind formatted = Process.run_exn ~prog ~args () in let%bind raw = Reader.file_contents file in if not (String.equal formatted raw) then ( @@ -64,7 +65,7 @@ let main dry_run check path = else return () else let prog, args = - ("ocamlformat", ["--doc-comments=before"; "-i"; file]) + ("ocamlformat", [ "--doc-comments=before"; "-i"; file ]) in if dry_run then dump prog args else @@ -77,16 +78,16 @@ let _cli = let open Command.Let_syntax in Command.async ~summary:"Format all ml and mli files" (let%map_open path = - flag "--path" ~aliases:["path"] ~doc:"Path to traverse" + flag "--path" ~aliases:[ "path" ] ~doc:"Path to traverse" (required string) - and dry_run = flag "--dry-run" ~aliases:["dry-run"] no_arg ~doc:"Dry run" + and dry_run = flag "--dry-run" ~aliases:[ "dry-run" ] no_arg ~doc:"Dry run" and check = - flag "--check" ~aliases:["check"] no_arg + flag "--check" ~aliases:[ "check" ] no_arg ~doc: "Return with an error code if there exists an ml file that was \ formatted improperly" in - fun () -> main dry_run check path) + fun () -> main dry_run check path ) |> Command.run let () = never_returns (Scheduler.go ()) diff --git a/src/app/replayer/dune b/src/app/replayer/dune index 394d304b180..8cd2e071d22 100644 --- a/src/app/replayer/dune +++ b/src/app/replayer/dune @@ -3,66 +3,74 @@ (name replayer) (public_name replayer) (libraries - ;; opam libraries - stdio - sexplib0 - result - async.async_command - async - base.caml - bin_prot.shape - caqti-async - core_kernel - archive_lib - caqti-driver-postgresql - async_kernel - core - yojson - caqti - base - base.base_internalhash_types - uri - integers - async_unix - ;; local libraries - mina_wire_types - with_hash - snark_params - crypto_params - pickles_types - mina_numbers - coda_genesis_ledger - currency - pickles.backend - signature_lib - genesis_ledger_helper.lib - genesis_ledger_helper - mina_state - mina_ledger - mina_base - bounded_types - mina_base.import - mina_transaction - mina_transaction_logic - mina_caqti - genesis_constants - mina_runtime_config - logger - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - kimchi_backend_common - block_time - data_hash_lib - pickles - sgn - unsigned_extended - random_oracle - codable - ppx_version.runtime - cli_lib - mina_version - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;; opam libraries + stdio + sexplib0 + result + async.async_command + async + base.caml + bin_prot.shape + caqti-async + core_kernel + archive_lib + caqti-driver-postgresql + async_kernel + core + yojson + caqti + base + base.base_internalhash_types + uri + integers + async_unix + ;; local libraries + mina_wire_types + with_hash + snark_params + crypto_params + pickles_types + mina_numbers + coda_genesis_ledger + currency + pickles.backend + signature_lib + genesis_ledger_helper.lib + genesis_ledger_helper + mina_state + mina_ledger + mina_base + bounded_types + mina_base.import + mina_transaction + mina_transaction_logic + mina_caqti + genesis_constants + mina_runtime_config + logger + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + kimchi_backend_common + block_time + data_hash_lib + pickles + sgn + unsigned_extended + random_oracle + codable + ppx_version.runtime + cli_lib + mina_version) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/rosetta/README.md b/src/app/rosetta/README.md index b8fdf6fbc83..d99a25923e2 100644 --- a/src/app/rosetta/README.md +++ b/src/app/rosetta/README.md @@ -11,10 +11,9 @@ recommended 16GB) and then run the following: ```bash cat dockerfiles/Dockerfile-mina-rosetta \ | docker build -t mina-rosetta:v2.0.0 \ - --build-arg "network=berkeley" \ + --build-arg "network=devnet" \ --build-arg "image=debian:bullseye" \ --build-arg "deb_codename=bullseye" \ - --build-arg "deb_release=berkeley" \ --build-arg "deb_version=2.0.0berkeley-rc1-1551e2f" - ``` diff --git a/src/app/rosetta/dune b/src/app/rosetta/dune index 83be7c2bc27..d1120e13d02 100644 --- a/src/app/rosetta/dune +++ b/src/app/rosetta/dune @@ -4,10 +4,19 @@ (public_name rosetta) (modules rosetta) (modes native) - (libraries lib async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -15,10 +24,20 @@ (public_name rosetta-testnet) (modules rosetta_testnet_signatures) (modes native) - (libraries lib mina_signature_kind.testnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.testnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) (executable (package rosetta) @@ -26,7 +45,17 @@ (public_name rosetta-mainnet) (modules rosetta_mainnet_signatures) (modes native) - (libraries lib mina_signature_kind.mainnet async core_kernel base async.async_command core async_kernel genesis_constants) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version)) -) + (libraries + lib + mina_signature_kind.mainnet + async + core_kernel + base + async.async_command + core + async_kernel + genesis_constants) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version))) diff --git a/src/app/rosetta/lib/dune b/src/app/rosetta/lib/dune index 9c6bf25cd02..7515da8d47c 100644 --- a/src/app/rosetta/lib/dune +++ b/src/app/rosetta/lib/dune @@ -51,8 +51,7 @@ hex mina_numbers mina_transaction - mina_version - ) + mina_version) (preprocessor_deps ../../../graphql-ppx-config.inc ../../../../graphql_schema.json) diff --git a/src/app/rosetta/ocaml-signer/dune b/src/app/rosetta/ocaml-signer/dune index f72365e98d9..f4ac9458523 100644 --- a/src/app/rosetta/ocaml-signer/dune +++ b/src/app/rosetta/ocaml-signer/dune @@ -3,40 +3,39 @@ (public_name signer.cli) (modules signer_cli) (libraries - ;;opam libraries - async_kernel - core - core_kernel - async - base - lib - async_unix - async.async_command - ;;local libraries - kimchi_pasta - kimchi_pasta.basic - pickles - rosetta_coding - rosetta_lib - string_sign - signature_lib - mina_base - snark_params - pickles.backend - secrets - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + ;;opam libraries + async_kernel + core + core_kernel + async + base + lib + async_unix + async.async_command + ;;local libraries + kimchi_pasta + kimchi_pasta.basic + pickles + rosetta_coding + rosetta_lib + string_sign + signature_lib + mina_base + snark_params + pickles.backend + secrets) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -45,18 +44,18 @@ (modes native) (modules signer) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -65,18 +64,18 @@ (modes native) (modules signer_testnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) (executable (package signer) @@ -85,15 +84,15 @@ (modes native) (modules signer_mainnet_signatures) (libraries signer_cli async core_kernel base) - (instrumentation (backend bisect_ppx)) - (preprocess (pps - graphql_ppx - ppx_mina - ppx_deriving.show - ppx_deriving_yojson - ppx_base - ppx_let - ppx_version - ppx_compare - )) -) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + graphql_ppx + ppx_mina + ppx_deriving.show + ppx_deriving_yojson + ppx_base + ppx_let + ppx_version + ppx_compare))) diff --git a/src/app/rosetta/scripts/berkeley.env b/src/app/rosetta/scripts/berkeley.env deleted file mode 100644 index d64c3057ffe..00000000000 --- a/src/app/rosetta/scripts/berkeley.env +++ /dev/null @@ -1,5 +0,0 @@ -MINA_NETWORK=berkeley -PEER_LIST_URL=https://bootnodes.minaprotocol.com/networks/berkeley.txt -MINA_ARCHIVE_DUMP_URL=https://storage.googleapis.com/mina-archive-dumps -MINA_GENESIS_LEDGER_URL=https://673156464838-mina-genesis-ledgers.s3.us-west-2.amazonaws.com/berkeley/genesis_ledger.json -BLOCKS_BUCKET=https://storage.googleapis.com/mina_network_block_data diff --git a/src/app/runtime_genesis_ledger/dune b/src/app/runtime_genesis_ledger/dune index 0a1191a086d..a6dba29e46b 100644 --- a/src/app/runtime_genesis_ledger/dune +++ b/src/app/runtime_genesis_ledger/dune @@ -2,28 +2,29 @@ (name runtime_genesis_ledger) (public_name runtime_genesis_ledger) (libraries - ;;opam libraries - base - core_kernel - async_kernel - async - core - result - async.async_command - yojson - async_unix - ;;local libraries - genesis_constants - mina_runtime_config - mina_ledger - genesis_ledger_helper - bounded_types - logger - cache_dir - precomputed_values - coda_genesis_ledger - consensus - mina_base - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) + ;;opam libraries + base + core_kernel + async_kernel + async + core + result + async.async_command + yojson + async_unix + ;;local libraries + genesis_constants + mina_runtime_config + mina_ledger + genesis_ledger_helper + bounded_types + logger + cache_dir + precomputed_values + coda_genesis_ledger + consensus + mina_base) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_mina ppx_version ppx_let ppx_deriving_yojson))) diff --git a/src/app/snark_work_debugger/dune b/src/app/snark_work_debugger/dune index 0f8b91aaea9..9eda39a4279 100644 --- a/src/app/snark_work_debugger/dune +++ b/src/app/snark_work_debugger/dune @@ -4,33 +4,40 @@ (public_name snark_work_debugger) (modes native) (libraries - ;; opam libraries - base - async - async_kernel - async_unix - unix - core - core_kernel - sexplib0 - ;; local libraries - mina_base - snark_worker - bounded_types - snark_work_lib - transaction_witness - transaction_snark - genesis_constants - logger - error_json - transaction_protocol_state - mina_state - currency - mina_ledger - signature_lib - mina_base.import - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_here ppx_custom_printf ppx_version ppx_mina ppx_let ppx_deriving_yojson ppx_sexp_conv)) + ;; opam libraries + base + async + async_kernel + async_unix + unix + core + core_kernel + sexplib0 + ;; local libraries + mina_base + snark_worker + bounded_types + snark_work_lib + transaction_witness + transaction_snark + genesis_constants + logger + error_json + transaction_protocol_state + mina_state + currency + mina_ledger + signature_lib + mina_base.import) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_here + ppx_custom_printf + ppx_version + ppx_mina + ppx_let + ppx_deriving_yojson + ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) - diff --git a/src/app/validate_keypair/dune b/src/app/validate_keypair/dune index 78e187b0fa9..4346b5856f5 100644 --- a/src/app/validate_keypair/dune +++ b/src/app/validate_keypair/dune @@ -4,15 +4,16 @@ (public_name validate_keypair) (modes native) (libraries - ;; opam libraries - core_kernel - async - async_unix - ;; local libraries - mina_version - bounded_types - cli_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_let ppx_sexp_conv)) + ;; opam libraries + core_kernel + async + async_unix + ;; local libraries + mina_version + bounded_types + cli_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps ppx_version ppx_let ppx_sexp_conv)) (flags -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60)) diff --git a/src/app/zkapp_limits/dune b/src/app/zkapp_limits/dune index 6cb781b71b2..2c50a3c87ac 100644 --- a/src/app/zkapp_limits/dune +++ b/src/app/zkapp_limits/dune @@ -3,14 +3,22 @@ (name zkapp_limits) (public_name zkapp_limits) (libraries - ;; opam libraries - base - base.caml - core_kernel - ;; local libraries - mina_base - genesis_constants - bounded_types - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_version ppx_mina ppx_custom_printf ppx_let ppx_hash ppx_compare ppx_sexp_conv))) + ;; opam libraries + base + base.caml + core_kernel + ;; local libraries + mina_base + genesis_constants + bounded_types) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_version + ppx_mina + ppx_custom_printf + ppx_let + ppx_hash + ppx_compare + ppx_sexp_conv))) diff --git a/src/app/zkapp_test_transaction/dune b/src/app/zkapp_test_transaction/dune index ca1edcbdd3c..c81e6d48a82 100644 --- a/src/app/zkapp_test_transaction/dune +++ b/src/app/zkapp_test_transaction/dune @@ -3,24 +3,34 @@ (name zkapp_test_transaction) (public_name zkapp_test_transaction) (libraries - ;;opam libraries - async.async_command - base - core_kernel - core - async_kernel - zkapp_test_transaction_lib - async - async_unix - ;;local libraries - mina_graphql - mina_numbers - currency - bounded_types - cli_lib - mina_base - signature_lib - ) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_assert ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx))) + ;;opam libraries + async.async_command + base + core_kernel + core + async_kernel + zkapp_test_transaction_lib + async + async_unix + ;;local libraries + mina_graphql + mina_numbers + currency + bounded_types + cli_lib + mina_base + signature_lib) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_assert + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx))) diff --git a/src/app/zkapp_test_transaction/lib/dune b/src/app/zkapp_test_transaction/lib/dune index 3b0f5a18ded..81420ca2622 100644 --- a/src/app/zkapp_test_transaction/lib/dune +++ b/src/app/zkapp_test_transaction/lib/dune @@ -2,58 +2,69 @@ (name zkapp_test_transaction_lib) (public_name zkapp_test_transaction_lib) (libraries - ;;opam libraries - async_unix - graphql_parser - splittable_random - result - async - async_kernel - core - core_kernel - base - sexplib0 - base_quickcheck - ppx_inline_test.config - graphql-async - ;;local libraries - pickles - kimchi_pasta - kimchi_pasta.basic - bounded_types - random_oracle - mina_runtime_config - transaction_witness - transaction_protocol_state - coda_genesis_ledger - pickles_types - mina_numbers - random_oracle_input - genesis_ledger_helper - mina_state - mina_generators - mina_base - currency - mina_ledger - mina_graphql - genesis_constants - secrets - transaction_snark - signature_lib - data_hash_lib - consensus - with_hash - genesis_ledger_helper.lib - snark_params - pickles.backend - kimchi_backend - mina_base.import - mina_transaction_logic - mina_transaction - staged_ledger_diff - mina_wire_types - ) - (inline_tests (flags -verbose -show-counts)) - (instrumentation (backend bisect_ppx)) - (preprocess (pps ppx_custom_printf ppx_mina ppx_version ppx_let ppx_sexp_conv - ppx_hash ppx_compare ppx_deriving_yojson h_list.ppx ppx_inline_test))) + ;;opam libraries + async_unix + graphql_parser + splittable_random + result + async + async_kernel + core + core_kernel + base + sexplib0 + base_quickcheck + ppx_inline_test.config + graphql-async + ;;local libraries + pickles + kimchi_pasta + kimchi_pasta.basic + bounded_types + random_oracle + mina_runtime_config + transaction_witness + transaction_protocol_state + coda_genesis_ledger + pickles_types + mina_numbers + random_oracle_input + genesis_ledger_helper + mina_state + mina_generators + mina_base + currency + mina_ledger + mina_graphql + genesis_constants + secrets + transaction_snark + signature_lib + data_hash_lib + consensus + with_hash + genesis_ledger_helper.lib + snark_params + pickles.backend + kimchi_backend + mina_base.import + mina_transaction_logic + mina_transaction + staged_ledger_diff + mina_wire_types) + (inline_tests + (flags -verbose -show-counts)) + (instrumentation + (backend bisect_ppx)) + (preprocess + (pps + ppx_custom_printf + ppx_mina + ppx_version + ppx_let + ppx_sexp_conv + ppx_hash + ppx_compare + ppx_deriving_yojson + h_list.ppx + ppx_inline_test))) diff --git a/src/app/zkapps_examples/add_events/dune b/src/app/zkapps_examples/add_events/dune index 618100a98a1..d8b70325b92 100644 --- a/src/app/zkapps_examples/add_events/dune +++ b/src/app/zkapps_examples/add_events/dune @@ -1,19 +1,20 @@ (library (name zkapps_add_events) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - mina_base - pickles - pickles_types - snarky.backendless - snark_params - signature_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + mina_base + pickles + pickles_types + snarky.backendless + snark_params + signature_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess (pps ppx_version))) diff --git a/src/app/zkapps_examples/calls/dune b/src/app/zkapps_examples/calls/dune index eaaa1d87239..0bf875a0810 100644 --- a/src/app/zkapps_examples/calls/dune +++ b/src/app/zkapps_examples/calls/dune @@ -1,33 +1,32 @@ (library (name zkapps_calls) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - with_hash - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + with_hash + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx))) + (pps ppx_version h_list.ppx))) diff --git a/src/app/zkapps_examples/dune b/src/app/zkapps_examples/dune index 1db21807142..393112daabf 100644 --- a/src/app/zkapps_examples/dune +++ b/src/app/zkapps_examples/dune @@ -2,36 +2,35 @@ (name zkapps_examples) (public_name zkapps_examples) (libraries - ;; opam libraries - base - async_kernel - core_kernel - ;; local libraries - mina_numbers - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - pickles - pickles.backend - pickles_types - random_oracle - random_oracle_input - snarky.backendless - snark_params - bounded_types - sgn - signature_lib - tuple_lib - with_hash) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + async_kernel + core_kernel + ;; local libraries + mina_numbers + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + pickles + pickles.backend + pickles_types + random_oracle + random_oracle_input + snarky.backendless + snark_params + bounded_types + sgn + signature_lib + tuple_lib + with_hash) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_let - ppx_version))) + (pps ppx_let ppx_version))) diff --git a/src/app/zkapps_examples/empty_update/dune b/src/app/zkapps_examples/empty_update/dune index 10790066528..11e0c98a278 100644 --- a/src/app/zkapps_examples/empty_update/dune +++ b/src/app/zkapps_examples/empty_update/dune @@ -1,28 +1,28 @@ (library (name zkapps_empty_update) (libraries - ;; opam libraries - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/initialize_state/dune b/src/app/zkapps_examples/initialize_state/dune index 5dc9bcf172f..63590bd026c 100644 --- a/src/app/zkapps_examples/initialize_state/dune +++ b/src/app/zkapps_examples/initialize_state/dune @@ -1,29 +1,29 @@ (library (name zkapps_initialize_state) (libraries - ;; opam libraries - base - core_kernel - ;; local libraries - crypto_params - currency - pasta_bindings - kimchi_backend - kimchi_backend_common - kimchi_pasta - kimchi_pasta.basic - mina_base - pickles - pickles.backend - pickles_types - random_oracle - snarky.backendless - snark_params - sgn - signature_lib - tuple_lib - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + base + core_kernel + ;; local libraries + crypto_params + currency + pasta_bindings + kimchi_backend + kimchi_backend_common + kimchi_pasta + kimchi_pasta.basic + mina_base + pickles + pickles.backend + pickles_types + random_oracle + snarky.backendless + snark_params + sgn + signature_lib + tuple_lib + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version))) + (pps ppx_version))) diff --git a/src/app/zkapps_examples/test/actions/dune b/src/app/zkapps_examples/test/actions/dune index baa85f43623..3c02027061c 100644 --- a/src/app/zkapps_examples/test/actions/dune +++ b/src/app/zkapps_examples/test/actions/dune @@ -40,7 +40,8 @@ zkapps_examples yojson mina_wire_types) - (inline_tests (flags -verbose -show-counts)) + (inline_tests + (flags -verbose -show-counts)) (preprocess (pps ppx_version ppx_jane)) (instrumentation diff --git a/src/app/zkapps_examples/test/add_events/dune b/src/app/zkapps_examples/test/add_events/dune index 583a5c247c7..e86b6cad48e 100644 --- a/src/app/zkapps_examples/test/add_events/dune +++ b/src/app/zkapps_examples/test/add_events/dune @@ -1,41 +1,42 @@ (library (name add_events) (libraries - ;; opam libraries - async - async_kernel - async_unix - core - core_kernel - ppx_inline_test.config - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - merkle_list_verifier - mina_base - mina_base.import - mina_ledger - mina_numbers - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - signature_lib - snark_params - transaction_snark - transaction_snark_tests - with_hash - zkapps_add_events - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + core + core_kernel + ppx_inline_test.config + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + merkle_list_verifier + mina_base + mina_base.import + mina_ledger + mina_numbers + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + signature_lib + snark_params + transaction_snark + transaction_snark_tests + with_hash + zkapps_add_events + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/calls/dune b/src/app/zkapps_examples/test/calls/dune index 25706b15533..d1ed0f79578 100644 --- a/src/app/zkapps_examples/test/calls/dune +++ b/src/app/zkapps_examples/test/calls/dune @@ -1,48 +1,49 @@ (library (name calls) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_calls - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_calls + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/empty_update/dune b/src/app/zkapps_examples/test/empty_update/dune index 534cf6ac7b1..c36a8be7eff 100644 --- a/src/app/zkapps_examples/test/empty_update/dune +++ b/src/app/zkapps_examples/test/empty_update/dune @@ -1,51 +1,51 @@ (executable (name empty_update) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_base - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_empty_update - zkapps_examples - bounded_types - ) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_base + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_empty_update + zkapps_examples + bounded_types) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) (rule (alias runtest) diff --git a/src/app/zkapps_examples/test/initialize_state/dune b/src/app/zkapps_examples/test/initialize_state/dune index eac8a441070..e5693dee970 100644 --- a/src/app/zkapps_examples/test/initialize_state/dune +++ b/src/app/zkapps_examples/test/initialize_state/dune @@ -1,48 +1,49 @@ (library (name initialize_state) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_initialize_state - zkapps_examples - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_initialize_state + zkapps_examples) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/optional_custom_gates/dune b/src/app/zkapps_examples/test/optional_custom_gates/dune index ecf9f3905ac..573edf08c77 100644 --- a/src/app/zkapps_examples/test/optional_custom_gates/dune +++ b/src/app/zkapps_examples/test/optional_custom_gates/dune @@ -1,43 +1,44 @@ (library - (name zkapp_optional_custom_gates_tests) + (name zkapp_optional_custom_gates_tests) (libraries - ;; opam libraries - ppx_inline_test.config - async - async_kernel - async_unix - core - core_kernel - result - base - base.caml - sexplib0 - integers - ;; local libraries - mina_base.import - transaction_snark - mina_ledger - mina_base - mina_generators - currency - pickles - pickles_types - mina_numbers - zkapp_command_builder - signature_lib - genesis_constants - sgn - transaction_snark_tests - test_util - mina_transaction_logic - zkapps_examples - pickles_optional_custom_gates_circuits - snark_keys_header - snark_params - data_hash_lib - ) - (library_flags -linkall) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane ppx_mina)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + ppx_inline_test.config + async + async_kernel + async_unix + core + core_kernel + result + base + base.caml + sexplib0 + integers + ;; local libraries + mina_base.import + transaction_snark + mina_ledger + mina_base + mina_generators + currency + pickles + pickles_types + mina_numbers + zkapp_command_builder + signature_lib + genesis_constants + sgn + transaction_snark_tests + test_util + mina_transaction_logic + zkapps_examples + pickles_optional_custom_gates_circuits + snark_keys_header + snark_params + data_hash_lib) + (library_flags -linkall) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane ppx_mina)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/test/tokens/dune b/src/app/zkapps_examples/test/tokens/dune index 48ecefc7bbe..05877f35fdf 100644 --- a/src/app/zkapps_examples/test/tokens/dune +++ b/src/app/zkapps_examples/test/tokens/dune @@ -1,49 +1,50 @@ (library (name tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core - core_kernel - ppx_inline_test.config - yojson - ;; local libraries - cache_dir - currency - data_hash_lib - genesis_constants - pasta_bindings - kimchi_backend - kimchi_pasta - kimchi_pasta.basic - merkle_ledger - mina_base - mina_base.import - mina_ledger - mina_numbers - mina_state - mina_transaction_logic - pickles - pickles.backend - pickles_types - random_oracle_input - random_oracle - sgn - signature_lib - snark_params - snarky.backendless - transaction_protocol_state - transaction_snark - transaction_snark_tests - with_hash - zkapps_tokens - zkapps_examples - bounded_types - ) - (inline_tests (flags -verbose -show-counts)) - (preprocess - (pps ppx_snarky ppx_version ppx_jane)) - (instrumentation (backend bisect_ppx))) + ;; opam libraries + async + async_kernel + async_unix + base + core + core_kernel + ppx_inline_test.config + yojson + ;; local libraries + cache_dir + currency + data_hash_lib + genesis_constants + pasta_bindings + kimchi_backend + kimchi_pasta + kimchi_pasta.basic + merkle_ledger + mina_base + mina_base.import + mina_ledger + mina_numbers + mina_state + mina_transaction_logic + pickles + pickles.backend + pickles_types + random_oracle_input + random_oracle + sgn + signature_lib + snark_params + snarky.backendless + transaction_protocol_state + transaction_snark + transaction_snark_tests + with_hash + zkapps_tokens + zkapps_examples + bounded_types) + (inline_tests + (flags -verbose -show-counts)) + (preprocess + (pps ppx_snarky ppx_version ppx_jane)) + (instrumentation + (backend bisect_ppx))) diff --git a/src/app/zkapps_examples/tokens/dune b/src/app/zkapps_examples/tokens/dune index 1e82e848990..d9f359830bd 100644 --- a/src/app/zkapps_examples/tokens/dune +++ b/src/app/zkapps_examples/tokens/dune @@ -1,40 +1,38 @@ (library (name zkapps_tokens) (libraries - ;; opam libraries - async - async_kernel - async_unix - base - core_kernel - ;; local libraries - cache_dir - crypto_params - currency - genesis_constants - kimchi_backend - kimchi_backend_common - kimchi_pasta - mina_base - mina_base.import - pasta_bindings - pickles - pickles.backend - pickles_base - pickles_types - random_oracle - random_oracle_input - sgn - signature_lib - snark_params - snarky.backendless - tuple_lib - with_hash - yojson - zkapps_examples) - (instrumentation (backend bisect_ppx)) + ;; opam libraries + async + async_kernel + async_unix + base + core_kernel + ;; local libraries + cache_dir + crypto_params + currency + genesis_constants + kimchi_backend + kimchi_backend_common + kimchi_pasta + mina_base + mina_base.import + pasta_bindings + pickles + pickles.backend + pickles_base + pickles_types + random_oracle + random_oracle_input + sgn + signature_lib + snark_params + snarky.backendless + tuple_lib + with_hash + yojson + zkapps_examples) + (instrumentation + (backend bisect_ppx)) (preprocess - (pps - ppx_version - h_list.ppx - ppx_let))) + (pps ppx_version h_list.ppx ppx_let))) diff --git a/src/dune-project b/src/dune-project index 91a68888957..3c4d5463c4c 100644 --- a/src/dune-project +++ b/src/dune-project @@ -135,6 +135,7 @@ (package (name parallel)) (package (name parallel_scan)) (package (name participating_state)) +(package (name patch_archive_test)) (package (name pasta_bindings)) (package (name perf_histograms)) (package (name pickles_base)) diff --git a/src/lib/cache_dir/native/cache_dir.ml b/src/lib/cache_dir/native/cache_dir.ml index 8450d40bfd7..d76cd00aa38 100644 --- a/src/lib/cache_dir/native/cache_dir.ml +++ b/src/lib/cache_dir/native/cache_dir.ml @@ -55,7 +55,7 @@ let load_from_s3 s3_bucket_prefix s3_install_path ~logger = @@ Monitor.try_with ~here:[%here] (fun () -> let each_uri (uri_string, file_path) = let open Deferred.Let_syntax in - [%log trace] "Downloading file from S3" + [%log trace] "Downloading file from S3: $url to $local_file_path" ~metadata: [ ("url", `String uri_string) ; ("local_file_path", `String file_path) diff --git a/src/lib/cli_lib/dune b/src/lib/cli_lib/dune index dc0f9c72a9d..05b36e7b400 100644 --- a/src/lib/cli_lib/dune +++ b/src/lib/cli_lib/dune @@ -45,7 +45,7 @@ kimchi_pasta.basic ppx_version.runtime gossip_net - mina_runtime_config) + runtime_config) (preprocess (pps ppx_version ppx_jane ppx_deriving_yojson ppx_deriving.make)) (instrumentation (backend bisect_ppx)) diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index fa06f656fc3..18297518678 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -88,14 +88,14 @@ type t = let proof_config_default : Runtime_config.Proof_keys.t = { level = Some Full - ; sub_windows_per_window = None - ; ledger_depth = None - ; work_delay = None + ; sub_windows_per_window = Some 11 + ; ledger_depth = Some 35 + ; work_delay = Some 2 ; block_window_duration_ms = Some 120000 - ; transaction_capacity = None - ; coinbase_amount = None - ; supercharged_coinbase_factor = None - ; account_creation_fee = None + ; transaction_capacity = Some (Log_2 7) + ; coinbase_amount = Some (Currency.Amount.of_mina_int_exn 720) + ; supercharged_coinbase_factor = Some 1 + ; account_creation_fee = Some (Currency.Fee.of_mina_string_exn "1") ; fork = None } diff --git a/src/lib/integration_test_local_engine/docker_node_config.ml b/src/lib/integration_test_local_engine/docker_node_config.ml index 16bba170998..f207055960d 100644 --- a/src/lib/integration_test_local_engine/docker_node_config.ml +++ b/src/lib/integration_test_local_engine/docker_node_config.ml @@ -105,6 +105,7 @@ module Base_node_config = struct mina accounts import -config-directory /root/.mina-config -privkey-path "$key_file" fi done + rm /var/lib/coda/config* # Execute the puppeteer script exec /mina_daemon_puppeteer.py "$@" |} diff --git a/src/lib/integration_test_local_engine/mina_docker.ml b/src/lib/integration_test_local_engine/mina_docker.ml index 2a29814bb5c..6784194c4ab 100644 --- a/src/lib/integration_test_local_engine/mina_docker.ml +++ b/src/lib/integration_test_local_engine/mina_docker.ml @@ -158,8 +158,17 @@ module Network_config = struct Some { Runtime_config.Daemon.default with txpool_max_size = Some txpool_max_size + ; peer_list_url = None + ; zkapp_proof_update_cost = Some 10.26 + ; zkapp_signed_single_update_cost = Some 9.140000000000001 + ; zkapp_signed_pair_update_cost = Some 10.08 + ; zkapp_transaction_cost_limit = Some 69.45 + ; max_event_elements = Some 100 + ; max_action_elements = Some 100 + ; zkapp_cmd_limit_hardcap = Some 128 ; slot_tx_end ; slot_chain_end + ; minimum_user_command_fee = Some (Currency.Fee.of_string "1000000") ; network_id } ; genesis = diff --git a/src/lib/mina_compile_config/dune b/src/lib/mina_compile_config/dune index fac058923b4..181360fbaf2 100644 --- a/src/lib/mina_compile_config/dune +++ b/src/lib/mina_compile_config/dune @@ -1,6 +1,6 @@ (library (name mina_compile_config) (public_name mina_compile_config) - (libraries mina_node_config mina_node_config.for_unit_tests core_kernel currency) + (libraries mina_node_config node_config_for_unit_tests core_kernel currency) (instrumentation (backend bisect_ppx)) (preprocess (pps ppx_version ppx_base ppx_deriving_yojson))) diff --git a/src/test/archive/patch_archive_test/dune b/src/test/archive/patch_archive_test/dune new file mode 100644 index 00000000000..109d27b4a77 --- /dev/null +++ b/src/test/archive/patch_archive_test/dune @@ -0,0 +1,31 @@ +(executable + (package patch_archive_test) + (name patch_archive_test) + (public_name patch_archive_test) + (libraries + async + async.async_command + core_kernel + caqti + caqti-async + caqti-driver-postgresql + integration_test_lib + archive_lib + block_time + mina_numbers + logger + mina_base + uri + base + async_kernel + core + async_unix + stdio + base.caml + result + mina_automation + bounded_types + ) + (preprocessor_deps ../../../config.mlh) + (instrumentation (backend bisect_ppx)) + (preprocess (pps ppx_version ppx_mina ppx_let ppx_hash ppx_compare ppx_sexp_conv h_list.ppx))) diff --git a/src/test/archive/patch_archive_test/patch_archive_test.ml b/src/test/archive/patch_archive_test/patch_archive_test.ml new file mode 100644 index 00000000000..8d0b2f91a67 --- /dev/null +++ b/src/test/archive/patch_archive_test/patch_archive_test.ml @@ -0,0 +1,160 @@ +(* patch_archive_test.ml *) + +(* test patching of archive databases + + test structure: + - import reference database for comparision (for example with 100 blocks) + - create new schema and export blocks from reference db with some missing ones + - patch the database with missing precomputed blocks + - compare original and copy +*) + +module Network_Data = struct + type t = + { init_script : String.t + ; precomputed_blocks_zip : String.t + ; genesis_ledger_file : String.t + ; replayer_input_file : String.t + ; folder : String.t + } + + let create folder = + { init_script = "archive_db.sql" + ; genesis_ledger_file = "input.json" + ; precomputed_blocks_zip = "precomputed_blocks.zip" + ; replayer_input_file = "replayer_input_file.json" + ; folder + } +end + +open Core_kernel +open Async +open Mina_automation + +let main ~db_uri ~network_data_folder () = + let open Deferred.Let_syntax in + let missing_blocks_count = 3 in + let network_name = "dummy" in + + let network_data = Network_Data.create network_data_folder in + + let output_folder = Filename.temp_dir_name ^ "/output" in + + let%bind output_folder = Unix.mkdtemp output_folder in + + let connection = Psql.Conn_str db_uri in + + let source_db_name = "patch_archive_test_source" in + let target_db_name = "patch_archive_test_target" in + let%bind _ = Psql.create_empty_db ~connection ~db:source_db_name in + let%bind _ = + Psql.run_script ~connection ~db:source_db_name + (network_data.folder ^ "/" ^ network_data.init_script) + in + let%bind () = Psql.create_mina_db ~connection ~db:target_db_name in + + let source_db = db_uri ^ "/" ^ source_db_name in + let target_db = db_uri ^ "/" ^ target_db_name in + + let extract_blocks = Extract_blocks.of_context Executor.AutoDetect in + let config = + { Extract_blocks.Config.archive_uri = source_db + ; range = Extract_blocks.Config.AllBlocks + ; output_folder = Some output_folder + ; network = Some network_name + ; include_block_height_in_name = true + } + in + let%bind _ = Extract_blocks.run extract_blocks ~config in + + let archive_blocks = Archive_blocks.of_context Executor.AutoDetect in + + let%bind extensional_files = + Sys.ls_dir output_folder + >>= Deferred.List.map ~f:(fun e -> + Deferred.return (output_folder ^ "/" ^ e) ) + in + + let n = + List.init missing_blocks_count ~f:(fun _ -> + Random.int (List.length extensional_files) ) + in + + let unpatched_extensional_files = + List.filteri extensional_files ~f:(fun i _ -> + not (List.mem n i ~equal:Int.equal) ) + |> List.dedup_and_sort ~compare:(fun left right -> + let scan_height item = + let item = + Filename.basename item |> Str.global_replace (Str.regexp "-") " " + in + Scanf.sscanf item "%s %d %s" (fun _ height _ -> height) + in + + let left_height = scan_height left in + let right_height = scan_height right in + + Int.compare left_height right_height ) + in + + let%bind _ = + Archive_blocks.run archive_blocks ~blocks:unpatched_extensional_files + ~archive_uri:target_db ~format:Extensional + in + + let%bind missing_blocks_auditor_path = + Missing_blocks_auditor.of_context Executor.AutoDetect + |> Missing_blocks_auditor.path + in + + let%bind archive_blocks_path = Archive_blocks.path archive_blocks in + + let config = + { Missing_blocks_guardian.Config.archive_uri = Uri.of_string target_db + ; precomputed_blocks = Uri.make ~scheme:"file" ~path:output_folder () + ; network = network_name + ; run_mode = Run + ; missing_blocks_auditor = missing_blocks_auditor_path + ; archive_blocks = archive_blocks_path + ; block_format = Extensional + } + in + + let missing_blocks_guardian = + Missing_blocks_guardian.of_context Executor.AutoDetect + in + + let%bind _ = Missing_blocks_guardian.run missing_blocks_guardian ~config in + + let replayer = Replayer.of_context Executor.AutoDetect in + + let%bind _ = + Replayer.run replayer ~archive_uri:target_db + ~input_config: + (network_data.folder ^ "/" ^ network_data.replayer_input_file) + ~interval_checkpoint:10 ~output_ledger:"./output_ledger" () + in + + Deferred.unit + +let () = + Command.( + run + (let open Let_syntax in + async ~summary:"Test patching of blocks in an archive database" + (let%map db_uri = + Param.flag "--source-uri" + ~doc: + "URI URI for connecting to the database (e.g., \ + postgres://$USER@localhost:5432)" + Param.(required string) + and network_data_folder = + Param.( + flag "--network-data-folder" ~aliases:[ "network-data-folder" ] + Param.(required string)) + ~doc: + "Path Path to folder containing network data. Usually it's sql \ + for db import, genesis ledger and zipped precomputed blocks \ + archive" + in + main ~db_uri ~network_data_folder ))) diff --git a/src/test/mina_automation/archive_blocks.ml b/src/test/mina_automation/archive_blocks.ml index 2c0dc47c89d..31e99b28a0e 100644 --- a/src/test/mina_automation/archive_blocks.ml +++ b/src/test/mina_automation/archive_blocks.ml @@ -9,7 +9,7 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/archive_blocks/archive_blocks.exe" - ~official_name:"/usr/local/bin/mina-archive-blocks" + ~official_name:"mina-archive-blocks" type format = Precomputed | Extensional diff --git a/src/test/mina_automation/executor.ml b/src/test/mina_automation/executor.ml index 2d52f940955..6fba161bf17 100644 --- a/src/test/mina_automation/executor.ml +++ b/src/test/mina_automation/executor.ml @@ -38,13 +38,23 @@ module Executor = struct let built_name t = Printf.sprintf "_build/default/%s" t.dune_name + let paths = + Option.value_map ~f:(String.split ~on:':') ~default:[] (Sys.getenv "PATH") + + let exists_at_path t prefix = + match%bind Sys.file_exists (prefix ^ "/" ^ t.official_name) with + | `Yes -> + Deferred.return (Some prefix) + | _ -> + Deferred.return None + let path t = match%bind Sys.file_exists (built_name t) with | `Yes -> Deferred.return (built_name t) | _ -> ( - match%bind Sys.file_exists t.official_name with - | `Yes -> + match%bind Deferred.List.find_map ~f:(exists_at_path t) paths with + | Some _ -> Deferred.return t.official_name | _ -> Deferred.return t.dune_name ) @@ -60,18 +70,7 @@ module Executor = struct ~metadata:[ ("app", `String (built_name t)) ] ; run_from_local t ~args ?env () | _ -> ( - let paths = - Option.value_map ~f:(String.split ~on:':') ~default:[] - (Sys.getenv "PATH") - in - let exists_at_path prefix = - match%bind Sys.file_exists (prefix ^ "/" ^ t.official_name) with - | `Yes -> - Deferred.return (Some prefix) - | _ -> - Deferred.return None - in - match%bind Deferred.List.find_map ~f:exists_at_path paths with + match%bind Deferred.List.find_map ~f:(exists_at_path t) paths with | Some prefix -> [%log debug] "running from %s" prefix ~metadata:[ ("app", `String t.official_name) ] ; diff --git a/src/test/mina_automation/extract_blocks.ml b/src/test/mina_automation/extract_blocks.ml index 7b05e9b00cb..bfb3ec9e2ae 100644 --- a/src/test/mina_automation/extract_blocks.ml +++ b/src/test/mina_automation/extract_blocks.ml @@ -55,6 +55,6 @@ end let of_context context = Executor.of_context ~context ~dune_name:"src/app/extract_blocks/extract_blocks.exe" - ~official_name:"/usr/local/bin/mina-extract-blocks" + ~official_name:"mina-extract-blocks" let run t ~config = run t ~args:(Config.to_args config) () diff --git a/src/test/mina_automation/missing_blocks_auditor.ml b/src/test/mina_automation/missing_blocks_auditor.ml index f28983711bb..4c6116a4c24 100644 --- a/src/test/mina_automation/missing_blocks_auditor.ml +++ b/src/test/mina_automation/missing_blocks_auditor.ml @@ -8,4 +8,4 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/missing_blocks_auditor/missing_blocks_auditor.exe" - ~official_name:"/usr/local/bin/mina-missing-blocks-auditor" + ~official_name:"mina-missing-blocks-auditor" diff --git a/src/test/mina_automation/missing_blocks_guardian.ml b/src/test/mina_automation/missing_blocks_guardian.ml index 42bbf0ec844..b74ea406855 100644 --- a/src/test/mina_automation/missing_blocks_guardian.ml +++ b/src/test/mina_automation/missing_blocks_guardian.ml @@ -45,7 +45,7 @@ end let of_context context = Executor.of_context ~context ~dune_name:"scripts/archive/missing-blocks-guardian.sh" - ~official_name:"/etc/mina/archive/missing-blocks-guardian.sh" + ~official_name:"mina-missing-blocks-guardian" let run t ~config = run t ~args:(Config.to_args config) ~env:(Config.to_envs config) () diff --git a/src/test/mina_automation/replayer.ml b/src/test/mina_automation/replayer.ml index 049f8071d95..fbdfe133d1c 100644 --- a/src/test/mina_automation/replayer.ml +++ b/src/test/mina_automation/replayer.ml @@ -68,7 +68,7 @@ include Executor let of_context context = Executor.of_context ~context ~dune_name:"src/app/replayer/replayer.exe" - ~official_name:"/usr/local/bin/mina-replayer" + ~official_name:"mina-replayer" let run t ~archive_uri ~input_config ~interval_checkpoint ?checkpoint_output_folder ?checkpoint_file_prefix ~output_ledger =