Merge branch 'compatible' into refactor/mina-rpcs

.gitignore

@@ -80,6 +80,7 @@ automation/services/watchdog/check_libp2p/check_libp2p
 *.terraform.lock.hcl
 *gcloud-keyfile.json*
 _opam
+opam_switches
 
 target
 ./release

Makefile

@@ -12,6 +12,14 @@ ifeq ($(DUNE_PROFILE),)
 DUNE_PROFILE := dev
 endif
 
+ifeq ($(OPAMSWITCH)$(IN_NIX_SHELL)$(CI)$(BUILDKITE),)
+# Sometimes opam replaces these env variables in shell with
+# an explicit mention of a particular switch (dereferenced from the value)
+OPAM_SWITCH_PREFIX := $(PWD)/_opam
+OCAML_TOPLEVEL_PATH := $(OPAM_SWITCH_PREFIX)/lib/toplevel
+PATH := $(OPAM_SWITCH_PREFIX)/bin:$(PATH)
+endif
+
 # Temp directory
 TMPDIR ?= /tmp
 
@@ -43,28 +51,27 @@ clean:
 	@rm -rf src/$(COVERAGE_DIR)
 	@rm -rf src/app/libp2p_helper/result src/libp2p_ipc/libp2p_ipc.capnp.go
 
+switch:
+	./scripts/update-opam-switch.sh
+
 # enforces the OCaml version being used
-ocaml_version:
+ocaml_version: switch
 	@if ! ocamlopt -config | grep "version:" | grep $(OCAML_VERSION); then echo "incorrect OCaml version, expected version $(OCAML_VERSION)" ; exit 1; fi
 
 # enforce machine word size
-ocaml_word_size:
+ocaml_word_size: switch
 	@if ! ocamlopt -config | grep "word_size:" | grep $(WORD_SIZE); then echo "invalid machine word size, expected $(WORD_SIZE)" ; exit 1; fi
 
 
 # Checks that the current opam switch contains the packages from opam.export at the same version.
 # This check is disabled in the pure nix environment (that does not use opam).
-check_opam_switch:
+check_opam_switch: switch
 ifneq ($(DISABLE_CHECK_OPAM_SWITCH), true)
-    ifeq (, $(shell which check_opam_switch))
-	$(warning The check_opam_switch binary was not found in the PATH.)
-	$(error The current opam switch should likely be updated by running: "opam switch import opam.export")
-    else
-	check_opam_switch opam.export
-    endif
+	@which check_opam_switch 2>/dev/null >/dev/null || ( echo "The check_opam_switch binary was not found in the PATH, try: opam switch import opam.export" >&2 && exit 1 )
+	@check_opam_switch opam.export
 endif
 
-ocaml_checks: ocaml_version ocaml_word_size check_opam_switch
+ocaml_checks: switch ocaml_version ocaml_word_size check_opam_switch
 
 libp2p_helper:
 ifeq (, $(MINA_LIBP2P_HELPER_PATH))
@@ -82,22 +89,22 @@ check: ocaml_checks libp2p_helper
 
 build: ocaml_checks reformat-diff libp2p_helper
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/generate_keypair/generate_keypair.exe src/app/validate_keypair/validate_keypair.exe src/app/runtime_genesis_ledger/runtime_genesis_ledger.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_all_sigs: ocaml_checks reformat-diff libp2p_helper
+build_all_sigs: ocaml_checks reformat-diff libp2p_helper build
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/logproc/logproc.exe src/app/cli/src/mina.exe src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune build src/app/cli/src/mina_testnet_signatures.exe src/app/cli/src/mina_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_archive: ocaml_checks reformat-diff
 	$(info Starting Build)
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
-build_archive_all_sigs: ocaml_checks reformat-diff
+build_archive_utils: ocaml_checks reformat-diff
 	$(info Starting Build)
-	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/archive/archive_testnet_signatures.exe src/app/archive/archive_mainnet_signatures.exe --profile=$(DUNE_PROFILE)
+	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && dune build src/app/archive/archive.exe src/app/replayer/replayer.exe src/app/archive_blocks/archive_blocks.exe src/app/extract_blocks/extract_blocks.exe src/app/missing_blocks_auditor/missing_blocks_auditor.exe --profile=$(DUNE_PROFILE)
 	$(info Build complete)
 
 build_rosetta: ocaml_checks
@@ -201,14 +208,7 @@ publish-macos:
 	@./scripts/publish-macos.sh
 
 deb:
-	./scripts/rebuild-deb.sh
-	./scripts/archive/build-release-archives.sh
-	@mkdir -p /tmp/artifacts
-	@cp _build/mina*.deb /tmp/artifacts/.
-
-deb_optimized:
-	./scripts/rebuild-deb.sh "optimized"
-	./scripts/archive/build-release-archives.sh
+	./scripts/debian/builder.sh
 	@mkdir -p /tmp/artifacts
 	@cp _build/mina*.deb /tmp/artifacts/.
 
@@ -222,14 +222,12 @@ build_or_download_pv_keys: ocaml_checks
 	(ulimit -s 65532 || true) && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
 	$(info Keys built)
 
-publish_debs:
-	@./buildkite/scripts/publish-deb.sh
-
 genesiskeys:
 	@mkdir -p /tmp/artifacts
 	@cp _build/default/src/lib/key_gen/sample_keypairs.ml /tmp/artifacts/.
 	@cp _build/default/src/lib/key_gen/sample_keypairs.json /tmp/artifacts/.
 
+
 ##############################################
 ## Genesis ledger in OCaml from running daemon
 
@@ -304,4 +302,4 @@ ml-docs: ocaml_checks
 # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
 # HACK: cat Makefile | egrep '^\w.*' | sed 's/:/ /' | awk '{print $1}' | grep -v myprocs | sort | xargs
 
-.PHONY: all build check-format clean deb dev mina-docker reformat doc_diagrams ml-docs macos-setup macos-setup-download setup-opam libp2p_helper dhall_types replayer missing_blocks_auditor extract_blocks archive_blocks ocaml_version ocaml_word_size ocaml_checks
+.PHONY: all build check-format clean deb dev mina-docker reformat doc_diagrams ml-docs macos-setup macos-setup-download setup-opam libp2p_helper dhall_types replayer missing_blocks_auditor extract_blocks archive_blocks ocaml_version ocaml_word_size ocaml_checks switch

README-ci-failures.md

@@ -29,8 +29,8 @@ details and links to the failures.
 ## CI environment mismatch
 
 The CI runs its jobs in multiple Docker images. The images that it is using are
-specified in `buildkite/src/Constants/COntainerImages.dhall`: the CI uses all
-Debian images prefixed by `minaToolchainBuster`.
+specified in `buildkite/src/Constants/ContainerImages.dhall`: the CI uses all
+Debian images prefixed by `minaToolchainBullseye`.
 
 These images are generated by the CI itself, in particular based on the content of the
 `dockerfiles` directory and the `opam.export` file (which describes versions of
@@ -46,7 +46,7 @@ In this case, you should:
    [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain)
    where they have been automatically uploaded. Look for your branch name and
    commit hash in the second column. You should find several images suffixed
-   with `-buster`, `-bullseye`, `-stretch` and `-focal` (Debian versions).
+   with `-bullseye` and `-focal` (Debian versions).
 3. For each such image, retrieve its full name and hash by hovering its link and
    clicking the `Copy full image name` tooltip that appears (or retrieve it on
    the image's page).

README-dev.md

@@ -250,6 +250,47 @@ dune exec src/app/cli/src/mina.exe -- -help
 
 The command line help is the place to learn about other options to the Mina CLI and how to connect to an existing network, such as Mainnet.
 
+## Building debian package locally
+
+Debian package can be build locally by using below commands:
+
+
+1. Build binaries
+```
+make build
+```
+2. Build debian for mina-devet (example for ubuntu 18.04):
+```
+./scripts/debian/build.sh daemon_devnet
+```
+
+## Building docker locally
+
+Prerequisites: 
+
+- debian package previously built
+- aptly app
+
+Steps:
+
+1. Start local debian repository
+```
+./scripts/debian/aptly.sh start -b -c focal -d _build/ -m unstable -l -p 8081
+```
+
+IMPORTANT: debians should be placed in _build folder
+
+2. Build docker:
+```
+./scripts/docker/build.sh --service mina-daemon -v 3.0.0-dkijania-local-debian-build-a099fc7 --network devnet --deb-codename focal --deb-version 3.0.0-dkijania-local-debian-build-a099fc7 
+```
+
+Where:
+
+`-v` - base docker tag
+`--deb-codename` - input debian codename (buster,bullseye etc.)
+`--deb-version` - version of debian which docker will host
+
 ## Using the Makefile
 
 The Makefile contains placeholder targets for all the common tasks that need to be done and automatically knows how to use Docker.

automation/terraform/modules/services/daemon/README.md

@@ -41,17 +41,31 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<DEB_VERSION> --build-arg deb_release=<mina package release channel, e.g. alpha>"`
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s mina-daemon -v 0.0.10-beta4 --extra-args "--build-arg deb_version=0.0.10-beta4-fff3b856 --build-arg deb_release=alpha`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `'deb_version` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
 ### daemon Container
 
-*(From the root of the `MinaProtocol/mina/automation` folder in the mina repository)*
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step> "`
+*(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 Example:
-`./scripts/release-docker.sh -s daemon -v 0.0.10-beta4 --extra-args "--build-arg base_image_tag=0.0.10-beta4"`
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`

automation/terraform/modules/services/faucet/README.md

@@ -63,14 +63,17 @@ The manual commands to release each container are the following:
 ### Mina-Daemon Container
 
 *(From the root of the `MinaProtocol/mina` repository)*
+`./scripts/docker/build.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<deb_version>"`
+and 
 
-### daemon Container
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
 
-*(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+Example:
+`./scripts/docker/build.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
+
+`./scripts/docker/release.sh -s mina-daemon -v 0.0.10-beta4 --deb_version=0.0.10-beta4-fff3b856 --deb_release=alpha`
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -80,5 +83,10 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s bot -v <major>.<minor>.<patch>`
+```
+./scripts/docker/build.sh -s bot -v <major>.<minor>.<patch>
+./scripts/docker/release.sh -s bot -v <major>.<minor>.<patch>
+```
+
+
 

automation/terraform/modules/services/graphql-proxy/README.md

@@ -54,13 +54,18 @@ The manual commands to release each container are the following:
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s mina-daemon -v <major>.<minor>.<patch> --extra-args "--build-arg deb_version=<MINA_VERSION>"`
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> --deb_version=<MINA_VERSION>"`
+
+and 
+
+`./scripts/docker/release.sh -s mina-daemon -v <major>.<minor>.<patch> -  --deb_version=<DEB_VERSION> --deb_release=<mina package release channel, e.g. alpha>"`
+
 
 ### daemon Container
 
 *(From the root of the `MinaProtocol/mina/automation` repository)*
 
-`./scripts/release-docker.sh -s daemon -v <major>.<minor>.<patch> --extra-args "--build-arg base_image_tag=<docker tag created in first step>"`
+`./scripts/docker/release.sh -s daemon -v <major>.<minor>.<patch> `
 
 The `--extra-args` argument is for passing additional parameters directly to the `docker build` command. It is used here to pass the required Dockerfile variable `base_image_tag` but can also be used to override Dockerfile variables with default values like so `--build-arg deb_repo=release`
 
@@ -70,4 +75,4 @@ The Faucet Dockerfile lives in the `MinaProtocol/mina` repository [here](https:/
 
 *(From the root of the `MinaProtocol/mina` repository)*
 
-`./scripts/release-docker.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`
+`./scripts/docker/release.sh -s graphql-public-proxy -v <major>.<minor>.<patch>`

buildkite/scripts/build-release.sh

@@ -14,7 +14,7 @@ echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkele
 
 
 echo "--- Prepare debian packages"
-./scripts/rebuild-deb.sh $@
+./scripts/debian/build.sh $@
 
 echo "--- Git diff after build is complete:"
 git diff --exit-code -- .

buildkite/scripts/promote-deb.sh → buildkite/scripts/debian/promote.sh

@@ -56,7 +56,7 @@ echo "Promoting debs: ${PACKAGE}_${VERSION} to Release: ${TO_COMPONENT} and Code
 if [[ -z "$NEW_VERSION" ]] || [[ "$NEW_VERSION" == "$VERSION" ]]; then
   deb-s3 copy --s3-region=us-west-2 --lock --bucket packages.o1test.net --preserve-versions --cache-control=max-age=120  $PACKAGE $CODENAME $TO_COMPONENT --versions $VERSION --arch $ARCH --component ${FROM_COMPONENT} --codename ${CODENAME}
 else
-  source scripts/reversion-deb.sh \
+  source scripts/debian/reversion.sh \
     --deb $PACKAGE  \
     --codename $CODENAME \
     --new-release $TO_COMPONENT \

buildkite/scripts/publish-deb.sh → buildkite/scripts/debian/publish.sh

@@ -2,7 +2,7 @@
 set -eox pipefail
 
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
-source "${SCRIPTPATH}/export-git-env-vars.sh"
+source "${SCRIPTPATH}/../export-git-env-vars.sh"
 
 DOWNLOAD_FOLDER=_build
 
@@ -15,7 +15,7 @@ if [ -z "$AWS_ACCESS_KEY_ID" ]; then
     exit 0
 fi
 
-source scripts/publish-deb.sh \
+source scripts/debian/publish.sh \
   --names "${DOWNLOAD_FOLDER}/mina-*.deb" \
   --release $MINA_DEB_RELEASE \
   --version $MINA_DEB_VERSION \

buildkite/scripts/run_promote_build_job.sh

@@ -18,7 +18,7 @@
 #          - "NETWORK=Devnet"
 #          - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
 #          - "NEW_VERSION=3.0.0fake-ddb6fc4"
-#          - "CODENAMES=Focal,Buster,Bullseye"
+#          - "CODENAMES=Focal,Bullseye"
 #          - "FROM_CHANNEL=Unstable"
 #          - "TO_CHANNEL=Experimental"
 #        image: codaprotocol/ci-toolchain-base:v3
@@ -41,7 +41,7 @@ function usage() {
   fi
   echo "  DEBIANS                     The comma delimitered debian names. For example: 'Daemon,Archive' "
   echo "  DOCKERS                     The comma delimitered docker names. For example: 'Daemon,Archive' "
-  echo "  CODENAMES                   The Debian codenames (Bullseye, Buster etc.)"
+  echo "  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)"
   echo "  FROM_VERSION                The Source Docker or Debian version "
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"

buildkite/scripts/run_verify_promoted_build_job.sh

@@ -18,7 +18,7 @@
 #          - "NETWORK=Devnet"
 #          - "FROM_VERSION=3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
 #          - "NEW_VERSION=3.0.0fake-ddb6fc4"
-#          - "CODENAMES=Focal,Buster,Bullseye"
+#          - "CODENAMES=Focal,Bullseye"
 #          - "FROM_CHANNEL=Unstable"
 #          - "TO_CHANNEL=Experimental"
 #        image: codaprotocol/ci-toolchain-base:v3
@@ -41,7 +41,7 @@ function usage() {
   fi
   echo "  DEBIANS                     The comma delimitered debian names. For example: 'Daemon,Archive' "
   echo "  DOCKERS                     The comma delimitered docker names. For example: 'Daemon,Archive' "
-  echo "  CODENAMES                   The Debian codenames (Bullseye, Buster etc.)"
+  echo "  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)"
   echo "  NEW_VERSION                 The new Debian version or new Docker tag"
   echo "  REMOVE_PROFILE_FROM_NAME    Should we remove profile suffix from debian name"
   echo "  PROFILE                     The Docker and Debian profile (Standard, Lightnet)"