Merge pull request #14278 from MinaProtocol/feature/berkeley-into-ram…

…pup-2023-10-05

Merge berkeley into rampup

.github/workflows/nix.yml

@@ -0,0 +1,24 @@
+name: "Build nix"
+on:
+  pull_request:
+  push:
+    branches:
+      - develop
+      - compatible
+      - berkeley
+      - releases/**
+      - main
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - uses: cachix/install-nix-action@v20
+    - uses: DeterminateSystems/magic-nix-cache-action@v2
+    - run: nix/pin.sh 
+    - run: nix build mina
+      env:
+        GIT_LFS_SKIP_SMUDGE: 1
+    - run: nix develop mina

.gitmodules

@@ -4,9 +4,6 @@
 [submodule "src/external/ppx_optcomp"]
 	path = src/external/ppx_optcomp
 	url = https://github.com/MinaProtocol/ppx_optcomp
-[submodule "frontend/wallet/tablecloth"]
-	path = frontend/wallet/tablecloth
-	url = https://github.com/bkase/tablecloth
 [submodule "src/lib/crypto/proof-systems"]
 	path = src/lib/crypto/proof-systems
 	url = https://github.com/o1-labs/proof-systems.git

CODEOWNERS

@@ -30,8 +30,6 @@
 /src/lib/ @MinaProtocol/protocol-eng-reviewers
 
 /src/lib/blake2/ @MinaProtocol/crypto-eng-reviewers
-/src/lib/cli_lib/ @MinaProtocol/product-eng-reviewers
-/src/lib/mina_commands @MinaProtocol/product-eng-reviewers
 /src/lib/mina_graphql @MinaProtocol/product-eng-reviewers
 /src/lib/mina_numbers/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/crs/ @MinaProtocol/crypto-eng-reviewers

Makefile

@@ -262,9 +262,6 @@ deb_optimized:
 	@mkdir -p /tmp/artifacts
 	@cp _build/mina*.deb /tmp/artifacts/.
 
-test_executive_deb:
-	./scripts/rebuild_test_executive_deb.sh
-
 build_pv_keys: ocaml_checks
 	$(info Building keys)
 	ulimit -s 65532 && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
@@ -275,9 +272,6 @@ build_or_download_pv_keys: ocaml_checks
 	ulimit -s 65532 && (ulimit -n 10240 || true) && env MINA_COMMIT_SHA1=$(GITLONGHASH) dune exec --profile=$(DUNE_PROFILE) src/lib/snark_keys/gen_keys/gen_keys.exe -- --generate-keys-only
 	$(info Keys built)
 
-publish_deb:
-	@./scripts/publish-deb.sh
-
 publish_debs:
 	@./buildkite/scripts/publish-deb.sh
 

README-dev.md

@@ -89,7 +89,7 @@ A number of C libraries are expected to be available in the system and are also
 If you use vim, add this snippet in your `.vimrc` file to use Merlin. (Note: Be sure to change the HOME directory to match yours.)
 
 ```bash
-let s:ocamlmerlin="/Users/USERNAME/.opam/4.07/share/merlin"
+let s:ocamlmerlin="/Users/USERNAME/.opam/4.14.0/share/merlin"
 execute "set rtp+=".s:ocamlmerlin."/vim"
 execute "set rtp+=".s:ocamlmerlin."/vimbufsync"
 let g:syntastic_ocaml_checkers=['merlin']
@@ -104,7 +104,7 @@ let g:syntastic_ocaml_checkers=['merlin']
 - If you use emacs, install the `opam` packages mentioned above and also install `tuareg`. Add the following to your `.emacs` file:
 
 ```lisp
-(let ((opam-share (ignore-errors (car (process-lines "opam" "config" "var" "share")))))
+(let ((opam-share (ignore-errors (car (process-lines "opam" "var" "share")))))
   (when (and opam-share (file-directory-p opam-share))
     ;; Register Merlin
     (add-to-list 'load-path (expand-file-name "emacs/site-lisp" opam-share))

automation/terraform/modules/kubernetes/testnet/helm.tf

@@ -157,3 +157,22 @@ resource "helm_release" "zkapps-dashboard" {
   timeout    = 600
   depends_on = [helm_release.archive_node]
 }
+
+# itn orchestrator
+
+resource "helm_release" "itn-orchestrator" {
+  provider   = helm.testnet_deploy
+  count      = var.expose_itn_graphql ? 1 : 0
+
+  name       = "${var.testnet_name}-itn-orchestrator"
+  repository = var.use_local_charts ? "" : local.mina_helm_repo
+  chart      = var.use_local_charts ? "../../../../helm/itn-orchestrator" : "itn-orchestrator"
+  version    = "0.1.0"
+  namespace  = kubernetes_namespace.testnet_namespace.metadata[0].name
+  values = [
+    yamlencode(local.itn_orchestrator_vars)
+  ]
+  wait       = false
+  timeout    = 600
+  depends_on = [helm_release.seeds]
+}

automation/terraform/modules/kubernetes/testnet/locals.tf

@@ -35,9 +35,13 @@ locals {
         metrics = "8081"
         p2p     = var.seed_external_port
       }
-      # seedPeersURL         = var.seed_peers_url
+      itn = {
+        port    = "3086"
+        keys    = var.itn_keys
+      }
       uploadBlocksToGCloud = var.upload_blocks_to_gcloud
       exposeGraphql        = var.expose_graphql
+      exposeItnGraphql     = var.expose_itn_graphql
     }
 
     priorityClass = var.priority_class
@@ -75,6 +79,7 @@ locals {
     uploadBlocksToGCloud = var.upload_blocks_to_gcloud
     # seedPeersURL         = var.seed_peers_url
     exposeGraphql        = var.expose_graphql
+    exposeItnGraphql     = var.expose_itn_graphql
     cpuRequest = var.cpu_request
     memRequest= var.mem_request
   }
@@ -216,4 +221,9 @@ locals {
     makeReportAccounts          = var.make_report_accounts
     seedPeersURL                = var.additional_peers
   }
+
+  itn_orchestrator_vars = {
+    testnetName = var.testnet_name
+    image       = var.itn_orchestrator_image
+  }
 }

automation/terraform/modules/kubernetes/testnet/variables.tf

@@ -26,6 +26,16 @@ variable "expose_graphql" {
   default = false
 }
 
+variable "expose_itn_graphql" {
+  type    = bool
+  default = false
+}
+
+variable "itn_keys" {
+  type    = string
+  default = ""
+}
+
 variable "use_local_charts" {
   type    = bool
   default = false
@@ -107,6 +117,12 @@ variable "watchdog_image" {
   default = "gcr.io/o1labs-192920/watchdog:latest"
 }
 
+variable "itn_orchestrator_image" {
+  type    = string
+  default = "gcr.io/o1labs-192920/itn_orchestrator_image:latest"
+}
+
+
 # this must be a string to avoid scientific notation truncation
 variable "mina_faucet_amount" {
   type    = string

automation/terraform/modules/o1-integration/inputs.tf

@@ -6,6 +6,21 @@ variable "deploy_graphql_ingress" {
   type = bool
 }
 
+variable "expose_itn_graphql" {
+  type    = bool
+  default = false
+}
+
+variable "itn_keys" {
+  type    = string
+  default = ""
+}
+
+variable "itn_orchestrator_image" {
+  type    = string
+  default = "gcr.io/o1labs-192920/itn_orchestrator_image:latest"
+}
+
 variable "aws_route53_zone_id" {
   type = string
 }

automation/terraform/modules/o1-integration/testnet.tf

@@ -4,6 +4,8 @@ module "kubernetes_testnet" {
 
   use_local_charts    = true
   expose_graphql      = var.deploy_graphql_ingress
+  expose_itn_graphql  = var.expose_itn_graphql
+  itn_keys            = var.itn_keys
   healthcheck_enabled = false
   deploy_watchdog     = false
 
@@ -12,7 +14,8 @@ module "kubernetes_testnet" {
   k8s_context    = var.k8s_context
   testnet_name   = var.testnet_name
   priority_class = kubernetes_priority_class.testnet_priority_class.metadata[0].name
-
+
+  itn_orchestrator_image        = var.itn_orchestrator_image
   mina_image                    = var.mina_image
   use_custom_entrypoint         = true
   custom_entrypoint             = "/mina_daemon_puppeteer.py"

automation/terraform/modules/o1-testnet/inputs.tf

@@ -80,6 +80,11 @@ variable "watchdog_image" {
   default = "gcr.io/o1labs-192920/watchdog:latest"
 }
 
+variable "itn_orchestrator_image" {
+  type    = string
+  default = "gcr.io/o1labs-192920/itn_orchestrator_image:latest"
+}
+
 # this must be a string to avoid scientific notation truncation
 variable "mina_faucet_amount" {
   type    = string

automation/terraform/modules/o1-testnet/testnet.tf

@@ -35,6 +35,7 @@ module "kubernetes_testnet" {
   mina_bots_image    = var.mina_bots_image
   mina_points_image  = var.mina_points_image
   watchdog_image     = var.watchdog_image
+  itn_orchestrator_image = var.itn_orchestrator_image
 
   mina_faucet_amount = var.mina_faucet_amount
   mina_faucet_fee    = var.mina_faucet_fee

automation/terraform/modules/testnet-alerts/templates/testnet-alert-rules.yml.tpl

@@ -610,28 +610,6 @@ groups:
       description: "No node has received transactions in their transaction pool in the last 2 slots (6 minutes) on network {{ $labels.testnet }}."
       runbook: "https://www.notion.so/minaprotocol/NoNewTransactions-27dbeafab8ea4d659ee6f748acb2fd6c"
 
-  - alert: HighUnparentedBlockCount
-    expr: max by (testnet) (Coda_Archive_unparented_blocks {${berkeley_testnet}}) > 30
-    for: ${alert_evaluation_duration}
-    labels:
-      testnet: "{{ $labels.testnet }}"
-      severity: critical
-    annotations:
-      summary: "{{ $labels.testnet }} has a critically high unparented block count"
-      description: "{{ $value }} Unparented block count is critically high on network {{ $labels.testnet }}."
-      runbook: "https://www.notion.so/minaprotocol/Archive-Node-Metrics-9edf9c51dd344f1fbf6722082a2e2465"
-
-  - alert: HighMissingBlockCount
-    expr: max by (testnet) (Coda_Archive_missing_blocks {${berkeley_testnet}}) > 30
-    for: ${alert_evaluation_duration}
-    labels:
-      testnet: "{{ $labels.testnet }}"
-      severity: critical
-    annotations:
-      summary: "{{ $labels.testnet }} has a critically high missing block count"
-      description: "{{ $value }} Missing block count is critically high on network {{ $labels.testnet }}."
-      runbook: "https://www.notion.so/minaprotocol/Archive-Node-Metrics-9edf9c51dd344f1fbf6722082a2e2465"
-
   - alert: FewBlocksPerHour
     expr: quantile by (testnet) (0.5, increase(Coda_Transition_frontier_max_blocklength_observed {${berkeley_testnet},${synced_status_filter}}  [30m])) < 1
     for: ${alert_evaluation_duration}
@@ -735,26 +713,50 @@ groups:
       description: "{{ $value }} blocks have been produced that contains an invalid blockchain snark proof in last hour"
       runbook: "https://www.notion.so/minaprotocol/LowInvalidProofPerHour-b6e88b9ae84f47169e7f86017ab9e340"
 
-  - alert: LowPostgresBlockHeightGrowth
-    expr: min by (testnet) (increase(Coda_Archive_max_block_height {${berkeley_testnet}}  [${alert_timeframe}])) < 1
-    for: ${alert_evaluation_duration}
+  - alert: NodeRestarted
+    expr: count by (testnet) (Coda_Runtime_process_uptime_ms_total {${berkeley_testnet}} < 360000) > 0
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning
     annotations:
-      summary: "{{ $labels.testnet }} rate of archival of network blocks in Postgres DB is lower than expected"
-      description: "The rate of {{ $value }} new blocks observed by archive postgres instances is low on network {{ $labels.testnet }}."
+      summary: "At least one of the nodes on {{ $labels.testnet }} restarted"
+      description: "{{ $value }} nodes on {{ $labels.testnet }} restarted"
+      runbook: "https://www.notion.so/minaprotocol/NodeRestarted-99a1cf710ff14aa6930a9f12ad5813a5"
+
+- name: Berkeley archive node alerts
+  rules:
+  - alert: HighUnparentedBlockCount
+    expr: max by (testnet) (Coda_Archive_unparented_blocks {${berkeley_testnet}}) > 30
+    for: ${alert_evaluation_duration}
+    labels:
+      testnet: "{{ $labels.testnet }}"
+      severity: critical
+    annotations:
+      summary: "{{ $labels.testnet }} has a critically high unparented block count"
+      description: "{{ $value }} Unparented block count is critically high on network {{ $labels.testnet }}."
       runbook: "https://www.notion.so/minaprotocol/Archive-Node-Metrics-9edf9c51dd344f1fbf6722082a2e2465"
 
-  - alert: NodeRestarted
-    expr: count by (testnet) (Coda_Runtime_process_uptime_ms_total {${berkeley_testnet}} < 360000) > 0
+  - alert: HighMissingBlockCount
+    expr: max by (testnet) (Coda_Archive_missing_blocks {${berkeley_testnet}}) > 30
+    for: ${alert_evaluation_duration}
+    labels:
+      testnet: "{{ $labels.testnet }}"
+      severity: critical
+    annotations:
+      summary: "{{ $labels.testnet }} has a critically high missing block count"
+      description: "{{ $value }} Missing block count is critically high on network {{ $labels.testnet }}."
+      runbook: "https://www.notion.so/minaprotocol/Archive-Node-Metrics-9edf9c51dd344f1fbf6722082a2e2465"
+
+  - alert: LowPostgresBlockHeightGrowth
+    expr: min by (testnet) (increase(Coda_Archive_max_block_height {${berkeley_testnet}}  [${alert_timeframe}])) < 1
+    for: ${alert_evaluation_duration}
     labels:
       testnet: "{{ $labels.testnet }}"
       severity: warning
     annotations:
-      summary: "At least one of the nodes on {{ $labels.testnet }} restarted"
-      description: "{{ $value }} nodes on {{ $labels.testnet }} restarted"
-      runbook: "https://www.notion.so/minaprotocol/NodeRestarted-99a1cf710ff14aa6930a9f12ad5813a5"
+      summary: "{{ $labels.testnet }} rate of archival of network blocks in Postgres DB is lower than expected"
+      description: "The rate of {{ $value }} new blocks observed by archive postgres instances is low on network {{ $labels.testnet }}."
+      runbook: "https://www.notion.so/minaprotocol/Archive-Node-Metrics-9edf9c51dd344f1fbf6722082a2e2465"
 
   - alert: UnparentedBlocksObserved
     expr: max by (testnet) (Coda_Archive_unparented_blocks {${berkeley_testnet}}) > 1