validator deploy tweaks (#2784)

### Description - fixes to deploy testnet4 for agents ### Drive-by changes - truncating labels to length 63 - adding `.relayer` to `agentConfig.contextChainNames` ### Related issues - for hyperlane-xyz/issues#574 ### Backward compatibility Yes ### Testing Manual --------- Co-authored-by: Trevor Porter <[email protected]> Co-authored-by: Nam Chu Hoai <[email protected]>
hyperlane-xyz · Oct 16, 2023 · 9168cca · 9168cca
1 parent a9e8b17
commit 9168cca
Show file tree

Hide file tree

Showing 46 changed files with 772 additions and 302 deletions.
diff --git a/rust/README.md b/rust/README.md
@@ -63,9 +63,9 @@ kubectl cp testnet3/fuji-hyperlane-agent-validator-0:/usr/share/hyperlane /tmp/f
 Configure additional env variables appropriately:
 
 ```bash
-HYP_BASE_DB=/tmp/fuji-validator-db
+HYP_DB=/tmp/fuji-validator-db
 CONFIG_FILES=./config/testnet_config.json
-HYP_BASE_TRACING_FMT=pretty
+HYP_TRACING_FMT=pretty
 DATABASE_URL=<READ_REPLICA_POSTGRES_URL> # for scraper
 ```
 

diff --git a/rust/agents/relayer/src/msg/metadata/base.rs b/rust/agents/relayer/src/msg/metadata/base.rs
@@ -138,7 +138,7 @@ impl BaseMetadataBuilder {
         Ok(proof)
     }
 
-    pub async fn highest_known_nonce(&self) -> Option<u32> {
+    pub async fn highest_known_leaf_index(&self) -> Option<u32> {
         self.origin_prover_sync.read().await.count().checked_sub(1)
     }
 

diff --git a/rust/agents/relayer/src/msg/metadata/multisig/legacy_multisig.rs b/rust/agents/relayer/src/msg/metadata/multisig/legacy_multisig.rs
@@ -37,15 +37,15 @@ impl MultisigIsmMetadataBuilder for LegacyMultisigMetadataBuilder {
         checkpoint_syncer: &MultisigCheckpointSyncer,
     ) -> Result<Option<MultisigMetadata>> {
         const CTX: &str = "When fetching LegacyMultisig metadata";
-        unwrap_or_none_result!(highest_nonce, self.highest_known_nonce().await);
+        unwrap_or_none_result!(highest_leaf_index, self.highest_known_leaf_index().await);
         unwrap_or_none_result!(
             quorum_checkpoint,
             checkpoint_syncer
                 .legacy_fetch_checkpoint_in_range(
                     validators,
                     threshold as usize,
                     message.nonce,
-                    highest_nonce,
+                    highest_leaf_index,
                 )
                 .await
                 .context(CTX)?

diff --git a/rust/agents/relayer/src/msg/metadata/multisig/merkle_root_multisig.rs b/rust/agents/relayer/src/msg/metadata/multisig/merkle_root_multisig.rs
@@ -7,6 +7,7 @@ use derive_new::new;
 use eyre::{Context, Result};
 use hyperlane_base::MultisigCheckpointSyncer;
 use hyperlane_core::{unwrap_or_none_result, HyperlaneMessage, H256};
+use tracing::debug;
 
 use crate::msg::metadata::BaseMetadataBuilder;
 
@@ -35,18 +36,23 @@ impl MultisigIsmMetadataBuilder for MerkleRootMultisigMetadataBuilder {
         checkpoint_syncer: &MultisigCheckpointSyncer,
     ) -> Result<Option<MultisigMetadata>> {
         const CTX: &str = "When fetching MerkleRootMultisig metadata";
-        unwrap_or_none_result!(highest_nonce, self.highest_known_nonce().await);
+        unwrap_or_none_result!(
+            highest_leaf_index,
+            self.highest_known_leaf_index().await,
+            debug!("Couldn't get highest known leaf index")
+        );
         unwrap_or_none_result!(
             quorum_checkpoint,
             checkpoint_syncer
                 .fetch_checkpoint_in_range(
                     validators,
                     threshold as usize,
                     message.nonce,
-                    highest_nonce
+                    highest_leaf_index
                 )
                 .await
-                .context(CTX)?
+                .context(CTX)?,
+            debug!("Couldn't get checkpoint in range")
         );
         unwrap_or_none_result!(
             proof,
@@ -58,7 +64,8 @@ impl MultisigIsmMetadataBuilder for MerkleRootMultisigMetadataBuilder {
             merkle_leaf_id,
             self.get_merkle_leaf_id_by_message_id(message.id())
                 .await
-                .context(CTX)?
+                .context(CTX)?,
+            debug!("Couldn't get merkle proof")
         );
         Ok(Some(MultisigMetadata::new(
             quorum_checkpoint.checkpoint.checkpoint,

diff --git a/rust/agents/relayer/src/msg/metadata/multisig/message_id_multisig.rs b/rust/agents/relayer/src/msg/metadata/multisig/message_id_multisig.rs
@@ -7,7 +7,7 @@ use derive_new::new;
 use eyre::{Context, Result};
 use hyperlane_base::MultisigCheckpointSyncer;
 use hyperlane_core::{unwrap_or_none_result, HyperlaneMessage, H256};
-use tracing::warn;
+use tracing::{debug, trace, warn};
 
 use crate::msg::metadata::BaseMetadataBuilder;
 
@@ -35,12 +35,23 @@ impl MultisigIsmMetadataBuilder for MessageIdMultisigMetadataBuilder {
         checkpoint_syncer: &MultisigCheckpointSyncer,
     ) -> Result<Option<MultisigMetadata>> {
         const CTX: &str = "When fetching MessageIdMultisig metadata";
+        unwrap_or_none_result!(
+            leaf_index,
+            self.get_merkle_leaf_id_by_message_id(message.id())
+                .await
+                .context(CTX)?,
+            debug!(
+                ?message,
+                "No merkle leaf found for message id, must have not been enqueued in the tree"
+            )
+        );
         unwrap_or_none_result!(
             quorum_checkpoint,
             checkpoint_syncer
-                .fetch_checkpoint(validators, threshold as usize, message.nonce)
+                .fetch_checkpoint(validators, threshold as usize, leaf_index)
                 .await
-                .context(CTX)?
+                .context(CTX)?,
+            trace!("No quorum checkpoint found")
         );
 
         if quorum_checkpoint.checkpoint.message_id != message.id() {

diff --git a/rust/agents/scraper/migration/src/m20230309_000001_create_table_domain.rs b/rust/agents/scraper/migration/src/m20230309_000001_create_table_domain.rs
@@ -46,6 +46,14 @@ const DOMAINS: &[RawDomain] = &[
         is_test_net: false,
         is_deprecated: false,
     },
+    RawDomain {
+        name: "basegoerli",
+        token: "ETH",
+        domain: 84531,
+        chain_id: 84531,
+        is_test_net: false,
+        is_deprecated: false,
+    },
     RawDomain {
         name: "bsc",
         token: "BNB",
@@ -150,6 +158,14 @@ const DOMAINS: &[RawDomain] = &[
         is_test_net: false,
         is_deprecated: false,
     },
+    RawDomain {
+        name: "scrollsepolia",
+        token: "ETH",
+        domain: 534351,
+        chain_id: 534351,
+        is_test_net: true,
+        is_deprecated: false,
+    },
     RawDomain {
         name: "sepolia",
         token: "ETH",
@@ -158,6 +174,14 @@ const DOMAINS: &[RawDomain] = &[
         is_test_net: true,
         is_deprecated: false,
     },
+    RawDomain {
+        name: "polygonzkevmtestnet",
+        token: "ETH",
+        domain: 1442,
+        chain_id: 1442,
+        is_test_net: true,
+        is_deprecated: false,
+    },
     RawDomain {
         name: "test1",
         token: "ETH",

diff --git a/rust/agents/scraper/src/db/block_cursor.rs b/rust/agents/scraper/src/db/block_cursor.rs
@@ -91,7 +91,7 @@ impl BlockCursor {
             };
             debug!(?model, "Inserting cursor");
             if let Err(e) = Insert::one(model).exec(&self.db).await {
-                warn!(error = ?e, "Failed to update database with new cursor")
+                warn!(error = ?e, "Failed to update database with new cursor. When you just started this, ensure that the migrations included this domain.")
             } else {
                 debug!(cursor = ?*inner, "Updated cursor")
             }

diff --git a/rust/agents/validator/src/submit.rs b/rust/agents/validator/src/submit.rs
@@ -3,12 +3,12 @@ use std::sync::Arc;
 use std::time::{Duration, Instant};
 use std::vec;
 
-use eyre::Result;
+use eyre::{bail, Result};
 use hyperlane_core::MerkleTreeHook;
 use prometheus::IntGauge;
 use tokio::time::sleep;
-use tracing::instrument;
 use tracing::{debug, info};
+use tracing::{error, instrument};
 
 use hyperlane_base::{db::HyperlaneRocksDB, CheckpointSyncer, CoreMetrics};
 use hyperlane_core::{
@@ -84,12 +84,16 @@ impl ValidatorSubmitter {
             };
 
             // ingest available messages from DB
-            while let Some(message) = self
+            while let Some(insertion) = self
                 .message_db
-                .retrieve_message_by_nonce(tree.count() as u32)?
+                .retrieve_merkle_tree_insertion_by_leaf_index(&(tree.count() as u32))?
             {
-                debug!(index = message.nonce, "Ingesting leaf to tree");
-                let message_id = message.id();
+                debug!(
+                    index = insertion.index(),
+                    queue_length = checkpoint_queue.len(),
+                    "Ingesting leaf to tree"
+                );
+                let message_id = insertion.message_id();
                 tree.ingest(message_id);
 
                 let checkpoint = self.checkpoint(&tree);
@@ -99,6 +103,19 @@ impl ValidatorSubmitter {
                     message_id,
                 });
 
+                if checkpoint.index == correctness_checkpoint.index {
+                    // We got to the right height, now lets compare whether we got the right tree
+                    if checkpoint.root != correctness_checkpoint.root {
+                        // Bad news, bail
+                        error!(
+                            ?checkpoint,
+                            ?correctness_checkpoint,
+                            "Incorrect tree root, something went wrong"
+                        );
+                        bail!("Incorrect tree root, something went wrong");
+                    }
+                }
+
                 // compare against every queued checkpoint to prevent ingesting past target
                 if checkpoint == correctness_checkpoint {
                     debug!(index = checkpoint.index, "Reached tree consistency");

diff --git a/rust/agents/validator/src/validator.rs b/rust/agents/validator/src/validator.rs
@@ -7,12 +7,12 @@ use futures_util::future::ready;
 use hyperlane_base::{
     db::{HyperlaneRocksDB, DB},
     run_all, BaseAgent, CheckpointSyncer, ContractSyncMetrics, CoreMetrics, HyperlaneAgentCore,
-    MessageContractSync,
+    WatermarkContractSync,
 };
 use hyperlane_core::{
     accumulator::incremental::IncrementalMerkle, Announcement, ChainResult, HyperlaneChain,
     HyperlaneContract, HyperlaneDomain, HyperlaneSigner, HyperlaneSignerExt, Mailbox,
-    MerkleTreeHook, TxOutcome, ValidatorAnnounce, H256, U256,
+    MerkleTreeHook, MerkleTreeInsertion, TxOutcome, ValidatorAnnounce, H256, U256,
 };
 use hyperlane_ethereum::{SingletonSigner, SingletonSignerHandle};
 use tokio::{task::JoinHandle, time::sleep};
@@ -30,7 +30,7 @@ pub struct Validator {
     #[as_ref]
     core: HyperlaneAgentCore,
     db: HyperlaneRocksDB,
-    message_sync: Arc<MessageContractSync>,
+    merkle_tree_hook_sync: Arc<WatermarkContractSync<MerkleTreeInsertion>>,
     mailbox: Arc<dyn Mailbox>,
     merkle_tree_hook: Arc<dyn MerkleTreeHook>,
     validator_announce: Arc<dyn ValidatorAnnounce>,
@@ -75,8 +75,8 @@ impl BaseAgent for Validator {
 
         let contract_sync_metrics = Arc::new(ContractSyncMetrics::new(&metrics));
 
-        let message_sync = settings
-            .build_message_indexer(
+        let merkle_tree_hook_sync = settings
+            .build_merkle_tree_hook_indexer(
                 &settings.origin_chain,
                 &metrics,
                 &contract_sync_metrics,
@@ -91,7 +91,7 @@ impl BaseAgent for Validator {
             db: msg_db,
             mailbox: mailbox.into(),
             merkle_tree_hook: merkle_tree_hook.into(),
-            message_sync,
+            merkle_tree_hook_sync,
             validator_announce: validator_announce.into(),
             signer,
             signer_instance: Some(Box::new(signer_instance)),
@@ -129,7 +129,7 @@ impl BaseAgent for Validator {
                     sleep(self.interval).await;
                 }
                 Ok(_) => {
-                    tasks.push(self.run_message_sync().await);
+                    tasks.push(self.run_merkle_tree_hook_sync().await);
                     for checkpoint_sync_task in self.run_checkpoint_submitters().await {
                         tasks.push(checkpoint_sync_task);
                     }
@@ -147,20 +147,13 @@ impl BaseAgent for Validator {
 }
 
 impl Validator {
-    async fn run_message_sync(&self) -> Instrumented<JoinHandle<Result<()>>> {
+    async fn run_merkle_tree_hook_sync(&self) -> Instrumented<JoinHandle<Result<()>>> {
         let index_settings =
             self.as_ref().settings.chains[self.origin_chain.name()].index_settings();
-        let contract_sync = self.message_sync.clone();
-        let cursor = contract_sync
-            .forward_backward_message_sync_cursor(index_settings)
-            .await;
-        tokio::spawn(async move {
-            contract_sync
-                .clone()
-                .sync("dispatched_messages", cursor)
-                .await
-        })
-        .instrument(info_span!("MailboxMessageSyncer"))
+        let contract_sync = self.merkle_tree_hook_sync.clone();
+        let cursor = contract_sync.rate_limited_cursor(index_settings).await;
+        tokio::spawn(async move { contract_sync.clone().sync("merkle_tree_hook", cursor).await })
+            .instrument(info_span!("MerkleTreeHookSyncer"))
     }
 
     async fn run_checkpoint_submitters(&self) -> Vec<Instrumented<JoinHandle<Result<()>>>> {

diff --git a/rust/chains/hyperlane-ethereum/src/interchain_gas.rs b/rust/chains/hyperlane-ethereum/src/interchain_gas.rs
@@ -97,8 +97,6 @@ where
             .query_with_meta()
             .await?;
 
-        println!("found gas payment events: {:?}", events);
-
         Ok(events
             .into_iter()
             .map(|(log, log_meta)| {

diff --git a/rust/config/testnet4_config.json b/rust/config/testnet4_config.json
@@ -44,7 +44,7 @@
       "proxyAdmin": "0x4eDBf5846D973c53AF478cf62aB5bC92807521e3",
       "mailbox": "0xEf9F292fcEBC3848bF4bB92a96a04F9ECBb78E59",
       "validatorAnnounce": "0x3726EE36a2A9e11a40d1ffD7D9A1A16e0154cDA0",
-      "merkleTreeHook": "0x48D4ede231344A19c92b9cA9Ac8B453116012169",
+      "merkleTreeHook": "0x221FA9CBaFcd6c1C3d206571Cf4427703e023FFa",
       "storageGasOracle": "0x8356113754C7aCa297Db3089b89F87CC125499fb",
       "interchainGasPaymaster": "0x1246529edDcA523AfE5c6b9414299633d2E16697",
       "aggregationHook": "0xdBabD76358897E68E4964647C1fb8Bf524f5EFdB",

diff --git a/rust/helm/agent-common/templates/_helpers.tpl b/rust/helm/agent-common/templates/_helpers.tpl
@@ -7,7 +7,7 @@ Expand the name of the chart.
 
 {{/*
 Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+We truncate at 63 chars - 11 because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
 {{- define "agent-common.fullname" -}}
@@ -49,7 +49,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Selector labels
 */}}
 {{- define "agent-common.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "agent-common.name" . }}
+app.kubernetes.io/name: {{ include "agent-common.name" . | trunc 63 | trimSuffix "-"}}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 

diff --git a/rust/helm/hyperlane-agent/templates/external-secret.yaml b/rust/helm/hyperlane-agent/templates/external-secret.yaml
@@ -27,7 +27,7 @@ spec:
    */}}
         {{- range .Values.hyperlane.chains }}
         {{- if not .disabled }}
-        HYP_BASE_CHAINS_{{ .name | upper }}_CUSTOMRPCURLS: {{ printf "'{{ .%s_rpcs | mustFromJson | join \",\" }}'" .name }}
+        HYP_CHAINS_{{ .name | upper }}_CUSTOMRPCURLS: {{ printf "'{{ .%s_rpcs | mustFromJson | join \",\" }}'" .name }}
         {{- end }}
         {{- end }}
   data:

diff --git a/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml b/rust/helm/hyperlane-agent/templates/relayer-external-secret.yaml
@@ -26,6 +26,9 @@ spec:
         HYP_CHAINS_{{ .name | upper }}_SIGNER_KEY: {{ printf "'{{ .%s_signer_key | toString }}'" .name }}
         {{- end }}
         {{- if and (eq .signer.type "aws") $.Values.hyperlane.relayer.aws }}
+        HYP_CHAINS_{{ .name | upper }}_SIGNER_TYPE: aws
+        HYP_CHAINS_{{ .name | upper }}_SIGNER_ID: {{ .signer.id }}
+        HYP_CHAINS_{{ .name | upper }}_SIGNER_REGION: {{ .signer.region}}
         AWS_ACCESS_KEY_ID: {{ print "'{{ .aws_access_key_id | toString }}'" }}
         AWS_SECRET_ACCESS_KEY: {{ print "'{{ .aws_secret_access_key | toString }}'" }}
         {{- end }}

diff --git a/rust/hyperlane-base/src/settings/mod.rs b/rust/hyperlane-base/src/settings/mod.rs
@@ -45,7 +45,7 @@
 //! ```
 //!
 //! and an environment variable is supplied which defines
-//! `HYP_BASE_CHAINS_TEST2_DOMAINID=1`, then the config parser will directly override the value of
+//! `HYP_CHAINS_TEST2_DOMAINID=1`, then the config parser will directly override the value of
 //! the field found in config to be `1`, since the fields in the environment variable name describe
 //! the path traversal to arrive at this field in the JSON config object.
 //!
@@ -57,7 +57,7 @@
 //! 1. The files matching `config/<env>/<config>.json`.
 //! 2. The order of configs in `CONFIG_FILES` with each sequential one
 //!    overwriting previous ones as appropriate.
-//! 3. Configuration env vars with the prefix `HYP_BASE` intended
+//! 3. Configuration env vars with the prefix `HYP` intended
 //!    to be shared by multiple agents in the same environment
 //!    E.g. `export HYP_CHAINS_ARBITRUM_DOMAINID=3000`
 //! 5. Arguments passed to the agent on the command line.

diff --git a/rust/hyperlane-core/src/utils.rs b/rust/hyperlane-core/src/utils.rs
@@ -243,7 +243,7 @@ macro_rules! unwrap_or_none_result {
     ($variable_name:ident, $e:expr $(, $else_e:expr)?) => {
         let Some($variable_name) = $e
         else {
-            $($else_e)?
+            $($else_e;)?
             return Ok(None);
         };
     };