fix: e2e working with aggregation 2/2 of message_id and merkle_root

rust/agents/relayer/src/merkle_tree/builder.rs

@@ -14,6 +14,7 @@ use crate::prover::{Prover, ProverError};
 /// Struct to sync prover.
 #[derive(Debug)]
 pub struct MerkleTreeBuilder {
+    #[allow(dead_code)]
     db: HyperlaneRocksDB,
     prover: Prover,
     incremental: IncrementalMerkle,
@@ -75,23 +76,16 @@ impl MerkleTreeBuilder {
     #[instrument(err, skip(self), level="debug", fields(prover_latest_index=self.count()-1))]
     pub fn get_proof(
         &self,
-        message_nonce: u32,
+        leaf_index: u32,
         root_index: u32,
     ) -> Result<Option<Proof>, MerkleTreeBuilderError> {
-        self.db
-            .retrieve_message_id_by_nonce(&message_nonce)?
-            .and_then(|message_id| {
-                self.db
-                    .retrieve_merkle_leaf_index_by_message_id(&message_id)
-                    .ok()
-                    .flatten()
-            })
-            .map(|leaf_index| {
-                self.prover
-                    .prove_against_previous(leaf_index as usize, root_index as usize)
-            })
-            .transpose()
-            .map_err(Into::into)
+        match self
+            .prover
+            .prove_against_previous(leaf_index as usize, root_index as usize)
+        {
+            Ok(proof) => Ok(Some(proof)),
+            Err(prover_err) => Err(MerkleTreeBuilderError::from(prover_err)),
+        }
     }
 
     pub fn count(&self) -> u32 {

rust/agents/relayer/src/msg/metadata/aggregation.rs

@@ -6,7 +6,7 @@ use derive_new::new;
 use eyre::Context;
 use tracing::{info, instrument};
 
-use hyperlane_core::{HyperlaneMessage, InterchainSecurityModule, H256, U256};
+use hyperlane_core::{HyperlaneMessage, InterchainSecurityModule, ModuleType, H256, U256};
 
 use super::{BaseMetadataBuilder, MetadataBuilder};
 
@@ -90,15 +90,15 @@ impl AggregationIsmMetadataBuilder {
         sub_modules: Vec<IsmAndMetadata>,
         message: &HyperlaneMessage,
         threshold: usize,
+        invalid_isms: Vec<(ModuleType, H256)>,
     ) -> Option<Vec<SubModuleMetadata>> {
         let gas_cost_results: Vec<_> = join_all(
             sub_modules
                 .iter()
                 .map(|module| module.ism.dry_run_verify(message, &(module.meta.metadata))),
         )
         .await;
-
-        // Filter out the ISMs without a gas cost estimate
+        // Filter out the ISMs with a gas cost estimate
         let metas_and_gas: Vec<_> = sub_modules
             .into_iter()
             .zip(gas_cost_results.into_iter())
@@ -107,7 +107,14 @@ impl AggregationIsmMetadataBuilder {
 
         let metas_and_gas_count = metas_and_gas.len();
         if metas_and_gas_count < threshold {
-            info!("Could not fetch all metadata: Found {metas_and_gas_count} of the {threshold} required ISM metadata pieces");
+            info!("Could not fetch all metadata: Found {metas_and_gas_count} of the {threshold} required ISM metadata pieces for message_id {message_id}", metas_and_gas_count=metas_and_gas_count, threshold=threshold, message_id=message.id());
+            for (module_type, ism_address) in invalid_isms {
+                info!(
+                    "Invalid ISM: {module_type} at {ism_address}",
+                    module_type = module_type,
+                    ism_address = ism_address
+                );
+            }
             return None;
         }
         Some(Self::n_cheapest_metas(metas_and_gas, threshold))
@@ -140,21 +147,34 @@ impl MetadataBuilder for AggregationIsmMetadataBuilder {
                 .map(|ism_address| self.base.build_ism(*ism_address)),
         )
         .await;
-
-        let filtered_sub_module_metas = metas
+        let (valid_sub_modules, invalid_sub_modules): (Vec<_>, Vec<_>) = metas
             .into_iter()
             .enumerate()
             .zip(sub_modules.into_iter())
-            .filter_map(|((index, meta_result), sub_module_result)| {
-                match (meta_result, sub_module_result) {
-                    (Ok(Some(meta)), Ok(ism)) => Some(IsmAndMetadata::new(ism, index, meta)),
-                    _ => None,
-                }
+            .partition(|&((_, ref meta_result), ref sub_module_result)| {
+                matches!((meta_result, sub_module_result), (Ok(Some(_)), Ok(_)))
+            });
+        let valid_sub_module_metas: Vec<_> = valid_sub_modules
+            .into_iter()
+            .map(|((index, meta_result), sub_module_result)| {
+                IsmAndMetadata::new(
+                    sub_module_result.unwrap(),
+                    index,
+                    meta_result.unwrap().unwrap(),
+                )
             })
             .collect();
 
+        let mut invalid_isms: Vec<(ModuleType, H256)> = Vec::new();
+        // get the address and module_type for each invalid sub module (for logging in cheapest_valid_metas)
+        for entry in &invalid_sub_modules {
+            let (_, ism_result) = entry;
+            if let Ok(ism_box) = ism_result {
+                invalid_isms.push((ism_box.module_type().await.unwrap(), ism_box.address()));
+            }
+        }
         let maybe_aggregation_metadata =
-            Self::cheapest_valid_metas(filtered_sub_module_metas, message, threshold)
+            Self::cheapest_valid_metas(valid_sub_module_metas, message, threshold, invalid_isms)
                 .await
                 .map(|mut metas| Self::format_metadata(&mut metas, ism_addresses.len()));
         Ok(maybe_aggregation_metadata)

rust/agents/relayer/src/msg/metadata/base.rs

@@ -116,12 +116,16 @@ impl BaseMetadataBuilder {
         }
     }
 
-    pub async fn get_proof(&self, nonce: u32, checkpoint: Checkpoint) -> Result<Option<Proof>> {
+    pub async fn get_proof(
+        &self,
+        leaf_index: u32,
+        checkpoint: Checkpoint,
+    ) -> Result<Option<Proof>> {
         const CTX: &str = "When fetching message proof";
         let proof = self.origin_prover_sync
             .read()
             .await
-            .get_proof(nonce, checkpoint.index)
+            .get_proof(leaf_index, checkpoint.index)
             .context(CTX)?
             .and_then(|proof| {
                 // checkpoint may be fraudulent if the root does not

rust/agents/validator/src/submit.rs

@@ -253,7 +253,6 @@ impl ValidatorSubmitter {
                 );
                 continue;
             }
-
             let signed_checkpoint = self.signer.sign(queued_checkpoint).await?;
             self.checkpoint_syncer
                 .write_checkpoint(&signed_checkpoint)

rust/hyperlane-base/src/types/multisig.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 
 use derive_new::new;
 use eyre::Result;
-use tracing::{debug, instrument, trace};
+use tracing::{debug, instrument};
 
 use hyperlane_core::{MultisigSignedCheckpoint, SignedCheckpointWithMessageId, H160, H256};
 
@@ -46,7 +46,7 @@ impl MultisigCheckpointSyncer {
                 // Gracefully handle errors getting the latest_index
                 match checkpoint_syncer.latest_index().await {
                     Ok(Some(index)) => {
-                        trace!(?address, ?index, "Validator returned latest index");
+                        debug!(?address, ?index, "Validator returned latest index");
                         latest_indices.push(index);
                     }
                     err => {

typescript/infra/config/environments/test/aggregationIsm.ts

@@ -9,6 +9,6 @@ export const aggregationIsm = (validatorKey: string): AggregationIsmConfig => {
       merkleRootMultisig(validatorKey),
       messageIdMultisig(validatorKey),
     ],
-    threshold: 1,
+    threshold: 2,
   };
 };