From ec54f2cef78cd9e02d54140683412e1c72618564 Mon Sep 17 00:00:00 2001 From: -f Date: Wed, 15 Jan 2025 14:31:21 +0530 Subject: [PATCH 1/6] mv update_metrics to checking for new range --- .../cursors/sequence_aware/forward.rs | 19 +++++-------------- .../src/invariants/termination_invariants.rs | 13 +++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/rust/main/hyperlane-base/src/contract_sync/cursors/sequence_aware/forward.rs b/rust/main/hyperlane-base/src/contract_sync/cursors/sequence_aware/forward.rs index 79917dd663..e1211334b2 100644 --- a/rust/main/hyperlane-base/src/contract_sync/cursors/sequence_aware/forward.rs +++ b/rust/main/hyperlane-base/src/contract_sync/cursors/sequence_aware/forward.rs @@ -101,17 +101,6 @@ impl ForwardSequenceAwareS } } - /// Get target sequence or return 0 if request failed - pub async fn target_sequence(&self) -> u32 { - let (count, _) = self - .latest_sequence_querier - .latest_sequence_count_and_tip() - .await - .ok() - .unwrap_or((None, 0)); - count.unwrap_or(0).saturating_sub(1) - } - /// Get the last indexed sequence or 0 if no logs have been indexed yet. pub fn last_sequence(&self) -> u32 { self.last_indexed_snapshot.sequence.unwrap_or(0) @@ -134,6 +123,10 @@ impl ForwardSequenceAwareS return Ok(None); }; + // for updating metrics even if there's no indexable events available + let max_sequence = onchain_sequence_count.saturating_sub(1) as i64; + self.update_metrics(max_sequence).await; + let current_sequence = self.current_indexing_snapshot.sequence; let range = match current_sequence.cmp(&onchain_sequence_count) { Ordering::Equal => { @@ -432,7 +425,7 @@ impl ForwardSequenceAwareS } // Updates the cursor metrics. - async fn update_metrics(&self) { + async fn update_metrics(&self, max_sequence: i64) { let mut labels = hashmap! { "event_type" => T::name(), "chain" => self.domain.name(), @@ -452,7 +445,6 @@ impl ForwardSequenceAwareS .set(sequence as i64); labels.remove("cursor_type"); - let max_sequence = self.target_sequence().await as i64; self.metrics .cursor_max_sequence .with(&labels) @@ -501,7 +493,6 @@ impl ContractSyncCursor logs: Vec<(Indexed, LogMeta)>, range: RangeInclusive, ) -> Result<()> { - self.update_metrics().await; // Remove any sequence duplicates, filter out any logs preceding our current snapshot, // and sort in ascending order. let logs = indexed_to_sequence_indexed_array(logs)? diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 6e69fd5e63..948e20dbc2 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -150,6 +150,19 @@ pub fn termination_invariants_met( return Ok(false); } + let merkle_tree_max_sequence = fetch_metric( + RELAYER_METRICS_PORT, + "hyperlane_cursor_max_sequence", + &hashmap! {"event_type" => "merkle_tree_insertion"}, + )?; + // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions + // LHS: sum(merkle_tree_max_sequence) + len(merkle_tree_max_sequence) (each is index so we add 1 to each) + // RHS: total_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) + assert_eq!( + merkle_tree_max_sequence.iter().sum::() + merkle_tree_max_sequence.len() as u32, + total_messages_expected + (config.kathy_messages as u32 / 4) * 2 + ); + if let Some((solana_cli_tools_path, solana_config_path)) = solana_cli_tools_path.zip(solana_config_path) { From e0d49f546a604ac8bebee745d9e05a82fc3f3a6f Mon Sep 17 00:00:00 2001 From: -f Date: Wed, 15 Jan 2025 14:40:59 +0530 Subject: [PATCH 2/6] rename --- .../main/agents/relayer/src/merkle_tree/processor.rs | 8 ++++---- rust/main/hyperlane-base/src/metrics/core.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rust/main/agents/relayer/src/merkle_tree/processor.rs b/rust/main/agents/relayer/src/merkle_tree/processor.rs index e12a7fc343..5523941d69 100644 --- a/rust/main/agents/relayer/src/merkle_tree/processor.rs +++ b/rust/main/agents/relayer/src/merkle_tree/processor.rs @@ -76,7 +76,7 @@ impl MerkleTreeProcessor { // Update the metrics // we assume that leaves are inserted in order so this will be monotonically increasing self.metrics - .latest_leaf_index_gauge + .latest_tree_insertion_index_gauge .set(insertion.index() as i64); Some(insertion) } else { @@ -89,14 +89,14 @@ impl MerkleTreeProcessor { #[derive(Debug)] pub struct MerkleTreeProcessorMetrics { - latest_leaf_index_gauge: IntGauge, + latest_tree_insertion_index_gauge: IntGauge, } impl MerkleTreeProcessorMetrics { pub fn new(metrics: &CoreMetrics, origin: &HyperlaneDomain) -> Self { Self { - latest_leaf_index_gauge: metrics - .latest_leaf_index() + latest_tree_insertion_index_gauge: metrics + .latest_tree_insertion_index() .with_label_values(&[origin.name()]), } } diff --git a/rust/main/hyperlane-base/src/metrics/core.rs b/rust/main/hyperlane-base/src/metrics/core.rs index d3cbaf3db4..44fd6d4b09 100644 --- a/rust/main/hyperlane-base/src/metrics/core.rs +++ b/rust/main/hyperlane-base/src/metrics/core.rs @@ -38,7 +38,7 @@ pub struct CoreMetrics { span_counts: IntCounterVec, span_events: IntCounterVec, last_known_message_nonce: IntGaugeVec, - latest_leaf_index: IntGaugeVec, + latest_tree_insertion_index: IntGaugeVec, submitter_queue_length: IntGaugeVec, operations_processed_count: IntCounterVec, @@ -113,9 +113,9 @@ impl CoreMetrics { registry )?; - let latest_leaf_index = register_int_gauge_vec_with_registry!( + let latest_tree_insertion_index = register_int_gauge_vec_with_registry!( opts!( - namespaced!("latest_leaf_index"), + namespaced!("latest_tree_insertion_index"), "Latest leaf index inserted into the merkle tree", const_labels_ref ), @@ -188,7 +188,7 @@ impl CoreMetrics { span_counts, span_events, last_known_message_nonce, - latest_leaf_index, + latest_tree_insertion_index, submitter_queue_length, @@ -325,8 +325,8 @@ impl CoreMetrics { /// /// Labels: /// - `origin`: Origin chain the leaf index is being tracked at. - pub fn latest_leaf_index(&self) -> IntGaugeVec { - self.latest_leaf_index.clone() + pub fn latest_tree_insertion_index(&self) -> IntGaugeVec { + self.latest_tree_insertion_index.clone() } /// Latest message nonce in the validator. From 26fc9ebe30e793883a43ca7db5098b80778e3d77 Mon Sep 17 00:00:00 2001 From: -f Date: Wed, 15 Jan 2025 20:45:11 +0530 Subject: [PATCH 3/6] update invariant --- .../src/invariants/termination_invariants.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 948e20dbc2..aed8213068 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -157,10 +157,14 @@ pub fn termination_invariants_met( )?; // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions // LHS: sum(merkle_tree_max_sequence) + len(merkle_tree_max_sequence) (each is index so we add 1 to each) - // RHS: total_messages_expected + (config.kathy_messages as u32 / 4) * 2 (double insertions) + // RHS: total_messages_expected + non_matching_igp_messages + (config.kathy_messages as u32 / 4) * 2 (double insertions) + let non_zero_sequence_count = + merkle_tree_max_sequence.iter().filter(|&&x| x > 0).count() as u32; assert_eq!( - merkle_tree_max_sequence.iter().sum::() + merkle_tree_max_sequence.len() as u32, - total_messages_expected + (config.kathy_messages as u32 / 4) * 2 + merkle_tree_max_sequence.iter().sum::() + non_zero_sequence_count, + total_messages_expected + + SOL_MESSAGES_WITH_NON_MATCHING_IGP + + (config.kathy_messages as u32 / 4) * 2 ); if let Some((solana_cli_tools_path, solana_config_path)) = From b8816c0cf8805fa920e85b45d71331297b00b4e2 Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 20 Jan 2025 21:18:23 +0530 Subject: [PATCH 4/6] change sol_messages_with_non_matching_igp --- .../utils/run-locally/src/invariants/termination_invariants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index aed8213068..8328c36b04 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -163,7 +163,7 @@ pub fn termination_invariants_met( assert_eq!( merkle_tree_max_sequence.iter().sum::() + non_zero_sequence_count, total_messages_expected - + SOL_MESSAGES_WITH_NON_MATCHING_IGP + + sol_messages_with_non_matching_igp + (config.kathy_messages as u32 / 4) * 2 ); From 0054186b1ceefb752e128cf2f2f2506d6d5af2e2 Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 20 Jan 2025 21:28:01 +0530 Subject: [PATCH 5/6] trevor doesn't like && --- .../run-locally/src/invariants/termination_invariants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index 8328c36b04..d24592a05e 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -158,8 +158,8 @@ pub fn termination_invariants_met( // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions // LHS: sum(merkle_tree_max_sequence) + len(merkle_tree_max_sequence) (each is index so we add 1 to each) // RHS: total_messages_expected + non_matching_igp_messages + (config.kathy_messages as u32 / 4) * 2 (double insertions) - let non_zero_sequence_count = - merkle_tree_max_sequence.iter().filter(|&&x| x > 0).count() as u32; + let non_zero_sequence_count = + merkle_tree_max_sequence.iter().filter(|&x| *x > 0).count() as u32; assert_eq!( merkle_tree_max_sequence.iter().sum::() + non_zero_sequence_count, total_messages_expected From a8d63c2babb898dd25ce6da5ee79ab65a2df780c Mon Sep 17 00:00:00 2001 From: -f Date: Mon, 20 Jan 2025 21:28:14 +0530 Subject: [PATCH 6/6] fmt --- .../utils/run-locally/src/invariants/termination_invariants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs index d24592a05e..50450210c6 100644 --- a/rust/main/utils/run-locally/src/invariants/termination_invariants.rs +++ b/rust/main/utils/run-locally/src/invariants/termination_invariants.rs @@ -158,7 +158,7 @@ pub fn termination_invariants_met( // check for each origin that the highest tree index seen by the syncer == # of messages sent + # of double insertions // LHS: sum(merkle_tree_max_sequence) + len(merkle_tree_max_sequence) (each is index so we add 1 to each) // RHS: total_messages_expected + non_matching_igp_messages + (config.kathy_messages as u32 / 4) * 2 (double insertions) - let non_zero_sequence_count = + let non_zero_sequence_count = merkle_tree_max_sequence.iter().filter(|&x| *x > 0).count() as u32; assert_eq!( merkle_tree_max_sequence.iter().sum::() + non_zero_sequence_count,