[forge] adjust thresholds for long-running realistic max, and haproxy…

… (cpu) (#10523)

### Description

Adjusting thresholds based on behavior after landing #10315

testsuite/forge-cli/src/main.rs

@@ -934,7 +934,7 @@ fn realistic_env_workload_sweep_test() -> ForgeConfig {
         criteria: [
             (3700, 0.35, 0.5, 0.8, 0.65),
             (2800, 0.35, 0.5, 1.2, 1.3),
-            (1800, 0.35, 0.5, 1.5, 2.7),
+            (1800, 0.35, 2.0, 1.5, 2.7),
             (950, 0.35, 0.65, 1.5, 2.9),
             // (150, 0.5, 1.0, 1.5, 0.65),
         ]
@@ -1693,23 +1693,26 @@ fn realistic_env_max_load_test(
                     (duration.as_secs() / 10).max(60),
                 )
                 .add_system_metrics_threshold(SystemMetricsThreshold::new(
-                    // Check that we don't use more than 14 CPU cores for 30% of the time.
-                    MetricsThreshold::new(14.0, max_cpu_threshold),
+                    // Check that we don't use more than 16 CPU cores for 30% of the time.
+                    MetricsThreshold::new(16.0, max_cpu_threshold),
                     // Check that we don't use more than 10 GB of memory for 30% of the time.
                     MetricsThreshold::new_gb(10.0, 30),
                 ))
                 .add_latency_threshold(3.4, LatencyType::P50)
                 .add_latency_threshold(4.5, LatencyType::P90)
                 .add_latency_breakdown_threshold(LatencyBreakdownThreshold::new_strict(vec![
                     (LatencyBreakdownSlice::QsBatchToPos, 0.35),
+                    // only reaches close to threshold during epoch change
                     (
                         LatencyBreakdownSlice::QsPosToProposal,
-                        if ha_proxy { 0.6 } else { 0.5 },
+                        if ha_proxy { 0.7 } else { 0.6 },
                     ),
-                    (LatencyBreakdownSlice::ConsensusProposalToOrdered, 0.8),
+                    // can be adjusted down if less backpressure
+                    (LatencyBreakdownSlice::ConsensusProposalToOrdered, 0.85),
+                    // can be adjusted down if less backpressure
                     (
                         LatencyBreakdownSlice::ConsensusOrderedToCommit,
-                        if ha_proxy { 1.2 } else { 0.65 },
+                        if ha_proxy { 1.3 } else { 0.75 },
                     ),
                 ]))
                 .add_chain_progress(StateProgressThreshold {