Read simulation data from file (#2540)

Co-authored-by: rymnc <[email protected]>

crates/services/gas_price_service/simulation/Cargo.toml

@@ -7,10 +7,18 @@ edition = "2021"
 anyhow = "1.0" # this is floating and needs to match what is used by other packages
 fuel-core-gas-price-service = { path = ".." }
 fuel-core-services = { path = "../../../services", features = ["test-helpers"] }
-fuel-core-storage = { path = "../../../storage", features = ["std", "test-helpers"] }
+fuel-core-storage = { path = "../../../storage", features = [
+  "std",
+  "test-helpers",
+] }
 fuel-core-types = { version = "0.40.2", path = "../../../types", default-features = false }
 fuel-gas-price-algorithm = { path = "../../../fuel-gas-price-algorithm" }
 async-trait = "0.1.83"
 tokio = "1.41.1"
+serde = { version = "1.0.217", features = ["derive"] }
+serde-reflection = "0.5.0"
+csv = "1.3.1"
+clap = { version = "4.5.24", features = ["derive"] }
+itertools = "0.14.0"
 
 [workspace]

crates/services/gas_price_service/simulation/src/data_sources.rs

@@ -4,9 +4,12 @@ use fuel_core_gas_price_service::{
         l2_block_source::L2BlockSource,
         utils::BlockInfo,
     },
-    v1::da_source_service::{
-        service::DaBlockCostsSource,
-        DaBlockCosts,
+    v1::{
+        algorithm::SharedV1Algorithm,
+        da_source_service::{
+            service::DaBlockCostsSource,
+            DaBlockCosts,
+        },
     },
 };
 
@@ -18,30 +21,65 @@ use fuel_core_types::fuel_types::BlockHeight;
 
 pub struct SimulatedL2Blocks {
     recv: tokio::sync::mpsc::Receiver<BlockInfo>,
+    shared_algo: SharedV1Algorithm,
 }
 
 impl SimulatedL2Blocks {
-    pub fn new(recv: tokio::sync::mpsc::Receiver<BlockInfo>) -> Self {
-        Self { recv }
+    pub fn new(
+        recv: tokio::sync::mpsc::Receiver<BlockInfo>,
+        shared_algo: SharedV1Algorithm,
+    ) -> Self {
+        Self { recv, shared_algo }
     }
 
-    pub fn new_with_sender() -> (Self, tokio::sync::mpsc::Sender<BlockInfo>) {
+    pub fn new_with_sender(
+        shared_algo: SharedV1Algorithm,
+    ) -> (Self, tokio::sync::mpsc::Sender<BlockInfo>) {
         let (send, recv) = tokio::sync::mpsc::channel(16);
-        (Self { recv }, send)
+        (Self { recv, shared_algo }, send)
     }
 }
 
 #[async_trait]
 impl L2BlockSource for SimulatedL2Blocks {
     async fn get_l2_block(&mut self) -> GasPriceResult<BlockInfo> {
-        // TODO: do we want to modify these values to somehow reflect the previously chosen gas
-        //   price better? We might be able to do that by having a handle to the shared algo.
-
-        self.recv.recv().await.ok_or({
+        let block = self.recv.recv().await.ok_or({
             GasPriceError::CouldNotFetchL2Block {
                 source_error: anyhow::anyhow!("no more blocks; channel closed"),
             }
-        })
+        })?;
+
+        let BlockInfo::Block {
+            gas_used,
+            height,
+            block_gas_capacity,
+            block_bytes,
+            ..
+        } = block
+        else {
+            return Err(GasPriceError::CouldNotFetchL2Block {
+                source_error: anyhow::anyhow!("unexpected genesis block"),
+            });
+        };
+
+        let gas = gas_used;
+        let new_gas_price = self.shared_algo.next_gas_price();
+        let gas_price_factor = 1_150_000; // TODO: Read from CLI/config
+
+        let mut fee = (gas as u128).checked_mul(new_gas_price as u128).expect(
+            "Impossible to overflow because multiplication of two `u64` <= `u128`",
+        );
+        fee = fee.div_ceil(gas_price_factor as u128);
+
+        let block = BlockInfo::Block {
+            height,
+            gas_used,
+            block_gas_capacity,
+            block_bytes,
+            block_fees: fee.try_into().expect("overflow"),
+            gas_price: new_gas_price,
+        };
+        Ok(block)
     }
 }
 

crates/services/gas_price_service/simulation/src/main.rs

@@ -1,15 +1,72 @@
+use std::path::{
+    Path,
+    PathBuf,
+};
+
 use crate::service::{
     get_service_controller,
     ServiceController,
 };
+use clap::{
+    Parser,
+    Subcommand,
+};
+use csv::StringRecord;
 use fuel_core_gas_price_service::{
     common::utils::BlockInfo,
     v1::da_source_service::DaBlockCosts,
 };
+use itertools::Itertools;
+use serde_reflection::{
+    Samples,
+    Tracer,
+    TracerConfig,
+};
 
 pub mod data_sources;
 pub mod service;
 
+#[allow(dead_code)]
+#[derive(Debug, serde::Deserialize)]
+struct Record {
+    l1_block_number: u64,
+    l1_blob_fee_wei: u64,
+    l2_block_number: u64,
+    l2_gas_fullness: u64,
+    l2_gas_capacity: u64,
+    l2_byte_size: u64,
+    l2_byte_capacity: u64,
+}
+
+pub(crate) fn fields_of_struct_in_order<T>() -> Vec<String>
+where
+    T: serde::de::DeserializeOwned,
+{
+    let mut tracer = Tracer::new(TracerConfig::default());
+    let samples = Samples::new();
+    tracer.trace_type::<T>(&samples).unwrap();
+    let type_name = std::any::type_name::<T>().split("::").last().unwrap();
+    let registry = tracer.registry().unwrap();
+    let Some(serde_reflection::ContainerFormat::Struct(fields)) = registry.get(type_name)
+    else {
+        panic!("No fields?")
+    };
+
+    fields.iter().map(|f| f.name.clone()).collect()
+}
+
+#[derive(Subcommand, Debug)]
+enum DataSource {
+    /// Load data from a CSV file.
+    File {
+        #[arg(short, long)]
+        path: PathBuf,
+    },
+    /// Generate arbitrary data (not supported yet).
+    Generated,
+}
+
+#[derive(Debug)]
 pub struct Data {
     inner: Vec<(BlockInfo, Option<DaBlockCosts>)>,
 }
@@ -20,25 +77,80 @@ impl Data {
     }
 }
 
+impl From<&Record> for BlockInfo {
+    fn from(value: &Record) -> Self {
+        return BlockInfo::Block {
+            height: value.l2_block_number.try_into().unwrap(),
+            gas_used: value.l2_gas_fullness,
+            block_gas_capacity: value.l2_gas_capacity,
+            block_bytes: value.l2_byte_size,
+            block_fees: 0, // Will be overwritten by the simulation code
+            gas_price: 0,  // Will be overwritten by the simulation code
+        }
+    }
+}
+
 struct SimulationResults {}
 
-fn get_data() -> anyhow::Result<Data> {
-    // TODO
-    let data = Data { inner: vec![] };
+fn get_data(source: &DataSource) -> anyhow::Result<Data> {
+    let records = match source {
+        DataSource::File { path } => {
+            let headers = csv::StringRecord::from(fields_of_struct_in_order::<Record>());
+            let mut rdr = csv::ReaderBuilder::new()
+                .has_headers(true)
+                .from_path(path)
+                .unwrap();
+
+            let records: Result<Vec<Record>, _> = rdr
+                .records()
+                .map(|line_entry| {
+                    let line = line_entry?;
+                    line.deserialize(Some(&headers))
+                })
+                .collect();
+            records?
+        }
+        DataSource::Generated => unimplemented!(),
+    };
+
+    let mut data = vec![];
+    let groups = records.iter().chunk_by(|record| record.l1_block_number);
+    for (l1_block_number, block_records) in groups.into_iter() {
+        let blocks: Vec<_> = block_records.into_iter().collect();
+        let mut blocks_iter = blocks.iter().peekable();
+
+        while let Some(block_record) = blocks_iter.next() {
+            let l2_block: BlockInfo = (*block_record).into();
+            let da_block_costs = blocks_iter.peek().is_none().then(|| {
+                // TODO: Check if these are generated correctly.
+                let bundle_id: u32 = l1_block_number as u32; // Could be an arbitrary number, but we use L1 block number for convenience.
+                let bundle_size_bytes: u32 = 0; // Modify scrape tool to provide this
+                let range = blocks.first().unwrap().l2_block_number as u32
+                    ..=blocks.last().unwrap().l2_block_number as u32;
+                let blob_cost_wei = block_record.l1_blob_fee_wei as u128;
+
+                DaBlockCosts {
+                    bundle_id,
+                    l2_blocks: range,
+                    bundle_size_bytes,
+                    blob_cost_wei,
+                }
+            });
+            data.push((l2_block, da_block_costs));
+        }
+    }
+
+    let data = Data { inner: data };
     Ok(data)
 }
+
 async fn simulation(
     data: &Data,
     service_controller: &mut ServiceController,
 ) -> anyhow::Result<SimulationResults> {
     let res = SimulationResults {};
     for (block, maybe_costs) in data.get_iter() {
         service_controller.advance(block, maybe_costs).await?
-        // GET GAS PRICE
-
-        // MODIFY WITH GAS PRICE FACTOR
-
-        // RECORD LATEST VALUES
     }
     Ok(res)
 }
@@ -48,9 +160,16 @@ fn display_results(_results: SimulationResults) -> anyhow::Result<()> {
     Ok(())
 }
 
+#[derive(Parser, Debug)]
+struct Args {
+    #[command(subcommand)]
+    data_source: DataSource,
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    let data = get_data()?;
+    let args = Args::parse();
+    let data = get_data(&args.data_source)?;
     let mut service_controller = get_service_controller().await;
     let results = simulation(&data, &mut service_controller).await?;
     display_results(results)?;

crates/services/gas_price_service/simulation/src/service.rs

@@ -73,7 +73,7 @@ fn read_metadata_from_file(_metadata_path: &str) -> V1Metadata {
     // TODO: read from file and/or CLI
     V1Metadata {
         new_scaled_exec_price: 0,
-        l2_block_height: 0,
+        l2_block_height: 999, // TODO: Use first L2 block height from the CSV
         new_scaled_da_gas_price: 0,
         gas_price_factor: NonZero::new(100).unwrap(),
         total_da_rewards_excess: 0,
@@ -139,7 +139,8 @@ pub async fn get_service_controller() -> ServiceController {
     let algo = algorithm_updater.algorithm();
     let shared_algo = SharedV1Algorithm::new_with_algorithm(algo);
 
-    let (l2_block_source, l2_block_sender) = SimulatedL2Blocks::new_with_sender();
+    let (l2_block_source, l2_block_sender) =
+        SimulatedL2Blocks::new_with_sender(shared_algo.clone());
     let (da_block_source, da_costs_sender) = SimulatedDACosts::new_with_sender();
 
     let poll_interval = poll_interval();