From 20dafa80d0b4d043b50f0e21f55a776968079a8f Mon Sep 17 00:00:00 2001 From: Nasr Date: Mon, 27 Jan 2025 12:26:54 +0700 Subject: [PATCH 1/3] feat(torii-indexer): add option for strict model reader block --- crates/torii/cli/src/options.rs | 11 +++++++++++ crates/torii/indexer/src/processors/mod.rs | 1 + .../torii/indexer/src/processors/register_event.rs | 10 +++++++--- .../torii/indexer/src/processors/register_model.rs | 10 +++++++--- crates/torii/indexer/src/processors/upgrade_event.rs | 12 ++++++++---- crates/torii/indexer/src/processors/upgrade_model.rs | 12 ++++++++---- 6 files changed, 42 insertions(+), 14 deletions(-) diff --git a/crates/torii/cli/src/options.rs b/crates/torii/cli/src/options.rs index c89d3a2475..266eec7cd9 100644 --- a/crates/torii/cli/src/options.rs +++ b/crates/torii/cli/src/options.rs @@ -90,6 +90,16 @@ impl Default for RelayOptions { #[derive(Debug, clap::Args, Clone, Serialize, Deserialize, PartialEq)] #[command(next_help_heading = "Indexing options")] pub struct IndexingOptions { + /// Whether or not to read models from the block number they were registered in. + /// If false, models will be read from the latest block. + #[arg( + long = "indexing.strict_model_reader", + default_value_t = false, + help = "Whether or not to read models from the block number they were registered in." + )] + #[serde(default)] + pub strict_model_reader: bool, + /// Chunk size of the events page when indexing using events #[arg(long = "indexing.events_chunk_size", default_value_t = DEFAULT_EVENTS_CHUNK_SIZE, help = "Chunk size of the events page to fetch from the sequencer.")] #[serde(default = "default_events_chunk_size")] @@ -175,6 +185,7 @@ pub struct IndexingOptions { impl Default for IndexingOptions { fn default() -> Self { Self { + strict_model_reader: false, events_chunk_size: DEFAULT_EVENTS_CHUNK_SIZE, blocks_chunk_size: DEFAULT_BLOCKS_CHUNK_SIZE, pending: true, diff --git a/crates/torii/indexer/src/processors/mod.rs b/crates/torii/indexer/src/processors/mod.rs index 420dd798a7..04f6c38150 100644 --- a/crates/torii/indexer/src/processors/mod.rs +++ b/crates/torii/indexer/src/processors/mod.rs @@ -30,6 +30,7 @@ pub mod upgrade_model; pub struct EventProcessorConfig { pub historical_events: HashSet, pub namespaces: HashSet, + pub strict_model_reader: bool, } impl EventProcessorConfig { diff --git a/crates/torii/indexer/src/processors/register_event.rs b/crates/torii/indexer/src/processors/register_event.rs index df4a536625..d41b820db3 100644 --- a/crates/torii/indexer/src/processors/register_event.rs +++ b/crates/torii/indexer/src/processors/register_event.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use dojo_world::contracts::abigen::world::Event as WorldEvent; use dojo_world::contracts::model::ModelReader; use dojo_world::contracts::world::WorldContractReader; -use starknet::core::types::Event; +use starknet::core::types::{BlockId, Event}; use starknet::providers::Provider; use torii_sqlite::Sql; use tracing::{debug, info}; @@ -47,7 +47,7 @@ where &self, world: &WorldContractReader

, db: &mut Sql, - _block_number: u64, + block_number: u64, block_timestamp: u64, _event_id: &str, event: &Event, @@ -79,7 +79,11 @@ where // Called model here by language, but it's an event. Torii rework will make clear // distinction. - let model = world.model_reader(&namespace, &name).await?; + let model = if config.strict_model_reader { + world.model_reader_with_block(&namespace, &name, BlockId::Number(block_number)).await? + } else { + world.model_reader(&namespace, &name).await? + }; let schema = model.schema().await?; let layout = model.layout().await?; diff --git a/crates/torii/indexer/src/processors/register_model.rs b/crates/torii/indexer/src/processors/register_model.rs index dae000613a..bd96b7bb8a 100644 --- a/crates/torii/indexer/src/processors/register_model.rs +++ b/crates/torii/indexer/src/processors/register_model.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use dojo_world::contracts::abigen::world::Event as WorldEvent; use dojo_world::contracts::model::ModelReader; use dojo_world::contracts::world::WorldContractReader; -use starknet::core::types::Event; +use starknet::core::types::{BlockId, Event}; use starknet::providers::Provider; use torii_sqlite::Sql; use tracing::{debug, info}; @@ -47,7 +47,7 @@ where &self, world: &WorldContractReader

, db: &mut Sql, - _block_number: u64, + block_number: u64, block_timestamp: u64, _event_id: &str, event: &Event, @@ -77,7 +77,11 @@ where return Ok(()); } - let model = world.model_reader(&namespace, &name).await?; + let model = if config.strict_model_reader { + world.model_reader_with_block(&namespace, &name, BlockId::Number(block_number)).await? + } else { + world.model_reader(&namespace, &name).await? + }; let schema = model.schema().await?; let layout = model.layout().await?; diff --git a/crates/torii/indexer/src/processors/upgrade_event.rs b/crates/torii/indexer/src/processors/upgrade_event.rs index 3b9597b3ad..babe19959b 100644 --- a/crates/torii/indexer/src/processors/upgrade_event.rs +++ b/crates/torii/indexer/src/processors/upgrade_event.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use dojo_world::contracts::abigen::world::Event as WorldEvent; use dojo_world::contracts::model::ModelReader; use dojo_world::contracts::world::WorldContractReader; -use starknet::core::types::Event; +use starknet::core::types::{BlockId, Event}; use starknet::providers::Provider; use torii_sqlite::Sql; use tracing::{debug, info}; @@ -47,11 +47,11 @@ where &self, world: &WorldContractReader

, db: &mut Sql, - _block_number: u64, + block_number: u64, block_timestamp: u64, _event_id: &str, event: &Event, - _config: &EventProcessorConfig, + config: &EventProcessorConfig, ) -> Result<(), Error> { // Torii version is coupled to the world version, so we can expect the event to be well // formed. @@ -88,7 +88,11 @@ where let namespace = model.namespace; let prev_schema = model.schema; - let model = world.model_reader(&namespace, &name).await?; + let model = if config.strict_model_reader { + world.model_reader_with_block(&namespace, &name, BlockId::Number(block_number)).await? + } else { + world.model_reader(&namespace, &name).await? + }; let new_schema = model.schema().await?; let schema_diff = new_schema.diff(&prev_schema); // No changes to the schema. This can happen if torii is re-run with a fresh database. diff --git a/crates/torii/indexer/src/processors/upgrade_model.rs b/crates/torii/indexer/src/processors/upgrade_model.rs index fe46304b72..2998000b58 100644 --- a/crates/torii/indexer/src/processors/upgrade_model.rs +++ b/crates/torii/indexer/src/processors/upgrade_model.rs @@ -5,7 +5,7 @@ use async_trait::async_trait; use dojo_world::contracts::abigen::world::Event as WorldEvent; use dojo_world::contracts::model::ModelReader; use dojo_world::contracts::world::WorldContractReader; -use starknet::core::types::Event; +use starknet::core::types::{BlockId, Event}; use starknet::providers::Provider; use torii_sqlite::Sql; use tracing::{debug, info}; @@ -47,11 +47,11 @@ where &self, world: &WorldContractReader

, db: &mut Sql, - _block_number: u64, + block_number: u64, block_timestamp: u64, _event_id: &str, event: &Event, - _config: &EventProcessorConfig, + config: &EventProcessorConfig, ) -> Result<(), Error> { // Torii version is coupled to the world version, so we can expect the event to be well // formed. @@ -86,7 +86,11 @@ where let namespace = model.namespace; let prev_schema = model.schema; - let model = world.model_reader(&namespace, &name).await?; + let model = if config.strict_model_reader { + world.model_reader_with_block(&namespace, &name, BlockId::Number(block_number)).await? + } else { + world.model_reader(&namespace, &name).await? + }; let new_schema = model.schema().await?; let schema_diff = new_schema.diff(&prev_schema); // No changes to the schema. This can happen if torii is re-run with a fresh database. From 4624009af3ef2e4e721bdc9a03100533ff8d9dde Mon Sep 17 00:00:00 2001 From: Nasr Date: Mon, 27 Jan 2025 12:47:51 +0700 Subject: [PATCH 2/3] event processor config --- crates/torii/runner/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/torii/runner/src/lib.rs b/crates/torii/runner/src/lib.rs index 8e0fae9b36..ed54013a84 100644 --- a/crates/torii/runner/src/lib.rs +++ b/crates/torii/runner/src/lib.rs @@ -177,6 +177,7 @@ impl Runner { polling_interval: Duration::from_millis(self.args.indexing.polling_interval), flags, event_processor_config: EventProcessorConfig { + strict_model_reader: self.args.indexing.strict_model_reader, historical_events: self.args.events.historical.into_iter().collect(), namespaces: self.args.indexing.namespaces.into_iter().collect(), }, From 06b0791c029490d66113e59b0f49bf31e348d4b5 Mon Sep 17 00:00:00 2001 From: Nasr Date: Wed, 29 Jan 2025 12:45:56 +0700 Subject: [PATCH 3/3] rename --- crates/torii/cli/src/options.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/torii/cli/src/options.rs b/crates/torii/cli/src/options.rs index 266eec7cd9..0746c9c952 100644 --- a/crates/torii/cli/src/options.rs +++ b/crates/torii/cli/src/options.rs @@ -90,16 +90,6 @@ impl Default for RelayOptions { #[derive(Debug, clap::Args, Clone, Serialize, Deserialize, PartialEq)] #[command(next_help_heading = "Indexing options")] pub struct IndexingOptions { - /// Whether or not to read models from the block number they were registered in. - /// If false, models will be read from the latest block. - #[arg( - long = "indexing.strict_model_reader", - default_value_t = false, - help = "Whether or not to read models from the block number they were registered in." - )] - #[serde(default)] - pub strict_model_reader: bool, - /// Chunk size of the events page when indexing using events #[arg(long = "indexing.events_chunk_size", default_value_t = DEFAULT_EVENTS_CHUNK_SIZE, help = "Chunk size of the events page to fetch from the sequencer.")] #[serde(default = "default_events_chunk_size")] @@ -180,12 +170,21 @@ pub struct IndexingOptions { )] #[serde(default)] pub world_block: u64, + + /// Whether or not to read models from the block number they were registered in. + /// If false, models will be read from the latest block. + #[arg( + long = "indexing.strict_model_reader", + default_value_t = false, + help = "Whether or not to read models from the block number they were registered in." + )] + #[serde(default)] + pub strict_model_reader: bool, } impl Default for IndexingOptions { fn default() -> Self { Self { - strict_model_reader: false, events_chunk_size: DEFAULT_EVENTS_CHUNK_SIZE, blocks_chunk_size: DEFAULT_BLOCKS_CHUNK_SIZE, pending: true, @@ -195,6 +194,7 @@ impl Default for IndexingOptions { max_concurrent_tasks: DEFAULT_MAX_CONCURRENT_TASKS, namespaces: vec![], world_block: 0, + strict_model_reader: false, } } } @@ -237,6 +237,10 @@ impl IndexingOptions { if self.world_block == 0 { self.world_block = other.world_block; } + + if !self.strict_model_reader { + self.strict_model_reader = other.strict_model_reader; + } } } }