diff --git a/Cargo.lock b/Cargo.lock index 0deaae5ed3581..84140a403fca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14053,6 +14053,30 @@ dependencies = [ "wiremock", ] +[[package]] +name = "sui-indexer-alt-e2e-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "datatest-stable", + "diesel", + "diesel-async", + "msim", + "prometheus", + "reqwest 0.12.5", + "serde_json", + "sui-indexer-alt", + "sui-indexer-alt-framework", + "sui-indexer-alt-jsonrpc", + "sui-pg-db", + "sui-transactional-test-runner", + "telemetry-subscribers", + "tokio", + "tokio-util 0.7.10", + "url", +] + [[package]] name = "sui-indexer-alt-framework" version = "1.42.0" diff --git a/Cargo.toml b/Cargo.toml index 7f25c4bc16a7e..fb2953055e5da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,6 +116,7 @@ members = [ "crates/sui-http", "crates/sui-indexer", "crates/sui-indexer-alt", + "crates/sui-indexer-alt-e2e-tests", "crates/sui-indexer-alt-framework", "crates/sui-indexer-alt-jsonrpc", "crates/sui-indexer-alt-metrics", @@ -640,6 +641,7 @@ sui-graphql-rpc-headers = { path = "crates/sui-graphql-rpc-headers" } sui-genesis-builder = { path = "crates/sui-genesis-builder" } sui-http = { path = "crates/sui-http" } sui-indexer = { path = "crates/sui-indexer" } +sui-indexer-alt = { path = "crates/sui-indexer-alt" } sui-indexer-alt-framework = { path = "crates/sui-indexer-alt-framework" } sui-indexer-alt-jsonrpc = { path = "crates/sui-indexer-alt-jsonrpc" } sui-indexer-alt-metrics = { path = "crates/sui-indexer-alt-metrics" } diff --git a/crates/sui-indexer-alt-e2e-tests/Cargo.toml b/crates/sui-indexer-alt-e2e-tests/Cargo.toml new file mode 100644 index 0000000000000..ec2d653113c00 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "sui-indexer-alt-e2e-tests" +version = "0.1.0" +authors = ["Mysten Labs "] +license = "Apache-2.0" +publish = false +edition = "2021" + +[lints] +workspace = true + +[[test]] +name = "tests" +harness = false + +[dependencies] + +[target.'cfg(msim)'.dependencies] +msim.workspace = true + +[dev-dependencies] +anyhow.workspace = true +async-trait.workspace = true +datatest-stable.workspace = true +diesel = { workspace = true, features = ["chrono"] } +diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] } +prometheus.workspace = true +reqwest.workspace = true +serde_json.workspace = true +telemetry-subscribers.workspace = true +tokio.workspace = true +tokio-util.workspace = true +url.workspace = true + +sui-indexer-alt.workspace = true +sui-indexer-alt-framework.workspace = true +sui-indexer-alt-jsonrpc.workspace = true +sui-pg-db.workspace = true +sui-transactional-test-runner.workspace = true diff --git a/crates/sui-indexer-alt-e2e-tests/src/lib.rs b/crates/sui-indexer-alt-e2e-tests/src/lib.rs new file mode 100644 index 0000000000000..07fe81c224d7d --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/src/lib.rs @@ -0,0 +1,6 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#![forbid(unsafe_code)] + +// Empty src/lib.rs to get rusty-tags working. diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.exp new file mode 100644 index 0000000000000..1229a6d938c78 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.exp @@ -0,0 +1,9 @@ +processed 2 tasks + +task 1, lines 6-10: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": "1337" +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.move new file mode 100644 index 0000000000000..98eb9c39edea2 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/custom_rgp.move @@ -0,0 +1,10 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --simulator --objects-snapshot-min-checkpoint-lag 2 --reference-gas-price 1337 + +//# run-jsonrpc +{ + "method": "suix_getReferenceGasPrice", + "params": [] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.exp new file mode 100644 index 0000000000000..0ca4885c8b17b --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.exp @@ -0,0 +1,9 @@ +processed 2 tasks + +task 1, lines 6-10: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": "1000" +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.move new file mode 100644 index 0000000000000..08d4ee307c39c --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/default_rgp.move @@ -0,0 +1,10 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --simulator --objects-snapshot-min-checkpoint-lag 2 + +//# run-jsonrpc +{ + "method": "suix_getReferenceGasPrice", + "params": [] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.exp new file mode 100644 index 0000000000000..3de9448d0819e --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.exp @@ -0,0 +1,30 @@ +processed 6 tasks + +task 1, lines 19-22: +//# run-graphql +Error: GraphQL queries are not supported in these tests + +task 2, line 24: +//# run-jsonrpc +Error: Missing JSON-RPC query + +task 3, lines 26-29: +//# run-jsonrpc +Error: Failed to parse JSON-RPC query + +task 4, lines 31-35: +//# run-jsonrpc +Error: Failed to parse JSON-RPC query + +task 5, lines 37-41: +//# run-jsonrpc --show-headers +Headers: { + "content-type": "application/json; charset=utf-8", + "content-length": "40", +} + +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": "1000" +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.move new file mode 100644 index 0000000000000..af6ac2db9d9ab --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/test_framework/framework.move @@ -0,0 +1,41 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --simulator --objects-snapshot-min-checkpoint-lag 2 + +// This test is checking details about the test runner: +// +// (1) It does not support GraphQL queries. +// (2) Tests will fail if the JSON-RPC query does not contian methods or +// params. +// - No JSON object. +// - Missing params. +// - Extra trailing comma (tricky!) +// (3) Displaying response headers is supported. +// +// The test description is at the top because the JSON does not have explicit +// syntax for comments. + +//# run-graphql +{ + chainIdentifier +} + +//# run-jsonrpc + +//# run-jsonrpc +{ + "method": "suix_getReferenceGasPrice" +} + +//# run-jsonrpc +{ + "method": "suix_getReferenceGasPrice", + "params": [], +} + +//# run-jsonrpc --show-headers +{ + "method": "suix_getReferenceGasPrice", + "params": [] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/tests.rs b/crates/sui-indexer-alt-e2e-tests/tests/tests.rs new file mode 100644 index 0000000000000..371b7f8aaa8be --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/tests.rs @@ -0,0 +1,268 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + error::Error, + net::{IpAddr, Ipv4Addr, SocketAddr}, + path::Path, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +use anyhow::{anyhow, bail, Context}; +use diesel::{dsl, ExpressionMethods, QueryDsl}; +use diesel_async::RunQueryDsl; +use prometheus::Registry; +use reqwest::Client; +use serde_json::{json, Value}; +use sui_indexer_alt::{config::IndexerConfig, start_indexer}; +use sui_indexer_alt_framework::{ingestion::ClientArgs, schema::watermarks, IndexerArgs}; +use sui_indexer_alt_jsonrpc::{start_rpc, RpcArgs}; +use sui_pg_db::{ + temp::{get_available_port, TempDb}, + Db, DbArgs, +}; +use sui_transactional_test_runner::{ + create_adapter, + offchain_state::{OffchainStateReader, TestResponse}, + run_tasks_with_adapter, + test_adapter::{OffChainConfig, SuiTestAdapter, PRE_COMPILED}, +}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use url::Url; + +struct OffchainCluster { + rpc_listen_address: SocketAddr, + indexer: JoinHandle<()>, + jsonrpc: JoinHandle<()>, + database: TempDb, + cancel: CancellationToken, +} + +struct OffchainReader { + db: Db, + rpc_url: Url, + client: Client, + queries: AtomicUsize, +} + +datatest_stable::harness!(run_test, "tests", r".*\.move$"); + +impl OffchainCluster { + /// Create a new off-chain cluster consisting of a temporary database, Indexer, and JSONRPC + /// service, to serve requests from E2E tests. + /// + /// NOTE: this cluster does not honour the following fields in `OffChainConfig`, because they + /// do not map to to how its components are implemented: + /// + /// - `snapshot_config.sleep_duration` -- there are multiple consistent pipelines, and each + /// controls the interval at which it runs. + /// - `retention_config`, as retention is not measured in epochs for this pipeline (it is + /// measured in checkpoints). + async fn new(config: &OffChainConfig) -> Self { + let cancel = CancellationToken::new(); + + let rpc_port = get_available_port(); + let rpc_listen_address = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), rpc_port); + + // We don't expose metrics in these tests, but we create a registry to collect them anyway. + let registry = Registry::new(); + + let database = TempDb::new().expect("Failed to create temporary database"); + + let db_args = DbArgs { + database_url: database.database().url().clone(), + ..Default::default() + }; + + let client_args = ClientArgs { + local_ingestion_path: Some(config.data_ingestion_path.clone()), + remote_store_url: None, + }; + + // The example config includes every pipeline, and we configure its consistent range using + // the off-chain config that was passed in. + let mut indexer_config = IndexerConfig::example(); + indexer_config.consistency.retention = Some(config.snapshot_config.snapshot_min_lag as u64); + + let rpc_args = RpcArgs { + rpc_listen_address, + ..Default::default() + }; + + let with_genesis = true; + let indexer = start_indexer( + db_args.clone(), + IndexerArgs::default(), + client_args, + indexer_config, + with_genesis, + ®istry, + cancel.child_token(), + ) + .await + .expect("Failed to start indexer"); + + let jsonrpc = start_rpc(db_args, rpc_args, ®istry, cancel.child_token()) + .await + .expect("Failed to start JSON-RPC server"); + + Self { + rpc_listen_address, + indexer, + jsonrpc, + database, + cancel, + } + } + + /// An implementation of the API that the test cluster uses to send reads to the off-chain + /// set-up. + async fn reader(&self) -> Box { + let db = Db::for_read(DbArgs { + database_url: self.database.database().url().clone(), + ..Default::default() + }) + .await + .expect("Failed to connect to database"); + + let rpc_url = Url::parse(&format!("http://{}/", self.rpc_listen_address)) + .expect("Failed to parse RPC URL"); + + Box::new(OffchainReader { + db, + rpc_url, + client: Client::new(), + queries: AtomicUsize::new(0), + }) + } + + /// Triggers cancellation of all downstream services, waits for them to stop and cleans up the + /// temporary database. + async fn stopped(self) { + self.cancel.cancel(); + let _ = self.indexer.await; + let _ = self.jsonrpc.await; + } +} + +impl OffchainReader { + /// Wait indefinitely until all pipelines have caught up with `checkpoint`. + async fn wait_for_checkpoint(&self, checkpoint: u64) { + loop { + if matches!(self.latest_checkpoint().await, Ok(latest) if latest >= checkpoint) { + break; + } else { + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + + /// Return the lowest checkpoint that we have committed data for across all pipelines, + /// according to the watermarks table. + /// + /// NOTE: We exclude pipelines that we know correspond to pruners, because they usually lag + /// behind their committer counterparts. + async fn latest_checkpoint(&self) -> anyhow::Result { + let mut conn = self + .db + .connect() + .await + .context("Failed to connect to database")?; + + // FIXME: It's not ideal that we have to enumerate pruners here -- if we forget to add one, + // tests will hang indefinitely. Hopefully, by moving these over to the framework's pruning + // support, we can avoid this complication. + const PRUNERS: &[&str] = &["coin_balance_buckets_pruner", "obj_info_pruner"]; + + let latest: Option = watermarks::table + .select(dsl::min(watermarks::checkpoint_hi_inclusive)) + .filter(watermarks::pipeline.ne_all(PRUNERS)) + .first(&mut conn) + .await?; + + latest + .map(|latest| latest as u64) + .ok_or_else(|| anyhow!("No checkpoints recorded yet")) + } +} + +#[async_trait::async_trait] +impl OffchainStateReader for OffchainReader { + async fn wait_for_objects_snapshot_catchup(&self, _: Duration) { + // Not necessary for `sui-indexer-alt` + } + + async fn wait_for_checkpoint_catchup(&self, checkpoint: u64, base_timeout: Duration) { + let _ = tokio::time::timeout(base_timeout, self.wait_for_checkpoint(checkpoint)).await; + } + + async fn wait_for_pruned_checkpoint(&self, _: u64, _: Duration) { + unimplemented!("Waiting for pruned checkpoints is not supported in these tests (add it if you need it)"); + } + + async fn execute_graphql(&self, _: String, _: bool) -> anyhow::Result { + bail!("GraphQL queries are not supported in these tests") + } + + async fn execute_jsonrpc(&self, method: String, params: Value) -> anyhow::Result { + let query = json!({ + "jsonrpc": "2.0", + "id": self.queries.fetch_add(1, Ordering::SeqCst), + "method": method, + "params": params, + }); + + let response = self + .client + .post(self.rpc_url.clone()) + .json(&query) + .send() + .await + .context("Request to JSON-RPC server failed")?; + + // Extract headers but remove the ones that will change from run to run. + let mut headers = response.headers().clone(); + headers.remove("date"); + + let body: Value = response + .json() + .await + .context("Failed to parse JSON-RPC response")?; + + Ok(TestResponse { + response_body: serde_json::to_string_pretty(&body)?, + http_headers: Some(headers), + service_version: None, + }) + } +} + +#[cfg_attr(not(msim), tokio::main)] +#[cfg_attr(msim, msim::main)] +async fn run_test(path: &Path) -> Result<(), Box> { + if cfg!(msim) { + return Ok(()); + } + + telemetry_subscribers::init_for_testing(); + + // start the adapter first to start the executor (simulacrum) + let (output, mut adapter) = + create_adapter::(path, Some(Arc::new(PRE_COMPILED.clone()))).await?; + + // configure access to the off-chain reader + let cluster = OffchainCluster::new(adapter.offchain_config.as_ref().unwrap()).await; + adapter.with_offchain_reader(cluster.reader().await); + + // run the tasks in the test + run_tasks_with_adapter(path, adapter, output).await?; + + // clean-up the off-chain cluster + cluster.stopped().await; + Ok(()) +}