From aae2ab231927ac6d35bee46530f2d19ad2c9aeed Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 11:56:25 +0000 Subject: [PATCH 01/30] revert(20962): [indexer-alt-jsonrpc] Add basic objects JSONRPC API This reverts commit cc2ec66216aab8f8b0030ec03e45e3eedada3cdc. This commit introduces quite a large merge conflict, and it was easier to handle it by backing it out and re-applying it on top, rather than trying to rebase the larger change over it. --- Cargo.lock | 4 - crates/sui-indexer-alt-jsonrpc/Cargo.toml | 5 - crates/sui-indexer-alt-jsonrpc/src/api/mod.rs | 1 - .../src/api/objects.rs | 175 ------------------ .../sui-indexer-alt-jsonrpc/src/data/mod.rs | 1 - .../src/data/objects.rs | 147 --------------- .../src/data/reader.rs | 21 --- crates/sui-indexer-alt-jsonrpc/src/lib.rs | 6 +- .../sui-indexer-alt-jsonrpc/src/test_env.rs | 55 ------ crates/sui-indexer-alt-schema/src/objects.rs | 2 +- 10 files changed, 2 insertions(+), 415 deletions(-) delete mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/objects.rs delete mode 100644 crates/sui-indexer-alt-jsonrpc/src/data/objects.rs delete mode 100644 crates/sui-indexer-alt-jsonrpc/src/test_env.rs diff --git a/Cargo.lock b/Cargo.lock index 12c1f16862437..d422a4936b65a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14143,15 +14143,12 @@ dependencies = [ "clap", "diesel", "diesel-async", - "hex", "jsonrpsee", "move-core-types", "pin-project-lite", "prometheus", "reqwest 0.12.9", "serde_json", - "sui-indexer-alt", - "sui-indexer-alt-framework", "sui-indexer-alt-metrics", "sui-indexer-alt-schema", "sui-json-rpc-types", @@ -14161,7 +14158,6 @@ dependencies = [ "sui-pg-db", "sui-types", "telemetry-subscribers", - "tempfile", "thiserror 1.0.69", "tokio", "tokio-util 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/crates/sui-indexer-alt-jsonrpc/Cargo.toml b/crates/sui-indexer-alt-jsonrpc/Cargo.toml index ac251ecde4cdb..bc1c09bab321a 100644 --- a/crates/sui-indexer-alt-jsonrpc/Cargo.toml +++ b/crates/sui-indexer-alt-jsonrpc/Cargo.toml @@ -19,7 +19,6 @@ bcs.workspace = true clap.workspace = true diesel = { workspace = true, features = ["chrono"] } diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] } -hex.workspace = true jsonrpsee = { workspace = true, features = ["macros", "server"] } pin-project-lite.workspace = true prometheus.workspace = true @@ -46,7 +45,3 @@ sui-types.workspace = true [dev-dependencies] reqwest.workspace = true serde_json.workspace = true -tempfile.workspace = true - -sui-indexer-alt-framework.workspace = true -sui-indexer-alt.workspace = true diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs index f5fda76c56463..885cd3b02da68 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs @@ -2,6 +2,5 @@ // SPDX-License-Identifier: Apache-2.0 pub(crate) mod governance; -pub(crate) mod objects; pub(crate) mod rpc_module; pub(crate) mod transactions; diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects.rs deleted file mode 100644 index 6930fd5b483cb..0000000000000 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects.rs +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -// FIXME: Add tests. -// TODO: Migrate to use BigTable for KV storage. - -use jsonrpsee::{core::RpcResult, proc_macros::rpc}; -use sui_indexer_alt_schema::objects::StoredObject; -use sui_json_rpc_types::{ - SuiGetPastObjectRequest, SuiObjectData, SuiObjectDataOptions, SuiObjectRef, - SuiPastObjectResponse, -}; -use sui_open_rpc::Module; -use sui_open_rpc_macros::open_rpc; -use sui_types::{ - base_types::{ObjectID, SequenceNumber}, - digests::ObjectDigest, - object::Object, -}; - -use crate::{ - context::Context, - error::{internal_error, invalid_params}, -}; - -use super::rpc_module::RpcModule; - -#[open_rpc(namespace = "sui", tag = "Objects API")] -#[rpc(server, namespace = "sui")] -trait ObjectsApi { - /// Note there is no software-level guarantee/SLA that objects with past versions - /// can be retrieved by this API, even if the object and version exists/existed. - /// The result may vary across nodes depending on their pruning policies. - /// Return the object information for a specified version - #[method(name = "tryGetPastObject")] - async fn try_get_past_object( - &self, - /// the ID of the queried object - object_id: ObjectID, - /// the version of the queried object. If None, default to the latest known version - version: SequenceNumber, - /// options for specifying the content to be returned - options: Option, - ) -> RpcResult; - - /// Note there is no software-level guarantee/SLA that objects with past versions - /// can be retrieved by this API, even if the object and version exists/existed. - /// The result may vary across nodes depending on their pruning policies. - /// Return the object information for a specified version - #[method(name = "tryMultiGetPastObjects")] - async fn try_multi_get_past_objects( - &self, - /// a vector of object and versions to be queried - past_objects: Vec, - /// options for specifying the content to be returned - options: Option, - ) -> RpcResult>; -} - -pub(crate) struct Objects(pub Context); - -#[derive(thiserror::Error, Debug)] -pub(crate) enum Error { - #[error("Object not found: {0} with version {1}")] - NotFound(ObjectID, SequenceNumber), - - #[error("Error converting to response: {0}")] - Conversion(anyhow::Error), - - #[error("Deserialization error: {0}")] - Deserialization(#[from] bcs::Error), -} - -#[async_trait::async_trait] -impl ObjectsApiServer for Objects { - async fn try_get_past_object( - &self, - object_id: ObjectID, - version: SequenceNumber, - options: Option, - ) -> RpcResult { - let Self(ctx) = self; - let Some(stored) = ctx - .loader() - .load_one((object_id, version)) - .await - .map_err(internal_error)? - else { - return Err(invalid_params(Error::NotFound(object_id, version))); - }; - - let options = options.unwrap_or_default(); - response(ctx, &stored, &options) - .await - .map_err(internal_error) - } - - async fn try_multi_get_past_objects( - &self, - past_objects: Vec, - options: Option, - ) -> RpcResult> { - let Self(ctx) = self; - let stored_objects = ctx - .loader() - .load_many(past_objects.iter().map(|p| (p.object_id, p.version))) - .await - .map_err(internal_error)?; - - let mut responses = Vec::with_capacity(past_objects.len()); - let options = options.unwrap_or_default(); - for request in past_objects { - if let Some(stored) = stored_objects.get(&(request.object_id, request.version)) { - responses.push( - response(ctx, stored, &options) - .await - .map_err(internal_error)?, - ); - } else { - responses.push(SuiPastObjectResponse::VersionNotFound( - request.object_id, - request.version, - )); - } - } - - Ok(responses) - } -} - -impl RpcModule for Objects { - fn schema(&self) -> Module { - ObjectsApiOpenRpc::module_doc() - } - - fn into_impl(self) -> jsonrpsee::RpcModule { - self.into_rpc() - } -} - -/// Convert the representation of an object from the database into the response format, -/// including the fields requested in the `options`. -/// FIXME: Actually use the options. -pub(crate) async fn response( - _ctx: &Context, - stored: &StoredObject, - _options: &SuiObjectDataOptions, -) -> Result { - let object_id = - ObjectID::from_bytes(&stored.object_id).map_err(|e| Error::Conversion(e.into()))?; - let version = SequenceNumber::from_u64(stored.object_version as u64); - - let Some(serialized_object) = &stored.serialized_object else { - return Ok(SuiPastObjectResponse::ObjectDeleted(SuiObjectRef { - object_id, - version, - digest: ObjectDigest::OBJECT_DIGEST_DELETED, - })); - }; - let object: Object = bcs::from_bytes(serialized_object).map_err(Error::Deserialization)?; - let object_data = SuiObjectData { - object_id, - version, - digest: object.digest(), - type_: None, - owner: None, - previous_transaction: None, - storage_rebate: None, - display: None, - content: None, - bcs: None, - }; - - Ok(SuiPastObjectResponse::VersionFound(object_data)) -} diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs index ea0cff800c73f..74f0034f2b978 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs @@ -1,7 +1,6 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -pub(crate) mod objects; pub(crate) mod package_resolver; pub(crate) mod reader; pub mod system_package_task; diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs b/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs deleted file mode 100644 index 68583e0792264..0000000000000 --- a/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashMap, sync::Arc}; - -use async_graphql::dataloader::Loader; -use sui_indexer_alt_schema::objects::StoredObject; -use sui_types::{base_types::ObjectID, base_types::SequenceNumber}; - -use super::reader::{ReadError, Reader}; - -/// Load objects by key (object_id, version). -#[async_trait::async_trait] -impl Loader<(ObjectID, SequenceNumber)> for Reader { - type Value = StoredObject; - type Error = Arc; - - async fn load( - &self, - keys: &[(ObjectID, SequenceNumber)], - ) -> Result, Self::Error> { - if keys.is_empty() { - return Ok(HashMap::new()); - } - - let conditions = keys - .iter() - .map(|key| { - format!( - "(object_id = '\\x{}'::bytea AND object_version = {})", - hex::encode(key.0.to_vec()), - key.1.value() - ) - }) - .collect::>(); - let query = format!("SELECT * FROM kv_objects WHERE {}", conditions.join(" OR ")); - - let mut conn = self.connect().await.map_err(Arc::new)?; - let objects: Vec = conn.raw_query(&query).await.map_err(Arc::new)?; - - let results: HashMap<_, _> = objects - .into_iter() - .map(|stored| { - ( - ( - ObjectID::from_bytes(&stored.object_id).unwrap(), - SequenceNumber::from_u64(stored.object_version as u64), - ), - stored, - ) - }) - .collect(); - - Ok(results) - } -} - -#[cfg(test)] -mod tests { - use diesel_async::RunQueryDsl; - use sui_indexer_alt_schema::{objects::StoredObject, schema::kv_objects}; - use sui_types::{ - base_types::{ObjectID, SequenceNumber, SuiAddress}, - object::{Object, Owner}, - }; - - use crate::test_env::IndexerReaderTestEnv; - - async fn insert_objects( - test_env: &IndexerReaderTestEnv, - id_versions: impl IntoIterator, - ) { - let mut conn = test_env.indexer.db().connect().await.unwrap(); - let stored_objects = id_versions - .into_iter() - .map(|(id, version)| { - let object = Object::with_id_owner_version_for_testing( - id, - version, - Owner::AddressOwner(SuiAddress::ZERO), - ); - let serialized_object = bcs::to_bytes(&object).unwrap(); - StoredObject { - object_id: id.to_vec(), - object_version: version.value() as i64, - serialized_object: Some(serialized_object), - } - }) - .collect::>(); - diesel::insert_into(kv_objects::table) - .values(stored_objects) - .execute(&mut conn) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_load_single_object() { - let test_env = IndexerReaderTestEnv::new().await; - let id_version = (ObjectID::ZERO, SequenceNumber::from_u64(1)); - insert_objects(&test_env, vec![id_version]).await; - let object = test_env - .loader() - .load_one(id_version) - .await - .unwrap() - .unwrap(); - assert_eq!(object.object_id, id_version.0.to_vec()); - assert_eq!(object.object_version, id_version.1.value() as i64); - } - - #[tokio::test] - async fn test_load_multiple_objects() { - let test_env = IndexerReaderTestEnv::new().await; - let mut id_versions = vec![ - (ObjectID::ZERO, SequenceNumber::from_u64(1)), - (ObjectID::ZERO, SequenceNumber::from_u64(2)), - (ObjectID::ZERO, SequenceNumber::from_u64(10)), - (ObjectID::from_single_byte(1), SequenceNumber::from_u64(1)), - (ObjectID::from_single_byte(1), SequenceNumber::from_u64(2)), - (ObjectID::from_single_byte(1), SequenceNumber::from_u64(10)), - ]; - insert_objects(&test_env, id_versions.clone()).await; - - let objects = test_env - .loader() - .load_many(id_versions.clone()) - .await - .unwrap(); - assert_eq!(objects.len(), id_versions.len()); - for (id, version) in &id_versions { - let object = objects.get(&(*id, *version)).unwrap(); - assert_eq!(object.object_id, id.to_vec()); - assert_eq!(object.object_version, version.value() as i64); - } - - // Add a ID/version that doesn't exist in the table. - // Query will still succeed, but will return the same set of objects as before. - id_versions.push((ObjectID::from_single_byte(2), SequenceNumber::from_u64(1))); - let objects = test_env - .loader() - .load_many(id_versions.clone()) - .await - .unwrap(); - assert_eq!(objects.len(), id_versions.len() - 1); - } -} diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs b/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs index 009dc6b51cb86..153f892f8f40c 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs @@ -9,7 +9,6 @@ use diesel::pg::Pg; use diesel::query_builder::QueryFragment; use diesel::query_dsl::methods::LimitDsl; use diesel::result::Error as DieselError; -use diesel::sql_query; use diesel_async::methods::LoadQuery; use diesel_async::RunQueryDsl; use prometheus::Registry; @@ -117,24 +116,4 @@ impl<'p> Connection<'p> { Ok(res?) } - - // TODO: Use the `RawSqlQuery` from GraphQL implementation. - pub(crate) async fn raw_query(&mut self, query: &str) -> Result, ReadError> - where - U: diesel::QueryableByName + Send + 'static, - { - debug!("Raw query: {}", query); - - let _guard = self.metrics.db_latency.start_timer(); - - let res = sql_query(query).load::(&mut self.conn).await; - - if res.is_ok() { - self.metrics.db_requests_succeeded.inc(); - } else { - self.metrics.db_requests_failed.inc(); - } - - Ok(res?) - } } diff --git a/crates/sui-indexer-alt-jsonrpc/src/lib.rs b/crates/sui-indexer-alt-jsonrpc/src/lib.rs index 41936cd7cf58d..15e318cd0defb 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/lib.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/lib.rs @@ -5,7 +5,6 @@ use std::net::SocketAddr; use std::sync::Arc; use anyhow::Context as _; -use api::objects::Objects; use api::rpc_module::RpcModule; use api::transactions::Transactions; use data::system_package_task::{SystemPackageTask, SystemPackageTaskArgs}; @@ -30,8 +29,6 @@ mod context; pub mod data; mod error; mod metrics; -#[cfg(test)] -mod test_env; #[derive(clap::Args, Debug, Clone)] pub struct RpcArgs { @@ -192,7 +189,7 @@ impl Default for RpcArgs { /// command-line). The service will continue to run until the cancellation token is triggered, and /// will signal cancellation on the token when it is shutting down. /// -/// The service may spin up auxiliary services (such as the system package task) to support +/// The service may spin up auxilliary services (such as the system package task) to support /// itself, and will clean these up on shutdown as well. pub async fn start_rpc( db_args: DbArgs, @@ -214,7 +211,6 @@ pub async fn start_rpc( rpc.add_module(Governance(context.clone()))?; rpc.add_module(Transactions(context.clone()))?; - rpc.add_module(Objects(context.clone()))?; let h_rpc = rpc.run().await.context("Failed to start RPC service")?; let h_system_package_task = system_package_task.run(); diff --git a/crates/sui-indexer-alt-jsonrpc/src/test_env.rs b/crates/sui-indexer-alt-jsonrpc/src/test_env.rs deleted file mode 100644 index c55c65304399b..0000000000000 --- a/crates/sui-indexer-alt-jsonrpc/src/test_env.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use async_graphql::dataloader::DataLoader; -use prometheus::Registry; -use sui_indexer_alt_framework::{ - ingestion::{ClientArgs, IngestionConfig}, - Indexer, IndexerArgs, -}; -use sui_indexer_alt_schema::MIGRATIONS; -use sui_pg_db::{temp::TempDb, DbArgs}; -use tempfile::tempdir; -use tokio_util::sync::CancellationToken; - -use crate::data::reader::Reader; -use crate::metrics::RpcMetrics; - -pub(crate) struct IndexerReaderTestEnv { - pub(crate) indexer: Indexer, - pub(crate) reader: Reader, - pub(crate) _temp_db: TempDb, -} - -impl IndexerReaderTestEnv { - pub(crate) async fn new() -> Self { - let temp_db = TempDb::new().unwrap(); - let db_args = DbArgs::new_for_testing(temp_db.database().url().clone()); - let registry = Registry::new(); - let indexer = Indexer::new( - db_args.clone(), - IndexerArgs::default(), - ClientArgs { - remote_store_url: None, - local_ingestion_path: Some(tempdir().unwrap().into_path()), - }, - IngestionConfig::default(), - &MIGRATIONS, - ®istry, - CancellationToken::new(), - ) - .await - .unwrap(); - let rpc_metrics = RpcMetrics::new(®istry); - let reader = Reader::new(db_args, rpc_metrics, ®istry).await.unwrap(); - Self { - indexer, - reader, - _temp_db: temp_db, - } - } - - pub(crate) fn loader(&self) -> DataLoader { - self.reader.as_data_loader() - } -} diff --git a/crates/sui-indexer-alt-schema/src/objects.rs b/crates/sui-indexer-alt-schema/src/objects.rs index dc30684fe9b5c..a2472ffe6522d 100644 --- a/crates/sui-indexer-alt-schema/src/objects.rs +++ b/crates/sui-indexer-alt-schema/src/objects.rs @@ -12,7 +12,7 @@ use sui_types::object::{Object, Owner}; use crate::schema::{coin_balance_buckets, kv_objects, obj_info, obj_versions}; -#[derive(Insertable, QueryableByName, Debug, Clone, FieldCount)] +#[derive(Insertable, Debug, Clone, FieldCount)] #[diesel(table_name = kv_objects, primary_key(object_id, object_version))] #[diesel(treat_none_as_default_value = false)] pub struct StoredObject { From 6a6ff0a2fa67a48105a5b34ef615c055cd5f27bd Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Wed, 22 Jan 2025 00:36:36 +0530 Subject: [PATCH 02/30] indexer-alt: add tx_digest -> tx_sequence_number index ## Description Add an index on `tx_digests` from digest to sequence number (the reverse of the primary key index). This is to make it easier to look up balance changes (keyed by sequence number in the database) based on their digest (the key we use for the KV store). The alternative was to make `tx_balances` keyed on digest, but in that case we would still need to retain the sequence number, because it is used for pruning. ## Test plan CI (Ran `./generate_schema.sh`) --- .../2025-01-21-161911_tx_digests_tx_sequence_number/down.sql | 1 + .../2025-01-21-161911_tx_digests_tx_sequence_number/up.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/down.sql create mode 100644 crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/up.sql diff --git a/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/down.sql b/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/down.sql new file mode 100644 index 0000000000000..e04d36dfb13b7 --- /dev/null +++ b/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS tx_digests_tx_sequence_number; diff --git a/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/up.sql b/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/up.sql new file mode 100644 index 0000000000000..933bcdc2a0f82 --- /dev/null +++ b/crates/sui-indexer-alt-schema/migrations/2025-01-21-161911_tx_digests_tx_sequence_number/up.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS tx_digests_tx_sequence_number +ON tx_digests (tx_sequence_number); From e8c361885dfd12ca541a9713d5dd47ec04ef2510 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Wed, 22 Jan 2025 22:11:19 +0530 Subject: [PATCH 03/30] refactor(rpc-alt): introduce TransactionKey ## Description Rather than implementing the `StoredTransaction` DataLoader directly on `TransactionDigest` as the key type, introduce a new type wrapping the digest. This is so that we can implement other data loaders that use `TransactionDigest` as the key. ## Test plan CI --- .../src/api/transactions.rs | 3 ++- .../src/data/transactions.rs | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs index 133b8ae56c9d8..fd383841950a7 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs @@ -18,6 +18,7 @@ use sui_types::{ use crate::{ context::Context, + data::transactions::TransactionKey, error::{internal_error, invalid_params}, }; @@ -64,7 +65,7 @@ impl TransactionsApiServer for Transactions { let Self(ctx) = self; let Some(stored) = ctx .loader() - .load_one(digest) + .load_one(TransactionKey(digest)) .await .map_err(internal_error)? else { diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/data/transactions.rs index f53c24ad1c0e9..dfd3adb2041cb 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/transactions.rs @@ -13,15 +13,19 @@ use sui_types::digests::TransactionDigest; use super::reader::{ReadError, Reader}; +/// Key for fetching transaction contents (TransactionData, Effects, and Events) by digest. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct TransactionKey(pub TransactionDigest); + #[async_trait::async_trait] -impl Loader for Reader { +impl Loader for Reader { type Value = StoredTransaction; type Error = Arc; async fn load( &self, - keys: &[TransactionDigest], - ) -> Result, Self::Error> { + keys: &[TransactionKey], + ) -> Result, Self::Error> { use kv_transactions::dsl as t; if keys.is_empty() { @@ -30,7 +34,7 @@ impl Loader for Reader { let mut conn = self.connect().await.map_err(Arc::new)?; - let digests: BTreeSet<_> = keys.iter().map(|d| d.into_inner()).collect(); + let digests: BTreeSet<_> = keys.iter().map(|d| d.0.into_inner()).collect(); let transactions: Vec = conn .results(t::kv_transactions.filter(t::tx_digest.eq_any(digests))) .await @@ -44,7 +48,7 @@ impl Loader for Reader { Ok(keys .iter() .filter_map(|key| { - let slice: &[u8] = key.as_ref(); + let slice: &[u8] = key.0.as_ref(); Some((*key, digest_to_stored.get(slice).cloned()?)) }) .collect()) From 1c6d687967cdf4c186e320a835449ba50a999cef Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Thu, 23 Jan 2025 00:31:14 +0530 Subject: [PATCH 04/30] rpc-alt: showBalanceChanges for getTransactionBlock ## Description Implement showBalanceChanges for getTransactionBlock. Data is fetched using a data loader, and the response logic has changed slightly so that the output corresponding to each response option gets its own function (rather than handling everything in one function). This refactoring was done to handle the fact that if balance changes are requested we: - Want to perform an extra DB load, which we ideally want to do concurrently with fetching the transaction. - Want to use the presence or absence of an `Option` to decide whether to add the corresponding output, rather than the flag from the option. ## Test plan New E2E test: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests ``` --- Cargo.lock | 1 + .../transactions/get_transaction_block.exp | 941 +++++++++--------- .../transactions/get_transaction_block.move | 31 +- crates/sui-indexer-alt-jsonrpc/Cargo.toml | 1 + .../src/api/transactions.rs | 224 +++-- .../sui-indexer-alt-jsonrpc/src/data/mod.rs | 1 + .../src/data/tx_balance_changes.rs | 62 ++ .../src/transactions.rs | 2 +- 8 files changed, 738 insertions(+), 525 deletions(-) create mode 100644 crates/sui-indexer-alt-jsonrpc/src/data/tx_balance_changes.rs diff --git a/Cargo.lock b/Cargo.lock index d422a4936b65a..c52ffe19b26f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14143,6 +14143,7 @@ dependencies = [ "clap", "diesel", "diesel-async", + "futures", "jsonrpsee", "move-core-types", "pin-project-lite", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp index 82ea3a9f7b5e4..8e3f127bb4dd6 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp @@ -1,15 +1,15 @@ -processed 9 tasks +processed 12 tasks init: -A: object(0,0) +A: object(0,0), B: object(0,1) -task 1, lines 12-46: +task 1, lines 14-48: //# publish created: object(1,0), object(1,1) -mutated: object(0,1) +mutated: object(0,2) gas summary: computation_cost: 1000000, storage_cost: 9211200, storage_rebate: 0, non_refundable_storage_fee: 0 -task 2, lines 48-54: +task 2, lines 50-56: //# programmable --sender A --inputs object(1,0) 42 @A //> 0: test::counter::inc(Input(0)); //> 1: test::counter::inc_by(Input(0), Input(1)); @@ -17,26 +17,34 @@ task 2, lines 48-54: //> 3: test::counter::inc_by(Input(0), Result(2)); //> 4: test::counter::take(Input(0), Input(1)); //> 5: TransferObjects([Result(4)], Input(2)) -events: Event { package_id: test, transaction_module: Identifier("counter"), sender: A, type_: StructTag { address: test, module: Identifier("counter"), name: Identifier("NFTMinted"), type_params: [] }, contents: [41, 96, 231, 235, 135, 134, 117, 102, 3, 204, 210, 31, 140, 81, 38, 14, 171, 188, 201, 196, 121, 128, 155, 196, 248, 198, 125, 113, 60, 127, 239, 247] } +events: Event { package_id: test, transaction_module: Identifier("counter"), sender: A, type_: StructTag { address: test, module: Identifier("counter"), name: Identifier("NFTMinted"), type_params: [] }, contents: [36, 128, 140, 176, 175, 75, 57, 9, 37, 183, 118, 215, 178, 121, 175, 49, 254, 87, 44, 244, 46, 73, 120, 94, 146, 242, 21, 112, 3, 28, 150, 50] } created: object(2,0) mutated: object(0,0), object(1,0) gas summary: computation_cost: 1000000, storage_cost: 3678400, storage_rebate: 1346796, non_refundable_storage_fee: 13604 -task 3, line 56: +task 3, lines 58-60: +//# programmable --sender A --inputs 42 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(3,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 4, line 62: //# create-checkpoint Checkpoint created: 1 -task 4, lines 58-62: +task 5, lines 64-68: //# run-jsonrpc Response: { "jsonrpc": "2.0", "id": 0, "result": { - "digest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs" + "digest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL" } } -task 5, lines 64-68: +task 6, lines 70-74: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -47,14 +55,14 @@ Response: { } } -task 6, lines 70-80: +task 7, lines 76-86: //# run-jsonrpc Response: { "jsonrpc": "2.0", "id": 2, "result": { - "digest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs", - "rawTransaction": "AAADAQFVgoNgXuarD3xAE3cOojA9IQQR26dH+uxMEfbX1R6dhgIAAAAAAAAAAQAIKgAAAAAAAAAAIPzMmkIbuxPBpmoaqY8K11Ap7elIV3ecaRW0T5QGi5IeBgDsz8HPY4GWIqXu67rnTG9rf6ekBFETZGH2d5atP48z/gdjb3VudGVyA2luYwABAQAAAOzPwc9jgZYipe7ruudMb2t/p6QEURNkYfZ3lq0/jzP+B2NvdW50ZXIGaW5jX2J5AAIBAAABAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIEY29pbgV2YWx1ZQEHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIDc3VpA1NVSQABAADsz8HPY4GWIqXu67rnTG9rf6ekBFETZGH2d5atP48z/gdjb3VudGVyBmluY19ieQACAQAAAgIAAOzPwc9jgZYipe7ruudMb2t/p6QEURNkYfZ3lq0/jzP+B2NvdW50ZXIEdGFrZQACAQAAAQEAAQECBAABAgD8zJpCG7sTwaZqGqmPCtdQKe3pSFd3nGkVtE+UBouSHgG/9elqSlwPc0glnHq/39mZQFwCuenA0NWexmmlOxBMegEAAAAAAAAAIKEd8xRTuu+n68ZlWvsLrCmzFa8z+W8E5apBshZjx/M//MyaQhu7E8GmahqpjwrXUCnt6UhXd5xpFbRPlAaLkh7oAwAAAAAAAADyBSoBAAAAAA==", + "digest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", + "rawTransaction": "AAADAQFBd+JUVDhx5XeiGSNJI5s4UQEfXa07GCfaFCUk98lC1wIAAAAAAAAAAQAIKgAAAAAAAAAAIPzMmkIbuxPBpmoaqY8K11Ap7elIV3ecaRW0T5QGi5IeBgBiHbdDLb65YtqclRm+ClCfrP6oKqHdC41gITwzJcAVqAdjb3VudGVyA2luYwABAQAAAGIdt0Mtvrli2pyVGb4KUJ+s/qgqod0LjWAhPDMlwBWoB2NvdW50ZXIGaW5jX2J5AAIBAAABAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIEY29pbgV2YWx1ZQEHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIDc3VpA1NVSQABAABiHbdDLb65YtqclRm+ClCfrP6oKqHdC41gITwzJcAVqAdjb3VudGVyBmluY19ieQACAQAAAgIAAGIdt0Mtvrli2pyVGb4KUJ+s/qgqod0LjWAhPDMlwBWoB2NvdW50ZXIEdGFrZQACAQAAAQEAAQECBAABAgD8zJpCG7sTwaZqGqmPCtdQKe3pSFd3nGkVtE+UBouSHgEoNNhdv+/c1m8EgRIxuoGIk3k+g6iV1TQC/ZnhMuNlYgEAAAAAAAAAIAxTyQqgv33v1e2KKeTKCujrlS8ekEWqYA2s3/WsyFOJ/MyaQhu7E8GmahqpjwrXUCnt6UhXd5xpFbRPlAaLkh7oAwAAAAAAAADyBSoBAAAAAA==", "rawEffects": [ 1, 0, @@ -99,144 +107,144 @@ Response: { 0, 0, 32, - 220, - 135, - 91, - 245, - 94, - 173, - 245, - 146, - 181, - 140, - 59, - 176, + 71, + 131, 38, - 179, - 59, - 185, - 90, - 176, + 79, + 17, + 216, + 21, + 248, + 103, + 109, + 39, + 21, + 255, + 78, + 166, + 184, + 208, + 203, + 136, + 251, + 233, + 245, + 69, + 196, + 224, + 62, 34, - 173, - 99, - 238, - 151, - 236, - 47, - 147, - 95, - 134, - 165, - 19, - 235, - 140, + 185, + 230, + 97, + 13, + 49, + 1, 1, - 2, 0, 0, 0, 1, 32, - 35, - 234, - 81, - 115, - 57, - 243, - 103, - 224, - 175, - 29, - 141, - 219, + 247, 189, - 113, - 214, - 68, - 130, - 116, - 106, - 7, - 113, - 255, - 34, - 214, + 139, + 228, + 72, + 94, + 84, + 72, + 63, + 93, 131, - 51, - 107, - 133, - 226, - 136, - 224, - 247, - 2, + 76, + 134, + 63, + 21, + 67, + 230, + 66, + 210, 32, - 33, - 96, - 204, - 109, - 165, - 71, - 188, + 20, + 99, + 180, 85, - 187, - 139, - 69, - 34, - 243, - 29, - 142, - 35, - 254, - 32, - 23, - 233, - 154, - 186, - 96, - 24, - 108, - 237, - 183, - 181, - 10, - 7, - 247, - 26, + 143, + 17, + 16, + 97, + 188, + 103, + 61, + 162, + 2, 32, - 221, - 81, - 192, + 54, + 234, + 13, + 174, + 78, + 101, + 36, + 117, + 198, + 255, + 182, + 156, + 30, + 104, + 31, + 203, 236, - 185, - 4, - 19, - 137, - 211, - 93, - 227, - 108, - 152, + 136, + 218, + 107, + 72, + 104, + 239, 189, - 2, - 229, - 194, - 247, - 14, - 181, - 1, + 141, + 220, + 137, + 224, + 89, + 71, + 179, + 22, + 32, + 126, + 244, + 78, + 39, + 248, + 204, + 239, + 74, 61, - 241, - 62, - 33, - 28, + 49, 12, - 188, - 144, - 73, - 7, - 241, + 142, + 174, + 1, + 118, + 168, + 68, + 249, + 237, + 130, + 182, + 116, + 246, + 94, + 37, + 145, + 92, + 15, + 145, + 214, + 235, + 75, 3, 0, 0, @@ -246,73 +254,73 @@ Response: { 0, 0, 3, - 41, - 96, - 231, - 235, - 135, - 134, - 117, - 102, - 3, - 204, - 210, - 31, + 36, + 128, 140, - 81, - 38, - 14, - 171, - 188, - 201, - 196, + 176, + 175, + 75, + 57, + 9, + 37, + 183, + 118, + 215, + 178, 121, - 128, - 155, - 196, - 248, - 198, - 125, - 113, - 60, - 127, - 239, - 247, + 175, + 49, + 254, + 87, + 44, + 244, + 46, + 73, + 120, + 94, + 146, + 242, + 21, + 112, + 3, + 28, + 150, + 50, 0, 1, 32, - 126, - 22, - 145, - 146, - 124, - 26, - 49, - 95, - 145, - 147, - 125, - 201, - 39, + 215, + 77, + 98, + 227, + 62, + 228, + 227, + 13, + 186, + 0, + 174, + 93, 120, + 55, + 98, 225, + 188, + 217, 15, - 254, - 207, - 64, - 27, - 22, - 94, - 36, - 157, - 253, - 97, - 75, - 239, - 138, - 119, + 249, + 201, + 219, + 136, + 57, + 183, + 175, + 246, 175, - 171, + 22, + 39, + 204, + 225, 0, 252, 204, @@ -347,165 +355,38 @@ Response: { 146, 30, 1, - 85, - 130, - 131, - 96, - 94, - 230, - 171, - 15, - 124, - 64, - 19, - 119, - 14, - 162, - 48, - 61, - 33, - 4, - 17, - 219, - 167, - 71, - 250, - 236, - 76, - 17, - 246, - 215, - 213, - 30, - 157, - 134, - 1, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 32, - 8, - 187, + 40, + 52, + 216, + 93, + 191, + 239, + 220, 214, - 160, - 189, - 68, - 36, - 168, - 209, - 42, - 181, - 76, - 230, - 154, - 178, - 60, - 232, - 212, - 184, - 234, - 131, - 17, - 90, - 251, + 111, + 4, + 129, + 18, + 49, + 186, + 129, + 136, + 147, + 121, 62, - 243, - 106, - 204, - 220, 131, - 101, - 241, - 2, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 1, - 32, - 51, - 146, - 165, - 254, - 55, - 171, - 56, - 11, - 40, - 150, - 38, - 170, - 157, - 12, - 67, - 243, - 59, - 11, - 42, - 189, - 229, - 229, - 71, - 29, - 100, - 7, - 175, - 219, - 213, - 162, - 167, - 36, - 2, - 2, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 191, - 245, - 233, - 106, - 74, - 92, - 15, - 115, - 72, - 37, - 156, - 122, - 191, - 223, - 217, - 153, - 64, - 92, - 2, - 185, - 233, - 192, - 208, - 213, - 158, - 198, - 105, - 165, - 59, - 16, - 76, - 122, + 168, + 149, + 213, + 52, + 2, + 253, + 153, + 225, + 50, + 227, + 101, + 98, 1, 1, 0, @@ -516,38 +397,38 @@ Response: { 0, 0, 32, - 161, - 29, - 243, - 20, + 12, 83, - 186, + 201, + 10, + 160, + 191, + 125, 239, - 167, - 235, - 198, - 101, - 90, - 251, - 11, - 172, + 213, + 237, + 138, 41, - 179, - 21, - 175, - 51, - 249, - 111, - 4, - 229, + 228, + 202, + 10, + 232, + 235, + 149, + 47, + 30, + 144, + 69, 170, - 65, - 178, - 22, - 99, - 199, - 243, - 63, + 96, + 13, + 172, + 223, + 245, + 172, + 200, + 83, + 137, 0, 252, 204, @@ -583,38 +464,38 @@ Response: { 30, 1, 32, - 114, - 196, - 58, - 191, - 102, - 185, - 166, - 160, - 211, - 88, - 87, - 226, - 103, - 38, - 180, + 164, + 47, + 177, + 32, + 0, 219, + 35, + 39, + 41, + 147, + 183, + 85, 201, - 133, - 52, - 178, - 16, - 75, - 6, - 13, - 233, - 0, - 163, - 95, + 93, + 129, + 83, + 40, + 170, + 243, + 154, + 248, + 24, + 55, + 71, + 193, + 69, + 124, + 159, + 9, + 48, + 210, 139, - 44, - 222, - 167, 0, 252, 204, @@ -649,19 +530,146 @@ Response: { 146, 30, 0, + 65, + 119, + 226, + 84, + 84, + 56, + 113, + 229, + 119, + 162, + 25, + 35, + 73, + 35, + 155, + 56, + 81, + 1, + 31, + 93, + 173, + 59, + 24, + 39, + 218, + 20, + 37, + 36, + 247, + 201, + 66, + 215, + 1, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 32, + 205, + 115, + 251, + 47, + 136, + 76, + 131, + 223, + 249, + 240, + 193, + 169, + 134, + 55, + 165, + 30, + 155, + 29, + 51, + 114, + 43, + 249, + 243, + 184, + 104, + 1, + 65, + 210, + 77, + 151, + 141, + 102, + 2, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 32, + 43, + 118, + 118, + 215, + 229, + 72, + 180, + 74, + 62, + 98, + 64, + 11, + 123, + 72, + 108, + 56, + 29, + 214, + 9, + 38, + 153, + 127, + 122, + 40, + 233, + 246, + 2, + 53, + 160, + 184, + 125, + 44, + 2, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, 0, 0 ] } } -task 7, lines 82-92: +task 8, lines 88-98: //# run-jsonrpc Response: { "jsonrpc": "2.0", "id": 3, "result": { - "digest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs", + "digest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", "transaction": { "data": { "messageVersion": "v1", @@ -671,7 +679,7 @@ Response: { { "type": "object", "objectType": "sharedObject", - "objectId": "0x558283605ee6ab0f7c4013770ea2303d210411dba747faec4c11f6d7d51e9d86", + "objectId": "0x4177e254543871e577a2192349239b3851011f5dad3b1827da142524f7c942d7", "initialSharedVersion": "2", "mutable": true }, @@ -689,7 +697,7 @@ Response: { "transactions": [ { "MoveCall": { - "package": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe", + "package": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8", "module": "counter", "function": "inc", "arguments": [ @@ -701,7 +709,7 @@ Response: { }, { "MoveCall": { - "package": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe", + "package": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8", "module": "counter", "function": "inc_by", "arguments": [ @@ -729,7 +737,7 @@ Response: { }, { "MoveCall": { - "package": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe", + "package": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8", "module": "counter", "function": "inc_by", "arguments": [ @@ -744,7 +752,7 @@ Response: { }, { "MoveCall": { - "package": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe", + "package": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8", "module": "counter", "function": "take", "arguments": [ @@ -775,9 +783,9 @@ Response: { "gasData": { "payment": [ { - "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "objectId": "0x2834d85dbfefdcd66f04811231ba818893793e83a895d53402fd99e132e36562", "version": 1, - "digest": "Bqw8MjtPSgoFarHYSq7HN77R9utjvxqtSFD32ydKXsiW" + "digest": "q7zZtCD3hWzn8bDfuAfVs4Jz9xuXbaRoRmscfDLfRGc" } ], "owner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", @@ -786,7 +794,7 @@ Response: { } }, "txSignatures": [ - "AER38jSf17YdkZqbJEPRwjAuX3kp0JL4vbLqU/pxl7D7X2MREYLcCnMbT+jF+txsUbVMfJroYTwYfx3t0ish7gR/UUY663bYjcm3XmNyULIgxJz1t5Z9vxfB+fp8WUoJKA==" + "AO0JoGECqyBeXhej35Fv0rYGG+O5yh5/wSFPK/f4GRybyqqSFTnojVZWloTzMZ8nW1xpkfDjQ/IrF8dbRPZCpAh/UUY663bYjcm3XmNyULIgxJz1t5Z9vxfB+fp8WUoJKA==" ] }, "effects": { @@ -803,55 +811,55 @@ Response: { }, "modifiedAtVersions": [ { - "objectId": "0x558283605ee6ab0f7c4013770ea2303d210411dba747faec4c11f6d7d51e9d86", - "sequenceNumber": "2" + "objectId": "0x2834d85dbfefdcd66f04811231ba818893793e83a895d53402fd99e132e36562", + "sequenceNumber": "1" }, { - "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", - "sequenceNumber": "1" + "objectId": "0x4177e254543871e577a2192349239b3851011f5dad3b1827da142524f7c942d7", + "sequenceNumber": "2" } ], "sharedObjects": [ { - "objectId": "0x558283605ee6ab0f7c4013770ea2303d210411dba747faec4c11f6d7d51e9d86", + "objectId": "0x4177e254543871e577a2192349239b3851011f5dad3b1827da142524f7c942d7", "version": 2, - "digest": "b6PSDcJ7F2jMDTvZMSsbuqexkSi2hDoiZQL46YxECNC" + "digest": "Eq19qfVDFudB3Kc63ha8KA3aCVEB5ZUFSEeBcmH7Qy5f" } ], - "transactionDigest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs", + "transactionDigest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", "created": [ { "owner": { "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" }, "reference": { - "objectId": "0x2960e7eb8786756603ccd21f8c51260eabbcc9c479809bc4f8c67d713c7feff7", + "objectId": "0x24808cb0af4b390925b776d7b279af31fe572cf42e49785e92f21570031c9632", "version": 3, - "digest": "9VCL8EztoKtT8h9294KcMTBy5TLZRncvYdRMvmepmKCv" + "digest": "FVT6ZPwya7HV99NvwWU9hPLY8GzegmazpQnpci2jSXSQ" } } ], "mutated": [ { "owner": { - "Shared": { - "initial_shared_version": 2 - } + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" }, "reference": { - "objectId": "0x558283605ee6ab0f7c4013770ea2303d210411dba747faec4c11f6d7d51e9d86", + "objectId": "0x2834d85dbfefdcd66f04811231ba818893793e83a895d53402fd99e132e36562", "version": 3, - "digest": "4UKVNcXyHWS2CtNG7pDLdMZjsibnRBsXQaiRP5sARUHh" + "digest": "C3v3QDwZPz6917cKrRVjVyZZpTBY4ci2GoiXHjbhV56z" } }, { "owner": { - "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + "Shared": { + "initial_shared_version": 2 + } }, "reference": { - "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "objectId": "0x4177e254543871e577a2192349239b3851011f5dad3b1827da142524f7c942d7", "version": 3, - "digest": "8j12PP5NaZ6Tn5rGVHb2SkU5rtYeV9G1md1fRRqBZ15Y" + "digest": "3vfJUp5PpXkpyBTNTLK9jTv2SZYXVLSJqHdtARmvTgi7" } } ], @@ -860,44 +868,89 @@ Response: { "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" }, "reference": { - "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "objectId": "0x2834d85dbfefdcd66f04811231ba818893793e83a895d53402fd99e132e36562", "version": 3, - "digest": "8j12PP5NaZ6Tn5rGVHb2SkU5rtYeV9G1md1fRRqBZ15Y" + "digest": "C3v3QDwZPz6917cKrRVjVyZZpTBY4ci2GoiXHjbhV56z" } }, - "eventsDigest": "3RCW6N38LzHG9yvdaWzk1Beaum5a3JSWP73MDLmygo3t", + "eventsDigest": "Hg5LZJqeAYNSHRnXi7E5zJQkvF1HrafWqZTstt1CkMUD", "dependencies": [ - "3FJ4fSrf7toVCANccxAbeJ5A1iSzwKLghCYcaz9atbCD", - "FtwQTnaYQ7BDSpxR9gtbr2B5XBjMGYXgv3KQngEak3ix" + "4hN1oBeozq3Hno8q9JfKkrTUQHs21Q2j7UWHk1bxSk5B", + "9YaSDYB2hY7DwGwATGe2y5D4d8BwtQjE8bj2wRQecqnr" ] } } } -task 8, lines 95-104: +task 9, lines 100-109: //# run-jsonrpc Response: { "jsonrpc": "2.0", "id": 4, "result": { - "digest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs", + "digest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", "events": [ { "id": { - "txDigest": "FqrQYPLunNvAQKn2EKrWAHc32942bjovH6VeFDgPQUQs", + "txDigest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", "eventSeq": "0" }, - "packageId": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe", + "packageId": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8", "transactionModule": "counter", "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", - "type": "0xeccfc1cf63819622a5eeebbae74c6f6b7fa7a40451136461f67796ad3f8f33fe::counter::NFTMinted", + "type": "0x621db7432dbeb962da9c9519be0a509facfea82aa1dd0b8d60213c3325c015a8::counter::NFTMinted", "parsedJson": { - "id": "0x2960e7eb8786756603ccd21f8c51260eabbcc9c479809bc4f8c67d713c7feff7" + "id": "0x24808cb0af4b390925b776d7b279af31fe572cf42e49785e92f21570031c9632" }, "bcsEncoding": "base64", - "bcs": "KWDn64eGdWYDzNIfjFEmDqu8ycR5gJvE+MZ9cTx/7/c=", + "bcs": "JICMsK9LOQklt3bXsnmvMf5XLPQuSXhekvIVcAMcljI=", "timestampMs": "0" } ] } } + +task 10, lines 111-120: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 5, + "result": { + "digest": "5p9wHYHPWxr5qSCDifjzxQLxv8Berk2tik2jtyfoQmQL", + "balanceChanges": [ + { + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "coinType": "0x2::sui::SUI", + "amount": "-3331604" + } + ] + } +} + +task 11, lines 122-131: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 6, + "result": { + "digest": "DQQTRmCg1KDocZb1JwChJa8nQjAG9UaY2Jy1GnFKamDQ", + "balanceChanges": [ + { + "owner": { + "AddressOwner": "0xa7b032703878aa74c3126935789fd1d4d7e111d5911b09247d6963061c312b5a" + }, + "coinType": "0x2::sui::SUI", + "amount": "42" + }, + { + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "coinType": "0x2::sui::SUI", + "amount": "-1997922" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.move index 8751b3371a04e..16ce19da547b2 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.move +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.move @@ -1,13 +1,15 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -//# init --protocol-version 70 --accounts A --addresses test=0x0 --simulator +//# init --protocol-version 70 --accounts A B --addresses test=0x0 --simulator // 1. Default behavior of getTransactionBlock (no options) // 2. "Not found" case // 3. Raw transaction and effects // 4. Structured transaction and effects // 5. Events +// 6a. Balance Changes (gas only) +// 6b. Balance Changes (transfer SUI) //# publish module test::counter { @@ -53,6 +55,10 @@ module test::counter { //> 4: test::counter::take(Input(0), Input(1)); //> 5: TransferObjects([Result(4)], Input(2)) +//# programmable --sender A --inputs 42 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + //# create-checkpoint //# run-jsonrpc @@ -91,7 +97,6 @@ module test::counter { ] } - //# run-jsonrpc { "method": "sui_getTransactionBlock", @@ -102,3 +107,25 @@ module test::counter { } ] } + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": [ + "@{digest_2}", + { + "showBalanceChanges": true + } + ] +} + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": [ + "@{digest_3}", + { + "showBalanceChanges": true + } + ] +} diff --git a/crates/sui-indexer-alt-jsonrpc/Cargo.toml b/crates/sui-indexer-alt-jsonrpc/Cargo.toml index bc1c09bab321a..013a70325b118 100644 --- a/crates/sui-indexer-alt-jsonrpc/Cargo.toml +++ b/crates/sui-indexer-alt-jsonrpc/Cargo.toml @@ -19,6 +19,7 @@ bcs.workspace = true clap.workspace = true diesel = { workspace = true, features = ["chrono"] } diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] } +futures.workspace = true jsonrpsee = { workspace = true, features = ["macros", "server"] } pin-project-lite.workspace = true prometheus.workspace = true diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs index fd383841950a7..48ec927a0d4bc 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs @@ -1,24 +1,31 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::str::FromStr; + use anyhow::anyhow; +use futures::future::OptionFuture; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use move_core_types::annotated_value::{MoveDatatypeLayout, MoveTypeLayout}; -use sui_indexer_alt_schema::transactions::StoredTransaction; +use sui_indexer_alt_schema::transactions::{ + BalanceChange, StoredTransaction, StoredTxBalanceChange, +}; use sui_json_rpc_types::{ - SuiEvent, SuiTransactionBlock, SuiTransactionBlockData, SuiTransactionBlockEvents, - SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, + BalanceChange as SuiBalanceChange, SuiEvent, SuiTransactionBlock, SuiTransactionBlockData, + SuiTransactionBlockEffects, SuiTransactionBlockEvents, SuiTransactionBlockResponse, + SuiTransactionBlockResponseOptions, }; use sui_open_rpc::Module; use sui_open_rpc_macros::open_rpc; use sui_types::{ digests::TransactionDigest, effects::TransactionEffects, error::SuiError, event::Event, - signature::GenericSignature, transaction::TransactionData, + signature::GenericSignature, transaction::TransactionData, TypeTag, }; +use tokio::join; use crate::{ context::Context, - data::transactions::TransactionKey, + data::{transactions::TransactionKey, tx_balance_changes::TxBalanceChangeKey}, error::{internal_error, invalid_params}, }; @@ -62,19 +69,69 @@ impl TransactionsApiServer for Transactions { digest: TransactionDigest, options: SuiTransactionBlockResponseOptions, ) -> RpcResult { + use Error as E; + let Self(ctx) = self; - let Some(stored) = ctx - .loader() - .load_one(TransactionKey(digest)) - .await + + let transaction = ctx.loader().load_one(TransactionKey(digest)); + let balance_changes: OptionFuture<_> = options + .show_balance_changes + .then(|| ctx.loader().load_one(TxBalanceChangeKey(digest))) + .into(); + + let (transaction, balance_changes) = join!(transaction, balance_changes); + + let transaction = transaction .map_err(internal_error)? - else { - return Err(invalid_params(Error::NotFound(digest))); - }; + .ok_or_else(|| invalid_params(E::NotFound(digest)))?; - response(ctx, &stored, &options) - .await - .map_err(internal_error) + // Balance changes might not be present because of pruning, in which case we return + // nothing, even if the changes were requested. + let balance_changes = balance_changes + .transpose() + .map_err(internal_error)? + .flatten(); + + let digest = TransactionDigest::try_from(transaction.tx_digest.clone()) + .map_err(E::Conversion) + .map_err(internal_error)?; + + let mut response = SuiTransactionBlockResponse::new(digest); + + if options.show_input { + response.transaction = Some( + input_response(ctx, &transaction) + .await + .map_err(internal_error)?, + ); + } + + if options.show_raw_input { + response.raw_transaction = transaction.raw_transaction.clone(); + } + + if options.show_effects { + response.effects = Some(effects_response(&transaction).map_err(internal_error)?); + } + + if options.show_raw_effects { + response.raw_effects = transaction.raw_effects.clone(); + } + + if options.show_events { + response.events = Some( + events_response(ctx, digest, &transaction) + .await + .map_err(internal_error)?, + ); + } + + if let Some(balance_changes) = balance_changes { + response.balance_changes = + Some(balance_changes_response(balance_changes).map_err(internal_error)?); + } + + Ok(response) } } @@ -88,79 +145,90 @@ impl RpcModule for Transactions { } } -/// Convert the representation of a transaction from the database into the response format, -/// including the fields requested in the `options`. -pub(crate) async fn response( +/// Extract a representation of the transaction's input data from the stored form. +async fn input_response( ctx: &Context, tx: &StoredTransaction, - options: &SuiTransactionBlockResponseOptions, -) -> Result { - use Error as E; +) -> Result { + let data: TransactionData = bcs::from_bytes(&tx.raw_transaction)?; + let tx_signatures: Vec = bcs::from_bytes(&tx.user_signatures)?; - let digest = TransactionDigest::try_from(tx.tx_digest.clone()).map_err(E::Conversion)?; - let mut response = SuiTransactionBlockResponse::new(digest); - - if options.show_input { - let data: TransactionData = bcs::from_bytes(&tx.raw_transaction)?; - let tx_signatures: Vec = bcs::from_bytes(&tx.user_signatures)?; - response.transaction = Some(SuiTransactionBlock { - data: SuiTransactionBlockData::try_from_with_package_resolver( - data, - ctx.package_resolver(), - ) + Ok(SuiTransactionBlock { + data: SuiTransactionBlockData::try_from_with_package_resolver(data, ctx.package_resolver()) .await - .map_err(E::Resolution)?, - tx_signatures, - }) - } + .map_err(Error::Resolution)?, + tx_signatures, + }) +} - if options.show_raw_input { - response.raw_transaction = tx.raw_transaction.clone(); - } +/// Extract a representation of the transaction's effects from the stored form. +fn effects_response(tx: &StoredTransaction) -> Result { + let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects)?; + effects.try_into().map_err(Error::Conversion) +} - if options.show_effects { - let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects)?; - response.effects = Some(effects.try_into().map_err(E::Conversion)?); - } +/// Extract the transaction's events from its stored form. +async fn events_response( + ctx: &Context, + digest: TransactionDigest, + tx: &StoredTransaction, +) -> Result { + use Error as E; + + let events: Vec = bcs::from_bytes(&tx.events)?; + let mut sui_events = Vec::with_capacity(events.len()); + + for (ix, event) in events.into_iter().enumerate() { + let layout = match ctx + .package_resolver() + .type_layout(event.type_.clone().into()) + .await + .map_err(|e| E::Resolution(e.into()))? + { + MoveTypeLayout::Struct(s) => MoveDatatypeLayout::Struct(s), + MoveTypeLayout::Enum(e) => MoveDatatypeLayout::Enum(e), + _ => { + return Err(E::Resolution(anyhow!( + "Event {ix} from {digest} is not a struct or enum: {}", + event.type_.to_canonical_string(/* with_prefix */ true) + ))); + } + }; + + let sui_event = SuiEvent::try_from( + event, + digest, + ix as u64, + Some(tx.timestamp_ms as u64), + layout, + ) + .map_err(E::Conversion)?; - if options.show_raw_effects { - response.raw_effects = tx.raw_effects.clone(); + sui_events.push(sui_event) } - if options.show_events { - let events: Vec = bcs::from_bytes(&tx.events)?; - let mut sui_events = Vec::with_capacity(events.len()); - - for (ix, event) in events.into_iter().enumerate() { - let layout = match ctx - .package_resolver() - .type_layout(event.type_.clone().into()) - .await - .map_err(|e| E::Resolution(e.into()))? - { - MoveTypeLayout::Struct(s) => MoveDatatypeLayout::Struct(s), - MoveTypeLayout::Enum(e) => MoveDatatypeLayout::Enum(e), - _ => { - return Err(E::Resolution(anyhow!( - "Event {ix} from {digest} is not a struct or enum: {}", - event.type_.to_canonical_string(/* with_prefix */ true) - ))); - } - }; - - let sui_event = SuiEvent::try_from( - event, - digest, - ix as u64, - Some(tx.timestamp_ms as u64), - layout, - ) - .map_err(E::Conversion)?; - - sui_events.push(sui_event) - } + Ok(SuiTransactionBlockEvents { data: sui_events }) +} - response.events = Some(SuiTransactionBlockEvents { data: sui_events }); +/// Extract the transaction's balance changes from their stored form. +fn balance_changes_response( + balance_changes: StoredTxBalanceChange, +) -> Result, Error> { + let balance_changes: Vec = bcs::from_bytes(&balance_changes.balance_changes)?; + let mut response = Vec::with_capacity(balance_changes.len()); + + for BalanceChange::V1 { + owner, + coin_type, + amount, + } in balance_changes + { + let coin_type = TypeTag::from_str(&coin_type).map_err(Error::Resolution)?; + response.push(SuiBalanceChange { + owner, + coin_type, + amount, + }); } Ok(response) diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs index 74f0034f2b978..91104075a8512 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod package_resolver; pub(crate) mod reader; pub mod system_package_task; pub(crate) mod transactions; +pub(crate) mod tx_balance_changes; diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/tx_balance_changes.rs b/crates/sui-indexer-alt-jsonrpc/src/data/tx_balance_changes.rs new file mode 100644 index 0000000000000..f66eddbd3c768 --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/data/tx_balance_changes.rs @@ -0,0 +1,62 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; + +use async_graphql::dataloader::Loader; +use diesel::{ExpressionMethods, JoinOnDsl, QueryDsl, SelectableHelper}; +use sui_indexer_alt_schema::{ + schema::{tx_balance_changes, tx_digests}, + transactions::StoredTxBalanceChange, +}; +use sui_types::digests::TransactionDigest; + +use super::reader::{ReadError, Reader}; + +/// Key for fetching a transaction's balance changes by digest. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct TxBalanceChangeKey(pub TransactionDigest); + +#[async_trait::async_trait] +impl Loader for Reader { + type Value = StoredTxBalanceChange; + type Error = Arc; + + async fn load( + &self, + keys: &[TxBalanceChangeKey], + ) -> Result, Self::Error> { + use tx_balance_changes::dsl as b; + use tx_digests::dsl as t; + + if keys.is_empty() { + return Ok(HashMap::new()); + } + + let mut conn = self.connect().await.map_err(Arc::new)?; + + let digests: BTreeSet<_> = keys.iter().map(|d| d.0.into_inner()).collect(); + let balance_changes: Vec<(Vec, StoredTxBalanceChange)> = conn + .results( + b::tx_balance_changes + .inner_join(t::tx_digests.on(b::tx_sequence_number.eq(t::tx_sequence_number))) + .select((t::tx_digest, StoredTxBalanceChange::as_select())) + .filter(t::tx_digest.eq_any(digests)), + ) + .await + .map_err(Arc::new)?; + + let digest_to_balance_changes: HashMap<_, _> = balance_changes.into_iter().collect(); + + Ok(keys + .iter() + .filter_map(|key| { + let slice: &[u8] = key.0.as_ref(); + Some((*key, digest_to_balance_changes.get(slice).cloned()?)) + }) + .collect()) + } +} diff --git a/crates/sui-indexer-alt-schema/src/transactions.rs b/crates/sui-indexer-alt-schema/src/transactions.rs index e3228fcde3827..9921ef1c801ff 100644 --- a/crates/sui-indexer-alt-schema/src/transactions.rs +++ b/crates/sui-indexer-alt-schema/src/transactions.rs @@ -64,7 +64,7 @@ pub struct StoredTxAffectedObject { pub sender: Vec, } -#[derive(Insertable, Debug, Clone, FieldCount, Queryable)] +#[derive(Insertable, Selectable, Debug, Clone, FieldCount, Queryable)] #[diesel(table_name = tx_balance_changes)] pub struct StoredTxBalanceChange { pub tx_sequence_number: i64, From 4a418bcca8cb94ede8ad91c8b6a8de6d75eb91cf Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Thu, 23 Jan 2025 01:55:19 +0530 Subject: [PATCH 05/30] rpc-alt: fix issue with Reader::results lifetimes ## Description Writing diesel is 1% thinking about the data you want access to, 9% figuring out what the DSL should be to represent it, and 90% staring into the type constraint abyss while crying. I have put in the requisite crying time, and now we can use `Reader` to run boxed queries. I'm not exactly sure what was wrong with the old constraints, but the new constraints work because they are derived manually from the constraints required by the functions called in the implementations of `results` and `first`. ## Test plan CI to ensure existing uses continue working. In a follow-up change, `results` will be used with a boxed select statement. --- .../src/data/reader.rs | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs b/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs index 153f892f8f40c..84261208f6d3c 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/reader.rs @@ -4,12 +4,13 @@ use std::sync::Arc; use async_graphql::dataloader::DataLoader; -use diesel::dsl::Limit; +use diesel::deserialize::FromSqlRow; +use diesel::expression::QueryMetadata; use diesel::pg::Pg; -use diesel::query_builder::QueryFragment; +use diesel::query_builder::{Query, QueryFragment, QueryId}; use diesel::query_dsl::methods::LimitDsl; +use diesel::query_dsl::CompatibleType; use diesel::result::Error as DieselError; -use diesel_async::methods::LoadQuery; use diesel_async::RunQueryDsl; use prometheus::Registry; use sui_indexer_alt_metrics::db::DbConnectionStatsCollector; @@ -75,12 +76,14 @@ impl Reader { } impl<'p> Connection<'p> { - pub(crate) async fn first(&mut self, query: Q) -> Result + pub(crate) async fn first<'q, Q, ST, U>(&mut self, query: Q) -> Result where - U: Send, - Q: RunQueryDsl + 'static, Q: LimitDsl, - Limit: LoadQuery<'static, db::ManagedConnection, U> + QueryFragment + Send, + Q::Output: Query + QueryFragment + QueryId + Send + 'q, + ::SqlType: CompatibleType, + U: Send + FromSqlRow + 'static, + Pg: QueryMetadata<::SqlType>, + ST: 'static, { let query = query.limit(1); debug!("{}", diesel::debug_query(&query)); @@ -97,11 +100,13 @@ impl<'p> Connection<'p> { Ok(res?) } - pub(crate) async fn results(&mut self, query: Q) -> Result, ReadError> + pub(crate) async fn results<'q, Q, ST, U>(&mut self, query: Q) -> Result, ReadError> where - U: Send, - Q: RunQueryDsl + 'static, - Q: LoadQuery<'static, db::ManagedConnection, U> + QueryFragment + Send, + Q: Query + QueryFragment + QueryId + Send + 'q, + Q::SqlType: CompatibleType, + U: Send + FromSqlRow + 'static, + Pg: QueryMetadata, + ST: 'static, { debug!("{}", diesel::debug_query(&query)); From a23e17dd05b3e6e94b9521cb97b33faee34a3c1a Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Fri, 24 Jan 2025 18:44:33 +0000 Subject: [PATCH 06/30] rpc-alt: showObjectChanges for getTransactionBlock ## Description Implement `showObjectChanges` for transaction blocks. This works by fetching all the objects at the versions mentioned in effects using a data loader, and then iterating through the Effects V2 `ObjectChange`s to translate them into JSON-RPC's `ObjectChange`s. There is not a 1:1 correspondence between these two (JSON-RPC's version is missing multiple cases that the Effects V2 types capture), and the exceptions have been noted in the code. This completes the implementation of `getTransactionBlock`. While working on this change, I was also able to fix the lifetime issue related to `BoxedSelectStatement` and `Reader::results` (as I needed to use this for the new object data loader). This fix is captured in its own commit. ## Test plan The behaviour currently implemented by the indexer and fullnode have been captured in new E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- object_changes ``` --- .../object_changes/create_delete.exp | 95 ++++++++ .../object_changes/create_delete.move | 44 ++++ .../create_wrap_unwrap_delete.exp | 108 +++++++++ .../create_wrap_unwrap_delete.move | 61 +++++ .../object_changes/publish_upgrade.exp | 110 +++++++++ .../object_changes/publish_upgrade.move | 35 +++ .../transactions/object_changes/transfer.exp | 63 ++++++ .../transactions/object_changes/transfer.move | 32 +++ .../transactions/object_changes/unwrap.exp | 66 ++++++ .../transactions/object_changes/unwrap.move | 43 ++++ .../transactions/object_changes/wrap.exp | 73 ++++++ .../transactions/object_changes/wrap.move | 42 ++++ .../src/api/transactions.rs | 209 +++++++++++++++++- .../sui-indexer-alt-jsonrpc/src/data/mod.rs | 1 + .../src/data/objects.rs | 64 ++++++ crates/sui-indexer-alt-jsonrpc/src/error.rs | 22 ++ crates/sui-indexer-alt-schema/src/objects.rs | 2 +- 17 files changed, 1058 insertions(+), 12 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.move create mode 100644 crates/sui-indexer-alt-jsonrpc/src/data/objects.rs diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.exp new file mode 100644 index 0000000000000..b085d1f016705 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.exp @@ -0,0 +1,95 @@ +processed 7 tasks + +init: +A: object(0,0) + +task 1, lines 9-23: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 4772800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 25-27: +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2196400, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 29-30: +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::delete(Input(0)) +mutated: object(0,0) +deleted: object(2,0) +gas summary: computation_cost: 1000000, storage_cost: 988000, storage_rebate: 2174436, non_refundable_storage_fee: 21964 + +task 4, line 32: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 34-38: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "4ukxTageudh4AU1Hv25XeNucQcX6R5Gw2BwAC19rkUjt", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "2", + "previousVersion": "1", + "digest": "2LSs7zxkU42HhLyqNWnwTGD5yEzzh1yZSaueQ31RJc6A" + }, + { + "type": "created", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0xf827985b0577822cde33d6014436abc7b3a1d13106e97b6939ecf7d466c1310f::M::O", + "objectId": "0xc7162cc23503adb20cb79297ed7ce9079f7430cdd57fe901c3c4d8becc36db8d", + "version": "2", + "digest": "8dXr7d9fBENNdxBX1G3jzb6s8V5QKDWewqUiM4hUNPYq" + } + ] + } +} + +task 6, lines 40-44: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "digest": "94ATSaPBkEeiAuZRjV7mBSpWffQuEkJ9RJjdPUH6eEu9", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "3", + "previousVersion": "2", + "digest": "GzRuf5gpzWgtRuVwzqfVPZED5fy1Apx2yRdbzUEwewmD" + }, + { + "type": "deleted", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "objectType": "0xf827985b0577822cde33d6014436abc7b3a1d13106e97b6939ecf7d466c1310f::M::O", + "objectId": "0xc7162cc23503adb20cb79297ed7ce9079f7430cdd57fe901c3c4d8becc36db8d", + "version": "3" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.move new file mode 100644 index 0000000000000..aee0d7ff0e441 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_delete.move @@ -0,0 +1,44 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses P0=0x0 --simulator + +// Test creating and deleting an object. Both of these operations should show +// up in object changes. + +//# publish +module P0::M { + public struct O has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): O { + O { id: object::new(ctx) } + } + + public fun delete(o: O) { + let O { id } = o; + id.delete(); + } +} + +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::delete(Input(0)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_2}", { "showObjectChanges": true }] +} + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_3}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.exp new file mode 100644 index 0000000000000..6e8a555844992 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.exp @@ -0,0 +1,108 @@ +processed 8 tasks + +init: +A: object(0,0) + +task 1, lines 9-36: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 6718400, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 38-40: +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2204000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 42-43: +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::wrap(Input(0)) +mutated: object(0,0), object(2,0) +gas summary: computation_cost: 1000000, storage_cost: 2447200, storage_rebate: 2181960, non_refundable_storage_fee: 22040 + +task 4, lines 45-47: +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::unwrap(Input(0)); +//> 1: P0::M::delete(Result(0)) +mutated: object(0,0), object(2,0) +unwrapped_then_deleted: object(_) +gas summary: computation_cost: 1000000, storage_cost: 2204000, storage_rebate: 2422728, non_refundable_storage_fee: 24472 + +task 5, line 49: +//# create-checkpoint +Checkpoint created: 1 + +task 6, lines 51-55: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "Aead3uHudTECLTGjJGQkNgYAF5YxjXRgGKE7hZZrDCjL", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x97f20c29a52b26bce2e4fcbd7d1c0a260ffdc7add9d796fe86b36b05147706a4::M::O", + "objectId": "0x071ffb59697a02c999cdc8dc53e8f34103d5b3a6b0aa81edf2c48e66de848a08", + "version": "3", + "previousVersion": "2", + "digest": "8psQ2GnnGsPK2FfDFzzATSKkmVuHhoVzEcuTN8QCsezQ" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "3", + "previousVersion": "2", + "digest": "9M8HLgH8K16BkshyD9aJFPoQ6iHxcNrqyEJNtiz5SZop" + } + ] + } +} + +task 7, lines 57-61: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "digest": "Hy6893StV7pDGxxyHWkpTesxUvDh1XdFVC2PXATZLAgA", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x97f20c29a52b26bce2e4fcbd7d1c0a260ffdc7add9d796fe86b36b05147706a4::M::O", + "objectId": "0x071ffb59697a02c999cdc8dc53e8f34103d5b3a6b0aa81edf2c48e66de848a08", + "version": "4", + "previousVersion": "3", + "digest": "9TN1KGrTdGpMHnKkP5xj6uJK6gtMJYugv3P1rfSbJpmP" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "4", + "previousVersion": "3", + "digest": "B5DRwEEFLfGp3KbXbJBv6jG4UzzmHdJW8iXi1Hqo3iSt" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.move new file mode 100644 index 0000000000000..6a36326b3e295 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/create_wrap_unwrap_delete.move @@ -0,0 +1,61 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses P0=0x0 --simulator + +// "created and wrapped" as well as "unwrapped and deleted" objects do not show +// up in object changes at all. + +//# publish +module P0::M { + public struct O has key, store { + id: UID, + i: Option, + } + + public struct I has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): O { + O { id: object::new(ctx), i: option::none() } + } + + public fun wrap(o: &mut O, ctx: &mut TxContext) { + o.i.fill(I { id: object::new(ctx) }); + } + + public fun unwrap(o: &mut O): I { + o.i.extract() + } + + public fun delete(i: I) { + let I { id } = i; + id.delete(); + } +} + +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::wrap(Input(0)) + +//# programmable --sender A --inputs object(2,0) +//> 0: P0::M::unwrap(Input(0)); +//> 1: P0::M::delete(Result(0)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_3}", { "showObjectChanges": true }] +} + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_4}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.exp new file mode 100644 index 0000000000000..42ab2de65f658 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.exp @@ -0,0 +1,110 @@ +processed 6 tasks + +init: +A: object(0,0) + +task 1, lines 9-12: +//# publish --upgradeable --sender A +created: object(1,0), object(1,1) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 5076800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 14-21: +//# upgrade --package P0 --upgrade-capability 1,1 --sender A +created: object(2,0) +mutated: object(0,0), object(1,1) +gas summary: computation_cost: 1000000, storage_cost: 5768400, storage_rebate: 2595780, non_refundable_storage_fee: 26220 + +task 3, line 23: +//# create-checkpoint +Checkpoint created: 1 + +task 4, lines 25-29: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "BfhUDo5S1qaoTNPZNbUWPTwievt5NJzyKN2AxmDEH6wz", + "objectChanges": [ + { + "type": "published", + "packageId": "0x7670061a5a8ace811e66a06a1358363b9bd7b8f2f9ee8a0ea262bc0e33b148c0", + "version": "1", + "digest": "FpCHKBbpdgVNCiqX5xe13rVGDqd2HYyGuCJ8hdj9Pv8L", + "modules": [ + "M" + ] + }, + { + "type": "created", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::package::UpgradeCap", + "objectId": "0xa3a64bd1bbfc45f234b17702cf453fa3660a94d11b5bf18efd69262ab49b9d9f", + "version": "2", + "digest": "5xPHPaX3axjkrFYdE3RmQZA6grnUDDE69TypXwJCd1va" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "2", + "previousVersion": "1", + "digest": "BRwz4gsomJX3ka2jqYBp7Ud2XSmL4We8weumCPsMSYSm" + } + ] + } +} + +task 5, lines 31-35: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "digest": "875EAzb9SBt8rhxjhYFD2bnuisUXhJN1m7si8BH8QuDF", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::package::UpgradeCap", + "objectId": "0xa3a64bd1bbfc45f234b17702cf453fa3660a94d11b5bf18efd69262ab49b9d9f", + "version": "3", + "previousVersion": "2", + "digest": "WSZ1LFNowwaMYKfqNajHYdp6DZGYdyisW9zSD36iP3m" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "3", + "previousVersion": "2", + "digest": "GehBiyivz5iaQDd3K5A38oNTfZLLcVgsgXhNo2vnYwXv" + }, + { + "type": "published", + "packageId": "0xead19ea504f857a14578d1547ae33f531f0dd612637e1aec30b9f00ae335d747", + "version": "2", + "digest": "Ftbo2Y9txgidKvz8UNdE7Z3phjtH8n4D3Dfs5WX8E3dP", + "modules": [ + "M", + "N" + ] + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.move new file mode 100644 index 0000000000000..8254ae1d339dd --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/publish_upgrade.move @@ -0,0 +1,35 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses P0=0x0 P1=0x0 --simulator + +// Publishes and upgrades of user packages both show up as "Published" object +// changes. + +//# publish --upgradeable --sender A +module P0::M { + public fun f(): u64 { 42 } +} + +//# upgrade --package P0 --upgrade-capability 1,1 --sender A +module P1::M { + public fun f(): u64 { 42 } +} + +module P1::N { + public fun g(): u64 { 43 } +} + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_1}", { "showObjectChanges": true }] +} + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_2}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.exp new file mode 100644 index 0000000000000..4444ac22a0c93 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.exp @@ -0,0 +1,63 @@ +processed 6 tasks + +init: +A: object(0,0), B: object(0,1) + +task 1, lines 8-17: +//# publish +created: object(1,0) +mutated: object(0,2) +gas summary: computation_cost: 1000000, storage_cost: 4544800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 19-21: +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2196400, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 23-24: +//# programmable --sender A --inputs object(2,0) @B +//> 0: TransferObjects([Input(0)], Input(1)) +mutated: object(0,0), object(2,0) +gas summary: computation_cost: 1000000, storage_cost: 2196400, storage_rebate: 2174436, non_refundable_storage_fee: 21964 + +task 4, line 26: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 28-32: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "CrVRtSGQYfefzNWfU6a6ZHmZzU2WMoCKn7nqHw4X2QBp", + "objectChanges": [ + { + "type": "transferred", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "recipient": { + "AddressOwner": "0xa7b032703878aa74c3126935789fd1d4d7e111d5911b09247d6963061c312b5a" + }, + "objectType": "0x1a5a31900509d00c257cef365edcbba723be53e2083ccbfda4e0a2ea0b284203::M::O", + "objectId": "0x00d176adb885597c7871d9e9ca7442c533dec70083b6275c03374be06b9b503f", + "version": "3", + "digest": "JAK8Ws8AWQcHjd8xsfBRXe9FKXVnp6Ei56CtTjqCjiah" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0x2834d85dbfefdcd66f04811231ba818893793e83a895d53402fd99e132e36562", + "version": "3", + "previousVersion": "2", + "digest": "GNoT9p5mUDUpCiAs4enq3HKjFZE8DjnKfC5xZyFZwgdR" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.move new file mode 100644 index 0000000000000..69afd2ccf207a --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/transfer.move @@ -0,0 +1,32 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A B --addresses P0=0x0 --simulator + +// When an object's owner changes, it is considered "transferred". + +//# publish +module P0::M { + public struct O has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): O { + O { id: object::new(ctx) } + } +} + +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(2,0) @B +//> 0: TransferObjects([Input(0)], Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_3}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.exp new file mode 100644 index 0000000000000..8fa467699268c --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.exp @@ -0,0 +1,66 @@ +processed 6 tasks + +init: +A: object(0,0) + +task 1, lines 8-27: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 6118000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 29-31: +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2447200, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 33-35: +//# programmable --sender A --inputs object(2,0) @A +//> 0: P0::M::unwrap(Input(0)); +//> 1: TransferObjects([Result(0)], Input(1)) +mutated: object(0,0), object(2,0) +unwrapped: object(3,0) +gas summary: computation_cost: 1000000, storage_cost: 3412400, storage_rebate: 2422728, non_refundable_storage_fee: 24472 + +task 4, line 37: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 39-43: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "JBHReYdBYTUoFhGZgCVG3jFUoHQSYTz7Sgt7yW6TPzFz", + "objectChanges": [ + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "3", + "previousVersion": "2", + "digest": "7jod7CYtkuwxrc8HcBNj5HYy3D32kQ45MxTZhvCzzfb8" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x54c05d65095bba68361751ed13055e1330ab7d78ac032dfbc03eb5eb0b43b6d3::M::O", + "objectId": "0xc1e6b79d26aafd5940fe4fcfcb5f83ae4101388c849e1d9bbd4f1048f4fe10ee", + "version": "3", + "previousVersion": "2", + "digest": "Htp48aPEeYX141zEKbAQW9bB2jyUsL2MKVDDT9B3PXcw" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.move new file mode 100644 index 0000000000000..5f2137b704ada --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/unwrap.move @@ -0,0 +1,43 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses P0=0x0 --simulator + +// Unwrapped objects don't show up in object changes. + +//# publish +module P0::M { + public struct O has key, store { + id: UID, + i: Option, + } + + public struct I has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): O { + let i = I { id: object::new(ctx) }; + O { id: object::new(ctx), i: option::some(i) } + } + + public fun unwrap(o: &mut O): I { + o.i.extract() + } +} + +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(2,0) @A +//> 0: P0::M::unwrap(Input(0)); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_3}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.exp new file mode 100644 index 0000000000000..2ff95619c9896 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.exp @@ -0,0 +1,73 @@ +processed 6 tasks + +init: +A: object(0,0) + +task 1, lines 8-26: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 5836800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 28-30: +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2196400, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 32-34: +//# programmable --sender A --inputs object(2,0) @A +//> 0: P0::M::wrap(Input(0)); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(3,0) +mutated: object(0,0) +wrapped: object(2,0) +gas summary: computation_cost: 1000000, storage_cost: 2447200, storage_rebate: 2174436, non_refundable_storage_fee: 21964 + +task 4, line 36: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 38-42: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "digest": "8uiu3WCLxRYnoP3P6NGn1YfdRK99b4iLZ21ifdxDNErd", + "objectChanges": [ + { + "type": "created", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x45204616e1053b2a44105712720357c178195724066d7b7eb32f10e798e99e1e::M::O", + "objectId": "0x70e651e9ac346ad05a4e094f593ef324d78806c295fdfef52ea24c0aa4180d99", + "version": "3", + "digest": "BFhy5Mn1Z2AiNBbVzowTAQGnP2SB9swXpWcyDgTJfeXg" + }, + { + "type": "mutated", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "objectType": "0x2::coin::Coin<0x2::sui::SUI>", + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": "3", + "previousVersion": "2", + "digest": "4xQ6qSchZVmud2VeEeAKaQKjbUj67XhcqA8w9oVXBt5C" + }, + { + "type": "wrapped", + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "objectType": "0x45204616e1053b2a44105712720357c178195724066d7b7eb32f10e798e99e1e::M::I", + "objectId": "0xea80aa0712647964efcf2b71dfab5b996245ab7302c1d010179a08fb9ca4d078", + "version": "3" + } + ] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.move new file mode 100644 index 0000000000000..01f18096bcff0 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/object_changes/wrap.move @@ -0,0 +1,42 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses P0=0x0 --simulator + +// Wrapped objects do show up in object changes. + +//# publish +module P0::M { + public struct O has key, store { + id: UID, + i: Option, + } + + public struct I has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): I { + I { id: object::new(ctx) } + } + + public fun wrap(i: I, ctx: &mut TxContext): O { + O { id: object::new(ctx), i: option::some(i) } + } +} + +//# programmable --sender A --inputs @A +//> 0: P0::M::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(2,0) @A +//> 0: P0::M::wrap(Input(0)); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_getTransactionBlock", + "params": ["@{digest_3}", { "showObjectChanges": true }] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs index 48ec927a0d4bc..49c237a700f46 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs @@ -11,22 +11,32 @@ use sui_indexer_alt_schema::transactions::{ BalanceChange, StoredTransaction, StoredTxBalanceChange, }; use sui_json_rpc_types::{ - BalanceChange as SuiBalanceChange, SuiEvent, SuiTransactionBlock, SuiTransactionBlockData, - SuiTransactionBlockEffects, SuiTransactionBlockEvents, SuiTransactionBlockResponse, - SuiTransactionBlockResponseOptions, + BalanceChange as SuiBalanceChange, ObjectChange as SuiObjectChange, SuiEvent, + SuiTransactionBlock, SuiTransactionBlockData, SuiTransactionBlockEffects, + SuiTransactionBlockEvents, SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, }; use sui_open_rpc::Module; use sui_open_rpc_macros::open_rpc; use sui_types::{ - digests::TransactionDigest, effects::TransactionEffects, error::SuiError, event::Event, - signature::GenericSignature, transaction::TransactionData, TypeTag, + base_types::{ObjectID, SequenceNumber}, + digests::{ObjectDigest, TransactionDigest}, + effects::{IDOperation, ObjectChange, TransactionEffects, TransactionEffectsAPI}, + error::SuiError, + event::Event, + object::Object, + signature::GenericSignature, + transaction::{TransactionData, TransactionDataAPI}, + TypeTag, }; use tokio::join; use crate::{ context::Context, - data::{transactions::TransactionKey, tx_balance_changes::TxBalanceChangeKey}, - error::{internal_error, invalid_params}, + data::{ + objects::ObjectVersionKey, transactions::TransactionKey, + tx_balance_changes::TxBalanceChangeKey, + }, + error::{internal_error, invalid_params, pruned, rpc_bail}, }; use super::rpc_module::RpcModule; @@ -87,10 +97,11 @@ impl TransactionsApiServer for Transactions { // Balance changes might not be present because of pruning, in which case we return // nothing, even if the changes were requested. - let balance_changes = balance_changes - .transpose() - .map_err(internal_error)? - .flatten(); + let balance_changes = match balance_changes.transpose().map_err(internal_error)? { + Some(None) => rpc_bail!(pruned("balance changes for transaction {digest}")), + Some(changes) => changes, + None => None, + }; let digest = TransactionDigest::try_from(transaction.tx_digest.clone()) .map_err(E::Conversion) @@ -131,6 +142,11 @@ impl TransactionsApiServer for Transactions { Some(balance_changes_response(balance_changes).map_err(internal_error)?); } + if options.show_object_changes { + response.object_changes = + Some(object_changes_response(ctx, digest, &transaction).await?); + } + Ok(response) } } @@ -233,3 +249,174 @@ fn balance_changes_response( Ok(response) } + +/// Extract the transaction's object changes. Object IDs and versions are fetched from the stored +/// transaction, and the object contents are fetched separately by a data loader. +async fn object_changes_response( + ctx: &Context, + digest: TransactionDigest, + tx: &StoredTransaction, +) -> RpcResult> { + let tx_data: TransactionData = bcs::from_bytes(&tx.raw_transaction) + .map_err(Error::from) + .map_err(internal_error)?; + + let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects) + .map_err(Error::from) + .map_err(internal_error)?; + + let mut keys = vec![]; + let native_changes = effects.object_changes(); + for change in &native_changes { + let id = change.id; + if let Some(version) = change.input_version { + keys.push(ObjectVersionKey(id, version.value())); + } + if let Some(version) = change.output_version { + keys.push(ObjectVersionKey(id, version.value())); + } + } + + let objects = ctx + .loader() + .load_many(keys) + .await + .map_err(|e| internal_error(e.to_string()))?; + + // Fetch and deserialize the contents of an object, based on its object ref. Assumes that all + // object versions that will be fetched in this way have come from a valid transaction, and + // have been passed to the data loader in the call above. This means that if they cannot be + // found, they must have been pruned. + let fetch_object = |id: ObjectID, + v: Option, + d: Option| + -> RpcResult> { + let Some(v) = v else { return Ok(None) }; + let Some(d) = d else { return Ok(None) }; + + let v = v.value(); + + let stored = objects + .get(&ObjectVersionKey(id, v)) + .ok_or_else(|| pruned(format!("Object {id} at version {v}")))?; + + let bytes = stored + .serialized_object + .as_ref() + .ok_or_else(|| internal_error("No content for object {id} at version {v}"))?; + + let o = bcs::from_bytes(bytes) + .map_err(Error::from) + .map_err(internal_error)?; + + Ok(Some((o, d))) + }; + + let mut changes = Vec::with_capacity(native_changes.len()); + + for change in native_changes { + let &ObjectChange { + id: object_id, + id_operation, + input_version, + input_digest, + output_version, + output_digest, + .. + } = &change; + + let input = fetch_object(object_id, input_version, input_digest)?; + let output = fetch_object(object_id, output_version, output_digest)?; + + use IDOperation as ID; + changes.push(match (id_operation, input, output) { + (ID::Created, Some((i, _)), _) => rpc_bail!(internal_error( + "Unexpected input version {} for object {object_id} created by transaction {digest}", + i.version().value(), + )), + + (ID::Deleted, _, Some((o, _))) => rpc_bail!(internal_error( + "Unexpected output version {} for object {object_id} deleted by transaction {digest}", + o.version().value(), + )), + + // The following cases don't end up in the output: created and wrapped objects, + // unwrapped objects (and by extension, unwrapped and deleted objects), system package + // upgrades (which happen in place). + (ID::Created, _, None) => continue, + (ID::None, None, _) => continue, + (ID::None, _, Some((o, _))) if o.is_package() => continue, + (ID::Deleted, None, _) => continue, + + (ID::Created, _, Some((o, d))) if o.is_package() => SuiObjectChange::Published { + package_id: object_id, + version: o.version(), + digest: d, + modules: o + .data + .try_as_package() + .unwrap() // SAFETY: Match guard checks that the object is a package. + .serialized_module_map() + .keys() + .cloned() + .collect(), + }, + + (ID::Created, _, Some((o, d))) => SuiObjectChange::Created { + sender: tx_data.sender(), + owner: o.owner().clone(), + object_type: o + .struct_tag() + .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + object_id, + version: o.version(), + digest: d, + }, + + (ID::None, Some((i, _)), Some((o, od))) if i.owner() != o.owner() => { + SuiObjectChange::Transferred { + sender: tx_data.sender(), + recipient: o.owner().clone(), + object_type: o + .struct_tag() + .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + object_id, + version: o.version(), + digest: od, + } + } + + (ID::None, Some((i, _)), Some((o, od))) => SuiObjectChange::Mutated { + sender: tx_data.sender(), + owner: o.owner().clone(), + object_type: o + .struct_tag() + .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + object_id, + version: o.version(), + previous_version: i.version(), + digest: od, + }, + + (ID::None, Some((i, _)), None) => SuiObjectChange::Wrapped { + sender: tx_data.sender(), + object_type: i + .struct_tag() + .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + object_id, + version: effects.lamport_version(), + }, + + (ID::Deleted, Some((i, _)), None) => SuiObjectChange::Deleted { + sender: tx_data.sender(), + object_type: i + .struct_tag() + .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + object_id, + version: effects.lamport_version(), + }, + }) + } + + Ok(changes) +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs index 91104075a8512..59670e50c36e1 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs @@ -1,6 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +pub(crate) mod objects; pub(crate) mod package_resolver; pub(crate) mod reader; pub mod system_package_task; diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs b/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs new file mode 100644 index 0000000000000..47021ca8d826d --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs @@ -0,0 +1,64 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::HashMap, sync::Arc}; + +use async_graphql::dataloader::Loader; +use diesel::{BoolExpressionMethods, ExpressionMethods, QueryDsl}; +use sui_indexer_alt_schema::{objects::StoredObject, schema::kv_objects}; +use sui_types::base_types::ObjectID; + +use super::reader::{ReadError, Reader}; + +/// Key for fetching a particular version of an object. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct ObjectVersionKey(pub ObjectID, pub u64); + +#[async_trait::async_trait] +impl Loader for Reader { + type Value = StoredObject; + type Error = Arc; + + async fn load( + &self, + keys: &[ObjectVersionKey], + ) -> Result, Self::Error> { + use kv_objects::dsl as o; + + if keys.is_empty() { + return Ok(HashMap::new()); + } + + let mut conn = self.connect().await.map_err(Arc::new)?; + + let mut query = o::kv_objects.into_boxed(); + + for ObjectVersionKey(id, version) in keys { + query = query.or_filter( + o::object_id + .eq(id.into_bytes()) + .and(o::object_version.eq(*version as i64)), + ); + } + + let objects: Vec = conn.results(query).await.map_err(Arc::new)?; + + let key_to_stored: HashMap<_, _> = objects + .iter() + .map(|stored| { + let id = &stored.object_id[..]; + let version = stored.object_version as u64; + ((id, version), stored) + }) + .collect(); + + Ok(keys + .iter() + .filter_map(|key| { + let slice: &[u8] = key.0.as_ref(); + let stored = *key_to_stored.get(&(slice, key.1))?; + Some((*key, stored.clone())) + }) + .collect()) + } +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/error.rs b/crates/sui-indexer-alt-jsonrpc/src/error.rs index b5a15663097a8..d219a9db9e065 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/error.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/error.rs @@ -8,11 +8,29 @@ //! in the service may return errors in a variety of types. Bodies of JSON-RPC method handlers are //! responsible for assigning an error code for these errors. +use std::fmt::Display; + use jsonrpsee::types::{ error::{INTERNAL_ERROR_CODE, INVALID_PARAMS_CODE}, ErrorObject, }; +/// Macro wrapping a call to an RPC error constructor (above) which adds support for format +/// strings, and immediate early return: +/// +/// rpc_bail!(internal_error("hello, {}", "world")) +/// +/// Becomes +/// +/// return Err(internal_error(format!("hello, {}", "world"))) +macro_rules! rpc_bail { + ($kind:ident ( $fmt:literal $(,$x:expr)* $(,)? ) ) => { + return Err(crate::error::$kind(format!($fmt, $($x),*))) + }; +} + +pub(crate) use rpc_bail; + pub(crate) fn internal_error(err: impl ToString) -> ErrorObject<'static> { ErrorObject::owned(INTERNAL_ERROR_CODE, err.to_string(), None::<()>) } @@ -20,3 +38,7 @@ pub(crate) fn internal_error(err: impl ToString) -> ErrorObject<'static> { pub(crate) fn invalid_params(err: impl ToString) -> ErrorObject<'static> { ErrorObject::owned(INVALID_PARAMS_CODE, err.to_string(), None::<()>) } + +pub(crate) fn pruned(what: impl Display) -> ErrorObject<'static> { + ErrorObject::owned(INVALID_PARAMS_CODE, format!("{what} pruned"), None::<()>) +} diff --git a/crates/sui-indexer-alt-schema/src/objects.rs b/crates/sui-indexer-alt-schema/src/objects.rs index a2472ffe6522d..485d6b8025ec4 100644 --- a/crates/sui-indexer-alt-schema/src/objects.rs +++ b/crates/sui-indexer-alt-schema/src/objects.rs @@ -12,7 +12,7 @@ use sui_types::object::{Object, Owner}; use crate::schema::{coin_balance_buckets, kv_objects, obj_info, obj_versions}; -#[derive(Insertable, Debug, Clone, FieldCount)] +#[derive(Insertable, Debug, Clone, FieldCount, Queryable)] #[diesel(table_name = kv_objects, primary_key(object_id, object_version))] #[diesel(treat_none_as_default_value = false)] pub struct StoredObject { From 1a797055352ea3d5740c31d2c69c1e9e2c8c75db Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sat, 25 Jan 2025 22:14:57 +0000 Subject: [PATCH 07/30] rpc-alt: suix_getLatestSuiSystemState ## Description Implement `suix_getLatestSuiSystemState` from the governance API. This works by fetching the latest version of a particular object, using just the `obj_versions` table. This would not work for arbitrary objects, because `obj_versions` is not updated when an object is deleted or wrapped. This means that when asking for the latest version of an object that is currently deleted or wrapped, we will see the version prior to the deletion/wrap. This should be fine for the system state wrapper and its inner object because these cannot be deleted or wrapped. There are two more simplifications that have been undertaken here, by ignoring consistency and watermarking. Although we do need to implement both of these for GraphQL, we can get away with not doing that yet (for JSONRPC) because our current JSONRPC implementation does neither of these things, and so therefore this should not be a regression. ## Test plan New E2E tests show inspecting the system state after operations that would tweak it (staking, changing the epoch, withdrawing stake): ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- system_state ``` --- .../tests/jsonrpc/governance/system_state.exp | 479 ++++++++++++++++++ .../jsonrpc/governance/system_state.move | 51 ++ .../src/api/governance.rs | 105 +++- .../sui-indexer-alt-jsonrpc/src/data/mod.rs | 1 + .../src/data/object_versions.rs | 62 +++ crates/sui-indexer-alt-schema/src/objects.rs | 2 +- 6 files changed, 696 insertions(+), 4 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.move create mode 100644 crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.exp new file mode 100644 index 0000000000000..b5e87756a4e81 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.exp @@ -0,0 +1,479 @@ +processed 13 tasks + +init: +A: object(0,0) + +task 1, lines 6-10: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "epoch": "0", + "protocolVersion": "70", + "systemStateVersion": "1", + "storageFundTotalObjectStorageRebates": "0", + "storageFundNonRefundableBalance": "0", + "referenceGasPrice": "1000", + "safeMode": false, + "safeModeStorageRewards": "0", + "safeModeComputationRewards": "0", + "safeModeStorageRebates": "0", + "safeModeNonRefundableStorageFee": "0", + "epochStartTimestampMs": "0", + "epochDurationMs": "86400000", + "stakeSubsidyStartEpoch": "0", + "maxValidatorCount": "150", + "minValidatorJoiningStake": "30000000000000000", + "validatorLowStakeThreshold": "20000000000000000", + "validatorVeryLowStakeThreshold": "15000000000000000", + "validatorLowStakeGracePeriod": "7", + "stakeSubsidyBalance": "9949400000000000000", + "stakeSubsidyDistributionCounter": "0", + "stakeSubsidyCurrentDistributionAmount": "1000000000000000", + "stakeSubsidyPeriodLength": "10", + "stakeSubsidyDecreaseRate": 1000, + "totalStake": "20000000000000000", + "activeValidators": [ + { + "suiAddress": "0xda83166d01afd7ddcf8af5f844f45aaa53f48548e5117c23f5a2978cfd422244", + "protocolPubkeyBytes": "qqgbtEP57SCwGrO7tmcKwy/daeoOFwANmrMTm1Qu4jUJRhi2VePz/brF9YAcjmJ7BLOpN8c5Ia7zYzTNmGtGoaUnjoYrbvDG9E05s9antwSmkHAIGsM8mkmeBkSjSBrt", + "networkPubkeyBytes": "ZeETulurG5EpRBoewpF26pyLQtpUqwH1T6LqgugHBIU=", + "workerPubkeyBytes": "3sE4/d+MbOSh9pesKr7b89TSO5gFBuyGUjVa4GldmFU=", + "proofOfPossessionBytes": "sIupbWI7yiRvXM22F2E5sJFRricflowHFu7yqXnzglaAvYTxInm4MSDNAgeMzHyJ", + "name": "validator-0", + "description": "", + "imageUrl": "", + "projectUrl": "", + "netAddress": "/ip4/127.0.0.1/tcp/8000/http", + "p2pAddress": "/ip4/127.0.0.1/udp/8001/http", + "primaryAddress": "/ip4/127.0.0.1/udp/8004/http", + "workerAddress": "/ip4/127.0.0.1/udp/8005/http", + "nextEpochProtocolPubkeyBytes": null, + "nextEpochProofOfPossession": null, + "nextEpochNetworkPubkeyBytes": null, + "nextEpochWorkerPubkeyBytes": null, + "nextEpochNetAddress": null, + "nextEpochP2pAddress": null, + "nextEpochPrimaryAddress": null, + "nextEpochWorkerAddress": null, + "votingPower": "10000", + "operationCapId": "0x03c47745906988d806cf7b623cebf960c79b370bb8b95c3ad24616e2aa29ca7b", + "gasPrice": "1000", + "commissionRate": "200", + "nextEpochStake": "20000000000000000", + "nextEpochGasPrice": "1000", + "nextEpochCommissionRate": "200", + "stakingPoolId": "0x2f4c0b14e06a9dd4724e823b2289e3356b2e987fd0f3435e3e00b616bbec111f", + "stakingPoolActivationEpoch": "0", + "stakingPoolDeactivationEpoch": null, + "stakingPoolSuiBalance": "20000000000000000", + "rewardsPool": "0", + "poolTokenBalance": "20000000000000000", + "pendingStake": "0", + "pendingTotalSuiWithdraw": "0", + "pendingPoolTokenWithdraw": "0", + "exchangeRatesId": "0x94071141bae9afff56a839bc873ae0b51333c8eec79314f79334fa137b317f6a", + "exchangeRatesSize": "1" + } + ], + "pendingActiveValidatorsId": "0x9541f20fb73c0586eececa4e9e872e7af6892c50420c05b9883d6d2865306f77", + "pendingActiveValidatorsSize": "0", + "pendingRemovals": [], + "stakingPoolMappingsId": "0x92aeb4c135bbe12a5d16fa81ef3e1e9211e8a76bbe861bc21aa6010fafe37cf1", + "stakingPoolMappingsSize": "1", + "inactivePoolsId": "0x5a2e52e99e944608af962628eed737de7bcdef3cb52e8177734d6f95a20ced4e", + "inactivePoolsSize": "0", + "validatorCandidatesId": "0xc92f51413cff79cbf74de822357864565fac59ba3079acf20711a6da5c2599da", + "validatorCandidatesSize": "0", + "atRiskValidators": [], + "validatorReportRecords": [] + } +} + +task 2, lines 12-14: +//# programmable --sender A --inputs 1000000000 object(0x5) @validator_0 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui_system::sui_system::request_add_stake(Input(1), Result(0), Input(2)) +events: Event { package_id: sui_system, transaction_module: Identifier("sui_system"), sender: A, type_: StructTag { address: sui_system, module: Identifier("validator"), name: Identifier("StakingRequestEvent"), type_params: [] }, contents: [47, 76, 11, 20, 224, 106, 157, 212, 114, 78, 130, 59, 34, 137, 227, 53, 107, 46, 152, 127, 208, 243, 67, 94, 62, 0, 182, 22, 187, 236, 17, 31, 218, 131, 22, 109, 1, 175, 215, 221, 207, 138, 245, 248, 68, 244, 90, 170, 83, 244, 133, 72, 229, 17, 124, 35, 245, 162, 151, 140, 253, 66, 34, 68, 252, 204, 154, 66, 27, 187, 19, 193, 166, 106, 26, 169, 143, 10, 215, 80, 41, 237, 233, 72, 87, 119, 156, 105, 21, 180, 79, 148, 6, 139, 146, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 202, 154, 59, 0, 0, 0, 0] } +created: object(2,0), object(2,1) +mutated: 0x0000000000000000000000000000000000000000000000000000000000000005, object(0,0) +deleted: object(_) +gas summary: computation_cost: 1000000, storage_cost: 15078400, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, line 16: +//# create-checkpoint +Checkpoint created: 1 + +task 4, lines 18-22: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "epoch": "0", + "protocolVersion": "70", + "systemStateVersion": "2", + "storageFundTotalObjectStorageRebates": "0", + "storageFundNonRefundableBalance": "0", + "referenceGasPrice": "1000", + "safeMode": false, + "safeModeStorageRewards": "0", + "safeModeComputationRewards": "0", + "safeModeStorageRebates": "0", + "safeModeNonRefundableStorageFee": "0", + "epochStartTimestampMs": "0", + "epochDurationMs": "86400000", + "stakeSubsidyStartEpoch": "0", + "maxValidatorCount": "150", + "minValidatorJoiningStake": "30000000000000000", + "validatorLowStakeThreshold": "20000000000000000", + "validatorVeryLowStakeThreshold": "15000000000000000", + "validatorLowStakeGracePeriod": "7", + "stakeSubsidyBalance": "9949400000000000000", + "stakeSubsidyDistributionCounter": "0", + "stakeSubsidyCurrentDistributionAmount": "1000000000000000", + "stakeSubsidyPeriodLength": "10", + "stakeSubsidyDecreaseRate": 1000, + "totalStake": "20000000000000000", + "activeValidators": [ + { + "suiAddress": "0xda83166d01afd7ddcf8af5f844f45aaa53f48548e5117c23f5a2978cfd422244", + "protocolPubkeyBytes": "qqgbtEP57SCwGrO7tmcKwy/daeoOFwANmrMTm1Qu4jUJRhi2VePz/brF9YAcjmJ7BLOpN8c5Ia7zYzTNmGtGoaUnjoYrbvDG9E05s9antwSmkHAIGsM8mkmeBkSjSBrt", + "networkPubkeyBytes": "ZeETulurG5EpRBoewpF26pyLQtpUqwH1T6LqgugHBIU=", + "workerPubkeyBytes": "3sE4/d+MbOSh9pesKr7b89TSO5gFBuyGUjVa4GldmFU=", + "proofOfPossessionBytes": "sIupbWI7yiRvXM22F2E5sJFRricflowHFu7yqXnzglaAvYTxInm4MSDNAgeMzHyJ", + "name": "validator-0", + "description": "", + "imageUrl": "", + "projectUrl": "", + "netAddress": "/ip4/127.0.0.1/tcp/8000/http", + "p2pAddress": "/ip4/127.0.0.1/udp/8001/http", + "primaryAddress": "/ip4/127.0.0.1/udp/8004/http", + "workerAddress": "/ip4/127.0.0.1/udp/8005/http", + "nextEpochProtocolPubkeyBytes": null, + "nextEpochProofOfPossession": null, + "nextEpochNetworkPubkeyBytes": null, + "nextEpochWorkerPubkeyBytes": null, + "nextEpochNetAddress": null, + "nextEpochP2pAddress": null, + "nextEpochPrimaryAddress": null, + "nextEpochWorkerAddress": null, + "votingPower": "10000", + "operationCapId": "0x03c47745906988d806cf7b623cebf960c79b370bb8b95c3ad24616e2aa29ca7b", + "gasPrice": "1000", + "commissionRate": "200", + "nextEpochStake": "20000001000000000", + "nextEpochGasPrice": "1000", + "nextEpochCommissionRate": "200", + "stakingPoolId": "0x2f4c0b14e06a9dd4724e823b2289e3356b2e987fd0f3435e3e00b616bbec111f", + "stakingPoolActivationEpoch": "0", + "stakingPoolDeactivationEpoch": null, + "stakingPoolSuiBalance": "20000000000000000", + "rewardsPool": "0", + "poolTokenBalance": "20000000000000000", + "pendingStake": "1000000000", + "pendingTotalSuiWithdraw": "0", + "pendingPoolTokenWithdraw": "0", + "exchangeRatesId": "0x94071141bae9afff56a839bc873ae0b51333c8eec79314f79334fa137b317f6a", + "exchangeRatesSize": "1" + } + ], + "pendingActiveValidatorsId": "0x9541f20fb73c0586eececa4e9e872e7af6892c50420c05b9883d6d2865306f77", + "pendingActiveValidatorsSize": "0", + "pendingRemovals": [], + "stakingPoolMappingsId": "0x92aeb4c135bbe12a5d16fa81ef3e1e9211e8a76bbe861bc21aa6010fafe37cf1", + "stakingPoolMappingsSize": "1", + "inactivePoolsId": "0x5a2e52e99e944608af962628eed737de7bcdef3cb52e8177734d6f95a20ced4e", + "inactivePoolsSize": "0", + "validatorCandidatesId": "0xc92f51413cff79cbf74de822357864565fac59ba3079acf20711a6da5c2599da", + "validatorCandidatesSize": "0", + "atRiskValidators": [], + "validatorReportRecords": [] + } +} + +task 6, line 26: +//# advance-epoch +Epoch advanced: 0 + +task 7, lines 28-32: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 2, + "result": { + "epoch": "1", + "protocolVersion": "70", + "systemStateVersion": "2", + "storageFundTotalObjectStorageRebates": "15078400", + "storageFundNonRefundableBalance": "0", + "referenceGasPrice": "1000", + "safeMode": false, + "safeModeStorageRewards": "0", + "safeModeComputationRewards": "0", + "safeModeStorageRebates": "0", + "safeModeNonRefundableStorageFee": "0", + "epochStartTimestampMs": "1", + "epochDurationMs": "86400000", + "stakeSubsidyStartEpoch": "0", + "maxValidatorCount": "150", + "minValidatorJoiningStake": "30000000000000000", + "validatorLowStakeThreshold": "20000000000000000", + "validatorVeryLowStakeThreshold": "15000000000000000", + "validatorLowStakeGracePeriod": "7", + "stakeSubsidyBalance": "9949400000000000000", + "stakeSubsidyDistributionCounter": "0", + "stakeSubsidyCurrentDistributionAmount": "1000000000000000", + "stakeSubsidyPeriodLength": "10", + "stakeSubsidyDecreaseRate": 1000, + "totalStake": "20000001001000000", + "activeValidators": [ + { + "suiAddress": "0xda83166d01afd7ddcf8af5f844f45aaa53f48548e5117c23f5a2978cfd422244", + "protocolPubkeyBytes": "qqgbtEP57SCwGrO7tmcKwy/daeoOFwANmrMTm1Qu4jUJRhi2VePz/brF9YAcjmJ7BLOpN8c5Ia7zYzTNmGtGoaUnjoYrbvDG9E05s9antwSmkHAIGsM8mkmeBkSjSBrt", + "networkPubkeyBytes": "ZeETulurG5EpRBoewpF26pyLQtpUqwH1T6LqgugHBIU=", + "workerPubkeyBytes": "3sE4/d+MbOSh9pesKr7b89TSO5gFBuyGUjVa4GldmFU=", + "proofOfPossessionBytes": "sIupbWI7yiRvXM22F2E5sJFRricflowHFu7yqXnzglaAvYTxInm4MSDNAgeMzHyJ", + "name": "validator-0", + "description": "", + "imageUrl": "", + "projectUrl": "", + "netAddress": "/ip4/127.0.0.1/tcp/8000/http", + "p2pAddress": "/ip4/127.0.0.1/udp/8001/http", + "primaryAddress": "/ip4/127.0.0.1/udp/8004/http", + "workerAddress": "/ip4/127.0.0.1/udp/8005/http", + "nextEpochProtocolPubkeyBytes": null, + "nextEpochProofOfPossession": null, + "nextEpochNetworkPubkeyBytes": null, + "nextEpochWorkerPubkeyBytes": null, + "nextEpochNetAddress": null, + "nextEpochP2pAddress": null, + "nextEpochPrimaryAddress": null, + "nextEpochWorkerAddress": null, + "votingPower": "10000", + "operationCapId": "0x03c47745906988d806cf7b623cebf960c79b370bb8b95c3ad24616e2aa29ca7b", + "gasPrice": "1000", + "commissionRate": "200", + "nextEpochStake": "20000001001000000", + "nextEpochGasPrice": "1000", + "nextEpochCommissionRate": "200", + "stakingPoolId": "0x2f4c0b14e06a9dd4724e823b2289e3356b2e987fd0f3435e3e00b616bbec111f", + "stakingPoolActivationEpoch": "0", + "stakingPoolDeactivationEpoch": null, + "stakingPoolSuiBalance": "20000001001000000", + "rewardsPool": "980000", + "poolTokenBalance": "20000001000019999", + "pendingStake": "0", + "pendingTotalSuiWithdraw": "0", + "pendingPoolTokenWithdraw": "0", + "exchangeRatesId": "0x94071141bae9afff56a839bc873ae0b51333c8eec79314f79334fa137b317f6a", + "exchangeRatesSize": "2" + } + ], + "pendingActiveValidatorsId": "0x9541f20fb73c0586eececa4e9e872e7af6892c50420c05b9883d6d2865306f77", + "pendingActiveValidatorsSize": "0", + "pendingRemovals": [], + "stakingPoolMappingsId": "0x92aeb4c135bbe12a5d16fa81ef3e1e9211e8a76bbe861bc21aa6010fafe37cf1", + "stakingPoolMappingsSize": "1", + "inactivePoolsId": "0x5a2e52e99e944608af962628eed737de7bcdef3cb52e8177734d6f95a20ced4e", + "inactivePoolsSize": "0", + "validatorCandidatesId": "0xc92f51413cff79cbf74de822357864565fac59ba3079acf20711a6da5c2599da", + "validatorCandidatesSize": "0", + "atRiskValidators": [], + "validatorReportRecords": [] + } +} + +task 8, lines 34-35: +//# programmable --sender A --inputs object(0x5) object(2,1) +//> 0: sui_system::sui_system::request_withdraw_stake(Input(0), Input(1)) +events: Event { package_id: sui_system, transaction_module: Identifier("sui_system"), sender: A, type_: StructTag { address: sui_system, module: Identifier("validator"), name: Identifier("UnstakingRequestEvent"), type_params: [] }, contents: [47, 76, 11, 20, 224, 106, 157, 212, 114, 78, 130, 59, 34, 137, 227, 53, 107, 46, 152, 127, 208, 243, 67, 94, 62, 0, 182, 22, 187, 236, 17, 31, 218, 131, 22, 109, 1, 175, 215, 221, 207, 138, 245, 248, 68, 244, 90, 170, 83, 244, 133, 72, 229, 17, 124, 35, 245, 162, 151, 140, 253, 66, 34, 68, 252, 204, 154, 66, 27, 187, 19, 193, 166, 106, 26, 169, 143, 10, 215, 80, 41, 237, 233, 72, 87, 119, 156, 105, 21, 180, 79, 148, 6, 139, 146, 30, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 202, 154, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] } +created: object(8,0) +mutated: 0x0000000000000000000000000000000000000000000000000000000000000005, object(0,0), object(2,0) +deleted: object(2,1) +gas summary: computation_cost: 1000000, storage_cost: 14774400, storage_rebate: 14927616, non_refundable_storage_fee: 150784 + +task 9, line 37: +//# create-checkpoint +Checkpoint created: 3 + +task 10, lines 39-43: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 3, + "result": { + "epoch": "1", + "protocolVersion": "70", + "systemStateVersion": "2", + "storageFundTotalObjectStorageRebates": "15078400", + "storageFundNonRefundableBalance": "0", + "referenceGasPrice": "1000", + "safeMode": false, + "safeModeStorageRewards": "0", + "safeModeComputationRewards": "0", + "safeModeStorageRebates": "0", + "safeModeNonRefundableStorageFee": "0", + "epochStartTimestampMs": "1", + "epochDurationMs": "86400000", + "stakeSubsidyStartEpoch": "0", + "maxValidatorCount": "150", + "minValidatorJoiningStake": "30000000000000000", + "validatorLowStakeThreshold": "20000000000000000", + "validatorVeryLowStakeThreshold": "15000000000000000", + "validatorLowStakeGracePeriod": "7", + "stakeSubsidyBalance": "9949400000000000000", + "stakeSubsidyDistributionCounter": "0", + "stakeSubsidyCurrentDistributionAmount": "1000000000000000", + "stakeSubsidyPeriodLength": "10", + "stakeSubsidyDecreaseRate": 1000, + "totalStake": "20000001001000000", + "activeValidators": [ + { + "suiAddress": "0xda83166d01afd7ddcf8af5f844f45aaa53f48548e5117c23f5a2978cfd422244", + "protocolPubkeyBytes": "qqgbtEP57SCwGrO7tmcKwy/daeoOFwANmrMTm1Qu4jUJRhi2VePz/brF9YAcjmJ7BLOpN8c5Ia7zYzTNmGtGoaUnjoYrbvDG9E05s9antwSmkHAIGsM8mkmeBkSjSBrt", + "networkPubkeyBytes": "ZeETulurG5EpRBoewpF26pyLQtpUqwH1T6LqgugHBIU=", + "workerPubkeyBytes": "3sE4/d+MbOSh9pesKr7b89TSO5gFBuyGUjVa4GldmFU=", + "proofOfPossessionBytes": "sIupbWI7yiRvXM22F2E5sJFRricflowHFu7yqXnzglaAvYTxInm4MSDNAgeMzHyJ", + "name": "validator-0", + "description": "", + "imageUrl": "", + "projectUrl": "", + "netAddress": "/ip4/127.0.0.1/tcp/8000/http", + "p2pAddress": "/ip4/127.0.0.1/udp/8001/http", + "primaryAddress": "/ip4/127.0.0.1/udp/8004/http", + "workerAddress": "/ip4/127.0.0.1/udp/8005/http", + "nextEpochProtocolPubkeyBytes": null, + "nextEpochProofOfPossession": null, + "nextEpochNetworkPubkeyBytes": null, + "nextEpochWorkerPubkeyBytes": null, + "nextEpochNetAddress": null, + "nextEpochP2pAddress": null, + "nextEpochPrimaryAddress": null, + "nextEpochWorkerAddress": null, + "votingPower": "10000", + "operationCapId": "0x03c47745906988d806cf7b623cebf960c79b370bb8b95c3ad24616e2aa29ca7b", + "gasPrice": "1000", + "commissionRate": "200", + "nextEpochStake": "20000000001000000", + "nextEpochGasPrice": "1000", + "nextEpochCommissionRate": "200", + "stakingPoolId": "0x2f4c0b14e06a9dd4724e823b2289e3356b2e987fd0f3435e3e00b616bbec111f", + "stakingPoolActivationEpoch": "0", + "stakingPoolDeactivationEpoch": null, + "stakingPoolSuiBalance": "20000001001000000", + "rewardsPool": "980000", + "poolTokenBalance": "20000001000019999", + "pendingStake": "0", + "pendingTotalSuiWithdraw": "1000000000", + "pendingPoolTokenWithdraw": "999999999", + "exchangeRatesId": "0x94071141bae9afff56a839bc873ae0b51333c8eec79314f79334fa137b317f6a", + "exchangeRatesSize": "2" + } + ], + "pendingActiveValidatorsId": "0x9541f20fb73c0586eececa4e9e872e7af6892c50420c05b9883d6d2865306f77", + "pendingActiveValidatorsSize": "0", + "pendingRemovals": [], + "stakingPoolMappingsId": "0x92aeb4c135bbe12a5d16fa81ef3e1e9211e8a76bbe861bc21aa6010fafe37cf1", + "stakingPoolMappingsSize": "1", + "inactivePoolsId": "0x5a2e52e99e944608af962628eed737de7bcdef3cb52e8177734d6f95a20ced4e", + "inactivePoolsSize": "0", + "validatorCandidatesId": "0xc92f51413cff79cbf74de822357864565fac59ba3079acf20711a6da5c2599da", + "validatorCandidatesSize": "0", + "atRiskValidators": [], + "validatorReportRecords": [] + } +} + +task 11, line 45: +//# advance-epoch +Epoch advanced: 1 + +task 12, lines 47-51: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 4, + "result": { + "epoch": "2", + "protocolVersion": "70", + "systemStateVersion": "2", + "storageFundTotalObjectStorageRebates": "14774400", + "storageFundNonRefundableBalance": "150784", + "referenceGasPrice": "1000", + "safeMode": false, + "safeModeStorageRewards": "0", + "safeModeComputationRewards": "0", + "safeModeStorageRebates": "0", + "safeModeNonRefundableStorageFee": "0", + "epochStartTimestampMs": "1", + "epochDurationMs": "86400000", + "stakeSubsidyStartEpoch": "0", + "maxValidatorCount": "150", + "minValidatorJoiningStake": "30000000000000000", + "validatorLowStakeThreshold": "20000000000000000", + "validatorVeryLowStakeThreshold": "15000000000000000", + "validatorLowStakeGracePeriod": "7", + "stakeSubsidyBalance": "9949400000000000000", + "stakeSubsidyDistributionCounter": "0", + "stakeSubsidyCurrentDistributionAmount": "1000000000000000", + "stakeSubsidyPeriodLength": "10", + "stakeSubsidyDecreaseRate": 1000, + "totalStake": "20000000002000000", + "activeValidators": [ + { + "suiAddress": "0xda83166d01afd7ddcf8af5f844f45aaa53f48548e5117c23f5a2978cfd422244", + "protocolPubkeyBytes": "qqgbtEP57SCwGrO7tmcKwy/daeoOFwANmrMTm1Qu4jUJRhi2VePz/brF9YAcjmJ7BLOpN8c5Ia7zYzTNmGtGoaUnjoYrbvDG9E05s9antwSmkHAIGsM8mkmeBkSjSBrt", + "networkPubkeyBytes": "ZeETulurG5EpRBoewpF26pyLQtpUqwH1T6LqgugHBIU=", + "workerPubkeyBytes": "3sE4/d+MbOSh9pesKr7b89TSO5gFBuyGUjVa4GldmFU=", + "proofOfPossessionBytes": "sIupbWI7yiRvXM22F2E5sJFRricflowHFu7yqXnzglaAvYTxInm4MSDNAgeMzHyJ", + "name": "validator-0", + "description": "", + "imageUrl": "", + "projectUrl": "", + "netAddress": "/ip4/127.0.0.1/tcp/8000/http", + "p2pAddress": "/ip4/127.0.0.1/udp/8001/http", + "primaryAddress": "/ip4/127.0.0.1/udp/8004/http", + "workerAddress": "/ip4/127.0.0.1/udp/8005/http", + "nextEpochProtocolPubkeyBytes": null, + "nextEpochProofOfPossession": null, + "nextEpochNetworkPubkeyBytes": null, + "nextEpochWorkerPubkeyBytes": null, + "nextEpochNetAddress": null, + "nextEpochP2pAddress": null, + "nextEpochPrimaryAddress": null, + "nextEpochWorkerAddress": null, + "votingPower": "10000", + "operationCapId": "0x03c47745906988d806cf7b623cebf960c79b370bb8b95c3ad24616e2aa29ca7b", + "gasPrice": "1000", + "commissionRate": "200", + "nextEpochStake": "20000000002000000", + "nextEpochGasPrice": "1000", + "nextEpochCommissionRate": "200", + "stakingPoolId": "0x2f4c0b14e06a9dd4724e823b2289e3356b2e987fd0f3435e3e00b616bbec111f", + "stakingPoolActivationEpoch": "0", + "stakingPoolDeactivationEpoch": null, + "stakingPoolSuiBalance": "20000000002000000", + "rewardsPool": "1960000", + "poolTokenBalance": "20000000000039999", + "pendingStake": "0", + "pendingTotalSuiWithdraw": "0", + "pendingPoolTokenWithdraw": "0", + "exchangeRatesId": "0x94071141bae9afff56a839bc873ae0b51333c8eec79314f79334fa137b317f6a", + "exchangeRatesSize": "3" + } + ], + "pendingActiveValidatorsId": "0x9541f20fb73c0586eececa4e9e872e7af6892c50420c05b9883d6d2865306f77", + "pendingActiveValidatorsSize": "0", + "pendingRemovals": [], + "stakingPoolMappingsId": "0x92aeb4c135bbe12a5d16fa81ef3e1e9211e8a76bbe861bc21aa6010fafe37cf1", + "stakingPoolMappingsSize": "1", + "inactivePoolsId": "0x5a2e52e99e944608af962628eed737de7bcdef3cb52e8177734d6f95a20ced4e", + "inactivePoolsSize": "0", + "validatorCandidatesId": "0xc92f51413cff79cbf74de822357864565fac59ba3079acf20711a6da5c2599da", + "validatorCandidatesSize": "0", + "atRiskValidators": [], + "validatorReportRecords": [] + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.move new file mode 100644 index 0000000000000..805ebef72717c --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/governance/system_state.move @@ -0,0 +1,51 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --simulator --accounts A + +//# run-jsonrpc +{ + "method": "suix_getLatestSuiSystemState", + "params": [] +} + +//# programmable --sender A --inputs 1000000000 object(0x5) @validator_0 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui_system::sui_system::request_add_stake(Input(1), Result(0), Input(2)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "suix_getLatestSuiSystemState", + "params": [] +} + +//# advance-clock --duration-ns 1000000 + +//# advance-epoch + +//# run-jsonrpc +{ + "method": "suix_getLatestSuiSystemState", + "params": [] +} + +//# programmable --sender A --inputs object(0x5) object(2,1) +//> 0: sui_system::sui_system::request_withdraw_stake(Input(0), Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "suix_getLatestSuiSystemState", + "params": [] +} + +//# advance-epoch + +//# run-jsonrpc +{ + "method": "suix_getLatestSuiSystemState", + "params": [] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs index c142c88eea4af..4a586fb4b802e 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs @@ -3,13 +3,32 @@ use diesel::{ExpressionMethods, QueryDsl}; -use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use jsonrpsee::{ + core::{DeserializeOwned, RpcResult}, + proc_macros::rpc, +}; use sui_indexer_alt_schema::schema::kv_epoch_starts; use sui_open_rpc::Module; use sui_open_rpc_macros::open_rpc; -use sui_types::sui_serde::BigInt; +use sui_types::{ + base_types::ObjectID, + dynamic_field::{derive_dynamic_field_id, Field}, + object::Object, + sui_serde::BigInt, + sui_system_state::{ + sui_system_state_inner_v1::SuiSystemStateInnerV1, + sui_system_state_inner_v2::SuiSystemStateInnerV2, + sui_system_state_summary::SuiSystemStateSummary, SuiSystemStateTrait, + SuiSystemStateWrapper, + }, + TypeTag, SUI_SYSTEM_STATE_OBJECT_ID, +}; -use crate::{context::Context, error::internal_error}; +use crate::{ + context::Context, + data::{object_versions::LatestObjectKey, objects::ObjectVersionKey}, + error::{internal_error, rpc_bail}, +}; use super::rpc_module::RpcModule; @@ -19,6 +38,10 @@ trait GovernanceApi { /// Return the reference gas price for the network as of the latest epoch. #[method(name = "getReferenceGasPrice")] async fn get_reference_gas_price(&self) -> RpcResult>; + + /// Return a summary of the latest version of the Sui System State object (0x5), on-chain. + #[method(name = "getLatestSuiSystemState")] + async fn get_latest_sui_system_state(&self) -> RpcResult; } pub(crate) struct Governance(pub Context); @@ -41,6 +64,82 @@ impl GovernanceApiServer for Governance { Ok((rgp as u64).into()) } + + async fn get_latest_sui_system_state(&self) -> RpcResult { + let Self(ctx) = self; + + /// Fetch the latest version of the object at ID `object_id`, and deserialize its contents + /// as a Rust type `T`, assuming that it is a Move object (not a package). + /// + /// This function extracts the common parts of object loading for this API, but it does not + /// generalize beyond that, because: + /// + /// - It assumes that the objects being loaded are never deleted or wrapped (because it + /// loads using `LatestObjectKey` directly without checking the live object set). + /// + /// - It first fetches one record from `obj_versions` and then fetches its contents. It is + /// easy to misuse this API to fetch multiple objects in sequence, in a loop, rather than + /// fetching them concurrently. + async fn fetch_latest( + ctx: &Context, + object_id: ObjectID, + ) -> Result { + let id_display = object_id.to_canonical_display(/* with_prefix */ true); + let loader = ctx.loader(); + + let latest_version = loader + .load_one(LatestObjectKey(object_id)) + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Failed to load latest version for {id_display}"))?; + + let stored = loader + .load_one(ObjectVersionKey( + object_id, + latest_version.object_version as u64, + )) + .await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Failed to load object for {id_display}"))? + .serialized_object + .ok_or_else(|| format!("Failed to load contents for {id_display}"))?; + + let object: Object = bcs::from_bytes(&stored).map_err(|e| e.to_string())?; + + let move_object = object + .data + .try_as_move() + .ok_or_else(|| format!("{id_display} is not a Move object"))?; + + bcs::from_bytes(move_object.contents()) + .map_err(|e| format!("Failed to deserialize contents for {id_display}: {e}")) + } + + let wrapper: SuiSystemStateWrapper = fetch_latest(ctx, SUI_SYSTEM_STATE_OBJECT_ID) + .await + .map_err(internal_error)?; + + let inner_id = derive_dynamic_field_id( + SUI_SYSTEM_STATE_OBJECT_ID, + &TypeTag::U64, + &bcs::to_bytes(&wrapper.version).map_err(internal_error)?, + ) + .map_err(internal_error)?; + + Ok(match wrapper.version { + 1 => fetch_latest::>(ctx, inner_id) + .await + .map_err(internal_error)? + .value + .into_sui_system_state_summary(), + 2 => fetch_latest::>(ctx, inner_id) + .await + .map_err(internal_error)? + .value + .into_sui_system_state_summary(), + v => rpc_bail!(internal_error("Unexpected inner system state version: {v}")), + }) + } } impl RpcModule for Governance { diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs index 59670e50c36e1..6fa3ba88f7fa7 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/mod.rs @@ -1,6 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +pub(crate) mod object_versions; pub(crate) mod objects; pub(crate) mod package_resolver; pub(crate) mod reader; diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs b/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs new file mode 100644 index 0000000000000..bb66c5331f8e9 --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs @@ -0,0 +1,62 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; + +use async_graphql::dataloader::Loader; +use diesel::{ExpressionMethods, QueryDsl}; +use sui_indexer_alt_schema::{objects::StoredObjVersion, schema::obj_versions}; +use sui_types::base_types::ObjectID; + +use super::reader::{ReadError, Reader}; + +/// Key for fetching the latest version of an object, not accounting for deletions or wraps. If the +/// object has been deleted or wrapped, the version before the delete/wrap is returned. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct LatestObjectKey(pub ObjectID); + +#[async_trait::async_trait] +impl Loader for Reader { + type Value = StoredObjVersion; + type Error = Arc; + + async fn load( + &self, + keys: &[LatestObjectKey], + ) -> Result, Self::Error> { + use obj_versions::dsl as v; + + if keys.is_empty() { + return Ok(HashMap::new()); + } + + let mut conn = self.connect().await.map_err(Arc::new)?; + + let ids: BTreeSet<_> = keys.iter().map(|k| k.0.into_bytes()).collect(); + let obj_versions: Vec = conn + .results( + v::obj_versions + .filter(v::object_id.eq_any(ids)) + .distinct_on(v::object_id) + .order((v::object_id, v::object_version.desc())), + ) + .await + .map_err(Arc::new)?; + + let id_to_stored: HashMap<_, _> = obj_versions + .into_iter() + .map(|stored| (stored.object_id.clone(), stored)) + .collect(); + + Ok(keys + .iter() + .filter_map(|key| { + let slice: &[u8] = key.0.as_ref(); + Some((*key, id_to_stored.get(slice).cloned()?)) + }) + .collect()) + } +} diff --git a/crates/sui-indexer-alt-schema/src/objects.rs b/crates/sui-indexer-alt-schema/src/objects.rs index 485d6b8025ec4..a23115348d561 100644 --- a/crates/sui-indexer-alt-schema/src/objects.rs +++ b/crates/sui-indexer-alt-schema/src/objects.rs @@ -21,7 +21,7 @@ pub struct StoredObject { pub serialized_object: Option>, } -#[derive(Insertable, Debug, Clone, FieldCount)] +#[derive(Insertable, Selectable, Debug, Clone, FieldCount, Queryable)] #[diesel(table_name = obj_versions, primary_key(object_id, object_version))] pub struct StoredObjVersion { pub object_id: Vec, From d50d78a00474c744909a3cdde468108c5d62c179 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 26 Jan 2025 15:59:40 +0000 Subject: [PATCH 08/30] easy(indexer-alt): re-use `read_config` for benchmark command ## Description There's a helper function for reading configs that we can use. ## Test plan CI --- crates/sui-indexer-alt/src/main.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/sui-indexer-alt/src/main.rs b/crates/sui-indexer-alt/src/main.rs index 583619933fc1e..5e76e2b7566d1 100644 --- a/crates/sui-indexer-alt/src/main.rs +++ b/crates/sui-indexer-alt/src/main.rs @@ -109,13 +109,7 @@ async fn main() -> Result<()> { benchmark_args, config, } => { - let config_contents = fs::read_to_string(config) - .await - .context("failed to read configuration TOML file")?; - - let indexer_config: IndexerConfig = toml::from_str(&config_contents) - .context("Failed to parse configuration TOML file.")?; - + let indexer_config = read_config(&config).await?; sui_indexer_alt::benchmark::run_benchmark(args.db_args, benchmark_args, indexer_config) .await?; } From 4d8fe4c3c5c53911f7474aa0d7c5d1193d5092c0 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 26 Jan 2025 16:01:15 +0000 Subject: [PATCH 09/30] fix(indexer-alt): check for extra fields on pruner config ## Description Missed a check in the config boilerplate for unrecognised fields in the pruner config. Spotted while adding configs to the RPC. ## Test plan CI --- crates/sui-indexer-alt/src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/sui-indexer-alt/src/config.rs b/crates/sui-indexer-alt/src/config.rs index ff9a7a5e1a0a6..23b37d6aa5926 100644 --- a/crates/sui-indexer-alt/src/config.rs +++ b/crates/sui-indexer-alt/src/config.rs @@ -230,6 +230,7 @@ impl CommitterLayer { impl PrunerLayer { pub fn finish(self, base: PrunerConfig) -> PrunerConfig { + check_extra("pruner", self.extra); PrunerConfig { interval_ms: self.interval_ms.unwrap_or(base.interval_ms), delay_ms: self.delay_ms.unwrap_or(base.delay_ms), From 8935f25b4af5ebd2140b1590f3fbad388f5758a8 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 26 Jan 2025 16:17:20 +0000 Subject: [PATCH 10/30] rpc-alt: add configs ## Description Add a TOML config system to the RPC service, using the same pattern we used for `sui-indexer-alt`. This will be used to configure page sizes and other limits for more complex RPC methods. ## Test plan Existing tests + test the new `generate-config` command: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests sui$ cargo run -p sui-indexer-alt-jsonrpc -- generate-config [transactions] default-page-size = 50 max-page-size = 100 ``` --- Cargo.lock | 3 + .../sui-indexer-alt-e2e-tests/tests/tests.rs | 5 +- crates/sui-indexer-alt-jsonrpc/Cargo.toml | 3 + .../src/api/transactions.rs | 20 +++++ crates/sui-indexer-alt-jsonrpc/src/args.rs | 31 ++++++-- crates/sui-indexer-alt-jsonrpc/src/config.rs | 77 +++++++++++++++++++ crates/sui-indexer-alt-jsonrpc/src/lib.rs | 12 ++- crates/sui-indexer-alt-jsonrpc/src/main.rs | 75 ++++++++++++------ 8 files changed, 195 insertions(+), 31 deletions(-) create mode 100644 crates/sui-indexer-alt-jsonrpc/src/config.rs diff --git a/Cargo.lock b/Cargo.lock index c52ffe19b26f4..684ef61b28deb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14149,7 +14149,9 @@ dependencies = [ "pin-project-lite", "prometheus", "reqwest 0.12.9", + "serde", "serde_json", + "sui-default-config", "sui-indexer-alt-metrics", "sui-indexer-alt-schema", "sui-json-rpc-types", @@ -14162,6 +14164,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-util 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.7.4", "tower-layer", "tracing", "url", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/tests.rs b/crates/sui-indexer-alt-e2e-tests/tests/tests.rs index 48bffaadbd027..aec5f7ac8b04c 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/tests.rs +++ b/crates/sui-indexer-alt-e2e-tests/tests/tests.rs @@ -21,7 +21,7 @@ use serde_json::{json, Value}; use sui_indexer_alt::{config::IndexerConfig, start_indexer}; use sui_indexer_alt_framework::{ingestion::ClientArgs, schema::watermarks, IndexerArgs}; use sui_indexer_alt_jsonrpc::{ - data::system_package_task::SystemPackageTaskArgs, start_rpc, RpcArgs, + config::RpcConfig, data::system_package_task::SystemPackageTaskArgs, start_rpc, RpcArgs, }; use sui_pg_db::{ temp::{get_available_port, TempDb}, @@ -96,6 +96,8 @@ impl OffchainCluster { ..Default::default() }; + let rpc_config = RpcConfig::example(); + // This configuration controls how often the RPC service checks for changes to system // packages. The default polling interval is probably too slow for changes to get picked // up, so tests that rely on this behaviour will always fail, but this is better than flaky @@ -119,6 +121,7 @@ impl OffchainCluster { db_args, rpc_args, system_package_task_args, + rpc_config, ®istry, cancel.child_token(), ) diff --git a/crates/sui-indexer-alt-jsonrpc/Cargo.toml b/crates/sui-indexer-alt-jsonrpc/Cargo.toml index 013a70325b118..735874400d606 100644 --- a/crates/sui-indexer-alt-jsonrpc/Cargo.toml +++ b/crates/sui-indexer-alt-jsonrpc/Cargo.toml @@ -23,17 +23,20 @@ futures.workspace = true jsonrpsee = { workspace = true, features = ["macros", "server"] } pin-project-lite.workspace = true prometheus.workspace = true +serde.workspace = true serde_json.workspace = true telemetry-subscribers.workspace = true thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true +toml.workspace = true tower-layer.workspace = true tracing.workspace = true url.workspace = true move-core-types.workspace = true +sui-default-config.workspace = true sui-indexer-alt-metrics.workspace = true sui-indexer-alt-schema.workspace = true sui-json-rpc-types.workspace = true diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs index 49c237a700f46..7af645e48014f 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs @@ -7,6 +7,7 @@ use anyhow::anyhow; use futures::future::OptionFuture; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use move_core_types::annotated_value::{MoveDatatypeLayout, MoveTypeLayout}; +use serde::{Deserialize, Serialize}; use sui_indexer_alt_schema::transactions::{ BalanceChange, StoredTransaction, StoredTxBalanceChange, }; @@ -57,6 +58,16 @@ trait TransactionsApi { pub(crate) struct Transactions(pub Context); +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TransactionsConfig { + /// The default page size limit when querying transactions, if none is provided. + pub default_page_size: usize, + + /// The largest acceptable page size when querying transactions. Requesting a page larger than + /// this is a user error. + pub max_page_size: usize, +} + #[derive(thiserror::Error, Debug)] pub(crate) enum Error { #[error("Transaction not found: {0}")] @@ -161,6 +172,15 @@ impl RpcModule for Transactions { } } +impl Default for TransactionsConfig { + fn default() -> Self { + Self { + default_page_size: 50, + max_page_size: 100, + } + } +} + /// Extract a representation of the transaction's input data from the stored form. async fn input_response( ctx: &Context, diff --git a/crates/sui-indexer-alt-jsonrpc/src/args.rs b/crates/sui-indexer-alt-jsonrpc/src/args.rs index ba1112fe626c8..1f6d370c28e5c 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/args.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/args.rs @@ -1,6 +1,8 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use std::path::PathBuf; + use sui_indexer_alt_metrics::MetricsArgs; use sui_pg_db::DbArgs; @@ -11,12 +13,29 @@ pub struct Args { #[command(flatten)] pub db_args: DbArgs, - #[command(flatten)] - pub rpc_args: RpcArgs, + #[command(subcommand)] + pub command: Command, +} - #[command(flatten)] - pub system_package_task_args: SystemPackageTaskArgs, +#[derive(clap::Subcommand, Debug, Clone)] +pub enum Command { + /// Run the RPC service. + Rpc { + #[command(flatten)] + rpc_args: RpcArgs, - #[command(flatten)] - pub metrics_args: MetricsArgs, + #[command(flatten)] + system_package_task_args: SystemPackageTaskArgs, + + #[command(flatten)] + metrics_args: MetricsArgs, + + /// Path to the RPC's configuration TOML file. If one is not provided, the default values for + /// the configuration will be set. + #[arg(long)] + config: Option, + }, + + /// Output the contents of the default configuration to STDOUT. + GenerateConfig, } diff --git a/crates/sui-indexer-alt-jsonrpc/src/config.rs b/crates/sui-indexer-alt-jsonrpc/src/config.rs new file mode 100644 index 0000000000000..d689f4a5a3445 --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/config.rs @@ -0,0 +1,77 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use std::mem; + +use sui_default_config::DefaultConfig; +use tracing::warn; + +use crate::api::transactions::TransactionsConfig; + +#[DefaultConfig] +#[derive(Clone, Default, Debug)] +pub struct RpcConfig { + /// Configuration for transaction-related RPC methods. + pub transactions: TransactionsLayer, + + #[serde(flatten)] + pub extra: toml::Table, +} + +#[DefaultConfig] +#[derive(Clone, Default, Debug)] +pub struct TransactionsLayer { + pub default_page_size: Option, + pub max_page_size: Option, + + #[serde(flatten)] + pub extra: toml::Table, +} + +impl RpcConfig { + /// Generate an example configuration, suitable for demonstrating the fields available to + /// configure. + pub fn example() -> Self { + Self { + transactions: TransactionsConfig::default().into(), + extra: Default::default(), + } + } + + pub fn finish(mut self) -> RpcConfig { + check_extra("top-level", mem::take(&mut self.extra)); + self + } +} + +impl TransactionsLayer { + pub fn finish(self, base: TransactionsConfig) -> TransactionsConfig { + check_extra("transactions", self.extra); + TransactionsConfig { + default_page_size: self.default_page_size.unwrap_or(base.default_page_size), + max_page_size: self.max_page_size.unwrap_or(base.max_page_size), + } + } +} + +impl From for TransactionsLayer { + fn from(config: TransactionsConfig) -> Self { + Self { + default_page_size: Some(config.default_page_size), + max_page_size: Some(config.max_page_size), + extra: Default::default(), + } + } +} + +/// Check whether there are any unrecognized extra fields and if so, warn about them. +fn check_extra(pos: &str, extra: toml::Table) { + if !extra.is_empty() { + warn!( + "Found unrecognized {pos} field{} which will be ignored. This could be \ + because of a typo, or because it was introduced in a newer version of the indexer:\n{}", + if extra.len() != 1 { "s" } else { "" }, + extra, + ) + } +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/lib.rs b/crates/sui-indexer-alt-jsonrpc/src/lib.rs index 15e318cd0defb..16ee86298163a 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/lib.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/lib.rs @@ -6,7 +6,8 @@ use std::sync::Arc; use anyhow::Context as _; use api::rpc_module::RpcModule; -use api::transactions::Transactions; +use api::transactions::{Transactions, TransactionsConfig}; +use config::RpcConfig; use data::system_package_task::{SystemPackageTask, SystemPackageTaskArgs}; use jsonrpsee::server::{RpcServiceBuilder, ServerBuilder}; use metrics::middleware::MetricsLayer; @@ -25,6 +26,7 @@ use crate::context::Context; mod api; pub mod args; +pub mod config; mod context; pub mod data; mod error; @@ -195,9 +197,17 @@ pub async fn start_rpc( db_args: DbArgs, rpc_args: RpcArgs, system_package_task_args: SystemPackageTaskArgs, + rpc_config: RpcConfig, registry: &Registry, cancel: CancellationToken, ) -> anyhow::Result> { + let RpcConfig { + transactions, + extra: _, + } = rpc_config.finish(); + + let _transactions_config = transactions.finish(TransactionsConfig::default()); + let mut rpc = RpcService::new(rpc_args, registry, cancel.child_token()) .context("Failed to create RPC service")?; diff --git a/crates/sui-indexer-alt-jsonrpc/src/main.rs b/crates/sui-indexer-alt-jsonrpc/src/main.rs index 9bdb6561a4e5b..3abe1aae4d000 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/main.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/main.rs @@ -4,44 +4,73 @@ use anyhow::Context; use clap::Parser; use prometheus::Registry; -use sui_indexer_alt_jsonrpc::{args::Args, start_rpc}; +use sui_indexer_alt_jsonrpc::{ + args::{Args, Command}, + config::RpcConfig, + start_rpc, +}; use sui_indexer_alt_metrics::MetricsService; +use tokio::fs; use tokio_util::sync::CancellationToken; #[tokio::main] async fn main() -> anyhow::Result<()> { - let Args { - db_args, - rpc_args, - system_package_task_args, - metrics_args, - } = Args::parse(); + let args = Args::parse(); // Enable tracing, configured by environment variables. let _guard = telemetry_subscribers::TelemetryConfig::new() .with_env() .init(); - let cancel = CancellationToken::new(); + match args.command { + Command::Rpc { + rpc_args, + system_package_task_args, + metrics_args, + config, + } => { + let rpc_config = if let Some(path) = config { + let contents = fs::read_to_string(path) + .await + .context("Failed to read configuration TOML file")?; - let registry = Registry::new_custom(Some("jsonrpc_alt".into()), None) - .context("Failed to create Prometheus registry.")?; + toml::from_str(&contents).context("Failed to parse configuration TOML file")? + } else { + RpcConfig::default() + }; - let metrics = MetricsService::new(metrics_args, registry, cancel.child_token()); + let cancel = CancellationToken::new(); - let h_rpc = start_rpc( - db_args, - rpc_args, - system_package_task_args, - metrics.registry(), - cancel.child_token(), - ) - .await?; + let registry = Registry::new_custom(Some("jsonrpc_alt".into()), None) + .context("Failed to create Prometheus registry.")?; - let h_metrics = metrics.run().await?; + let metrics = MetricsService::new(metrics_args, registry, cancel.child_token()); + + let h_rpc = start_rpc( + args.db_args, + rpc_args, + system_package_task_args, + rpc_config, + metrics.registry(), + cancel.child_token(), + ) + .await?; + + let h_metrics = metrics.run().await?; + + let _ = h_rpc.await; + cancel.cancel(); + let _ = h_metrics.await; + } + + Command::GenerateConfig => { + let config = RpcConfig::example(); + let config_toml = toml::to_string_pretty(&config) + .context("Failed to serialize default configuration to TOML.")?; + + println!("{config_toml}"); + } + } - let _ = h_rpc.await; - cancel.cancel(); - let _ = h_metrics.await; Ok(()) } From 80198bed01996d973176dbce67bbf0f07197939c Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Sun, 26 Jan 2025 17:48:16 +0000 Subject: [PATCH 11/30] fix(open-rpc): Handle empty doc comment lines ## Description The code that extracts doc comments to generate an OpenRPC schema panics on empty lines. This change fixes that, treating an empty line as a hard line break. ## Test plan Start the new JSONRPC implementation, and inspect its output which contains the OpenRPC schema it offers: ``` sui$ cargo run -p sui-indexer-alt-jsonrpc ``` A follow-up change introduces a doc comment that includes empty lines. --- crates/sui-open-rpc-macros/src/lib.rs | 48 +++++++++++++++++++-------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/crates/sui-open-rpc-macros/src/lib.rs b/crates/sui-open-rpc-macros/src/lib.rs index ccf58a392286a..002a441e8287c 100644 --- a/crates/sui-open-rpc-macros/src/lib.rs +++ b/crates/sui-open-rpc-macros/src/lib.rs @@ -334,21 +334,41 @@ fn respan_token_stream(stream: TokenStream2, span: Span) -> TokenStream2 { .collect() } +/// Find doc comments by looking for #[doc = "..."] attributes. +/// +/// Consecutive attributes are combined together. If there is a leading space, it will be removed, +/// and if there is trailing whitespace it will also be removed. Single newlines in doc comments +/// are replaced by spaces (soft wrapping), but double newlines (an empty line) are preserved. fn extract_doc_comments(attrs: &[Attribute]) -> String { - let s = attrs - .iter() - .filter(|attr| { - attr.path.is_ident("doc") - && match attr.parse_meta() { - Ok(syn::Meta::NameValue(meta)) => matches!(&meta.lit, syn::Lit::Str(_)), - _ => false, - } - }) - .map(|attr| { - let s = attr.tokens.to_string(); - s[4..s.len() - 1].to_string() - }) - .join(" "); + let mut s = String::new(); + let mut sep = ""; + for attr in attrs { + if !attr.path.is_ident("doc") { + continue; + } + + let Ok(syn::Meta::NameValue(meta)) = attr.parse_meta() else { + continue; + }; + + let syn::Lit::Str(lit) = &meta.lit else { + continue; + }; + + let token = lit.value(); + let line = token.strip_prefix(" ").unwrap_or(&token).trim_end(); + + if line.is_empty() { + s.push_str("\n\n"); + sep = ""; + } else { + s.push_str(sep); + sep = " "; + } + + s.push_str(line); + } + unescape(&s).unwrap_or_else(|| panic!("Cannot unescape doc comments : [{s}]")) } From 88390b4c619a693c93ad7a4a2d296c5722b04f81 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Mon, 27 Jan 2025 12:31:03 +0000 Subject: [PATCH 12/30] refactor(rpc-alt): rethink error patterns ## Description Refactor how errors are represented in the new JSONRPC implementation, based on the experience of handling errors so far. Thew new error module encourages the following pattern: - JSONRPC has its own `ErrorObject<'_>` and `RpcResult` -- RPC method implementations return these types. - Each RpcModule defines its own structured error type, using `thiserror` to represent user errors (errors that are the user's fault). - All RpcModules use `anyhow::Error` to represent internal errors, they add `context` to build up a stack explaining what went wrong. - A new type -- `RpcError` -- is used as glue in-between. We implement: - `From for RpcError`, - `From> for ErrorObject<'static>`, - `invalid_params(e: E) -> RpcError` (an explicit conversion function). - The actual implementations of RPC methods are typically fenced off in their own functions that accept the method parameters and a `context`, and returns a `Result>`. The trait implementation for the RpcModule delegates to this function and uses the `?` operator to trigger error conversion. The `error` module also exposes similar helpers to `anyhow`, to add context, return early with an error, create a formatted error, etc. ## Test plan Existing tests (this is a refactor). --- .../transactions/get_transaction_block.exp | 2 +- .../src/api/governance.rs | 185 +++++++------ .../src/api/transactions.rs | 253 +++++++++--------- crates/sui-indexer-alt-jsonrpc/src/error.rs | 115 ++++++-- 4 files changed, 319 insertions(+), 236 deletions(-) diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp index 8e3f127bb4dd6..f0a2c49d3dd0c 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/get_transaction_block.exp @@ -51,7 +51,7 @@ Response: { "id": 1, "error": { "code": -32602, - "message": "Transaction not found: 11111111111111111111111111111111" + "message": "Invalid Params: Transaction 11111111111111111111111111111111 not found" } } diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs index 4a586fb4b802e..8f182740bda6a 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs @@ -1,6 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use anyhow::Context as _; use diesel::{ExpressionMethods, QueryDsl}; use jsonrpsee::{ @@ -27,7 +28,7 @@ use sui_types::{ use crate::{ context::Context, data::{object_versions::LatestObjectKey, objects::ObjectVersionKey}, - error::{internal_error, rpc_bail}, + error::{internal_error, rpc_bail, InternalContext, RpcError}, }; use super::rpc_module::RpcModule; @@ -49,96 +50,11 @@ pub(crate) struct Governance(pub Context); #[async_trait::async_trait] impl GovernanceApiServer for Governance { async fn get_reference_gas_price(&self) -> RpcResult> { - use kv_epoch_starts::dsl as e; - - let Self(ctx) = self; - let mut conn = ctx.reader().connect().await.map_err(internal_error)?; - let rgp: i64 = conn - .first( - e::kv_epoch_starts - .select(e::reference_gas_price) - .order(e::epoch.desc()), - ) - .await - .map_err(internal_error)?; - - Ok((rgp as u64).into()) + Ok(rgp_response(&self.0).await?) } async fn get_latest_sui_system_state(&self) -> RpcResult { - let Self(ctx) = self; - - /// Fetch the latest version of the object at ID `object_id`, and deserialize its contents - /// as a Rust type `T`, assuming that it is a Move object (not a package). - /// - /// This function extracts the common parts of object loading for this API, but it does not - /// generalize beyond that, because: - /// - /// - It assumes that the objects being loaded are never deleted or wrapped (because it - /// loads using `LatestObjectKey` directly without checking the live object set). - /// - /// - It first fetches one record from `obj_versions` and then fetches its contents. It is - /// easy to misuse this API to fetch multiple objects in sequence, in a loop, rather than - /// fetching them concurrently. - async fn fetch_latest( - ctx: &Context, - object_id: ObjectID, - ) -> Result { - let id_display = object_id.to_canonical_display(/* with_prefix */ true); - let loader = ctx.loader(); - - let latest_version = loader - .load_one(LatestObjectKey(object_id)) - .await - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Failed to load latest version for {id_display}"))?; - - let stored = loader - .load_one(ObjectVersionKey( - object_id, - latest_version.object_version as u64, - )) - .await - .map_err(|e| e.to_string())? - .ok_or_else(|| format!("Failed to load object for {id_display}"))? - .serialized_object - .ok_or_else(|| format!("Failed to load contents for {id_display}"))?; - - let object: Object = bcs::from_bytes(&stored).map_err(|e| e.to_string())?; - - let move_object = object - .data - .try_as_move() - .ok_or_else(|| format!("{id_display} is not a Move object"))?; - - bcs::from_bytes(move_object.contents()) - .map_err(|e| format!("Failed to deserialize contents for {id_display}: {e}")) - } - - let wrapper: SuiSystemStateWrapper = fetch_latest(ctx, SUI_SYSTEM_STATE_OBJECT_ID) - .await - .map_err(internal_error)?; - - let inner_id = derive_dynamic_field_id( - SUI_SYSTEM_STATE_OBJECT_ID, - &TypeTag::U64, - &bcs::to_bytes(&wrapper.version).map_err(internal_error)?, - ) - .map_err(internal_error)?; - - Ok(match wrapper.version { - 1 => fetch_latest::>(ctx, inner_id) - .await - .map_err(internal_error)? - .value - .into_sui_system_state_summary(), - 2 => fetch_latest::>(ctx, inner_id) - .await - .map_err(internal_error)? - .value - .into_sui_system_state_summary(), - v => rpc_bail!(internal_error("Unexpected inner system state version: {v}")), - }) + Ok(latest_sui_system_state_response(&self.0).await?) } } @@ -151,3 +67,96 @@ impl RpcModule for Governance { self.into_rpc() } } + +/// Load data and generate response for `getReferenceGasPrice`. +async fn rgp_response(ctx: &Context) -> Result, RpcError> { + use kv_epoch_starts::dsl as e; + + let mut conn = ctx + .reader() + .connect() + .await + .context("Failed to connect to the database")?; + + let rgp: i64 = conn + .first( + e::kv_epoch_starts + .select(e::reference_gas_price) + .order(e::epoch.desc()), + ) + .await + .context("Failed to fetch the reference gas price")?; + + Ok((rgp as u64).into()) +} + +/// Load data and generate response for `getLatestSuiSystemState`. +async fn latest_sui_system_state_response( + ctx: &Context, +) -> Result { + let wrapper: SuiSystemStateWrapper = fetch_latest(ctx, SUI_SYSTEM_STATE_OBJECT_ID) + .await + .internal_context("Failed to fetch system state wrapper object")?; + + let inner_id = derive_dynamic_field_id( + SUI_SYSTEM_STATE_OBJECT_ID, + &TypeTag::U64, + &bcs::to_bytes(&wrapper.version).context("Failed to serialize system state version")?, + ) + .context("Failed to derive inner system state field ID")?; + + Ok(match wrapper.version { + 1 => fetch_latest::>(ctx, inner_id) + .await + .internal_context("Failed to fetch inner system state object")? + .value + .into_sui_system_state_summary(), + 2 => fetch_latest::>(ctx, inner_id) + .await + .internal_context("Failed to fetch inner system state object")? + .value + .into_sui_system_state_summary(), + v => rpc_bail!("Unexpected inner system state version: {v}"), + }) +} + +/// Fetch the latest version of the object at ID `object_id`, and deserialize its contents as a +/// Rust type `T`, assuming that it is a Move object (not a package). +/// +/// This function extracts the common parts of object loading for the latest system state object +/// API, but it does not generalize beyond that, because it assumes that the objects being loaded +/// are never deleted or wrapped and have always existed (because it loads using `LatestObjectKey` +/// directly without checking the live object set). +async fn fetch_latest( + ctx: &Context, + object_id: ObjectID, +) -> Result { + let loader = ctx.loader(); + + let latest_version = loader + .load_one(LatestObjectKey(object_id)) + .await + .context("Failed to load latest version")? + .ok_or_else(|| internal_error!("No latest version found"))?; + + let stored = loader + .load_one(ObjectVersionKey( + object_id, + latest_version.object_version as u64, + )) + .await + .context("Failed to load latest object")? + .ok_or_else(|| internal_error!("No data found"))? + .serialized_object + .ok_or_else(|| internal_error!("No content found"))?; + + let object: Object = + bcs::from_bytes(&stored).context("Failed to deserialize object contents")?; + + let move_object = object + .data + .try_as_move() + .ok_or_else(|| internal_error!("Not a Move object"))?; + + Ok(bcs::from_bytes(move_object.contents()).context("Failed to deserialize Move value")?) +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs index 7af645e48014f..51448cdc2486b 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs @@ -3,7 +3,7 @@ use std::str::FromStr; -use anyhow::anyhow; +use anyhow::Context as _; use futures::future::OptionFuture; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use move_core_types::annotated_value::{MoveDatatypeLayout, MoveTypeLayout}; @@ -22,7 +22,6 @@ use sui_types::{ base_types::{ObjectID, SequenceNumber}, digests::{ObjectDigest, TransactionDigest}, effects::{IDOperation, ObjectChange, TransactionEffects, TransactionEffectsAPI}, - error::SuiError, event::Event, object::Object, signature::GenericSignature, @@ -37,7 +36,7 @@ use crate::{ objects::ObjectVersionKey, transactions::TransactionKey, tx_balance_changes::TxBalanceChangeKey, }, - error::{internal_error, invalid_params, pruned, rpc_bail}, + error::{internal_error, invalid_params, rpc_bail, InternalContext, RpcError}, }; use super::rpc_module::RpcModule; @@ -70,17 +69,17 @@ pub struct TransactionsConfig { #[derive(thiserror::Error, Debug)] pub(crate) enum Error { - #[error("Transaction not found: {0}")] + #[error("Transaction {0} not found")] NotFound(TransactionDigest), - #[error("Error converting to response: {0}")] - Conversion(SuiError), + #[error("Balance changes for transaction {0} have been pruned")] + BalanceChangesPruned(TransactionDigest), - #[error("Error resolving type information: {0}")] - Resolution(anyhow::Error), - - #[error("Deserialization error: {0}")] - Deserialization(#[from] bcs::Error), + #[error( + "Transaction {0} affected object {} pruned at version {2}", + .1.to_canonical_display(/* with_prefix */ true), + )] + ObjectPruned(TransactionDigest, ObjectID, u64), } #[async_trait::async_trait] @@ -90,75 +89,11 @@ impl TransactionsApiServer for Transactions { digest: TransactionDigest, options: SuiTransactionBlockResponseOptions, ) -> RpcResult { - use Error as E; - let Self(ctx) = self; - let transaction = ctx.loader().load_one(TransactionKey(digest)); - let balance_changes: OptionFuture<_> = options - .show_balance_changes - .then(|| ctx.loader().load_one(TxBalanceChangeKey(digest))) - .into(); - - let (transaction, balance_changes) = join!(transaction, balance_changes); - - let transaction = transaction - .map_err(internal_error)? - .ok_or_else(|| invalid_params(E::NotFound(digest)))?; - - // Balance changes might not be present because of pruning, in which case we return - // nothing, even if the changes were requested. - let balance_changes = match balance_changes.transpose().map_err(internal_error)? { - Some(None) => rpc_bail!(pruned("balance changes for transaction {digest}")), - Some(changes) => changes, - None => None, - }; - - let digest = TransactionDigest::try_from(transaction.tx_digest.clone()) - .map_err(E::Conversion) - .map_err(internal_error)?; - - let mut response = SuiTransactionBlockResponse::new(digest); - - if options.show_input { - response.transaction = Some( - input_response(ctx, &transaction) - .await - .map_err(internal_error)?, - ); - } - - if options.show_raw_input { - response.raw_transaction = transaction.raw_transaction.clone(); - } - - if options.show_effects { - response.effects = Some(effects_response(&transaction).map_err(internal_error)?); - } - - if options.show_raw_effects { - response.raw_effects = transaction.raw_effects.clone(); - } - - if options.show_events { - response.events = Some( - events_response(ctx, digest, &transaction) - .await - .map_err(internal_error)?, - ); - } - - if let Some(balance_changes) = balance_changes { - response.balance_changes = - Some(balance_changes_response(balance_changes).map_err(internal_error)?); - } - - if options.show_object_changes { - response.object_changes = - Some(object_changes_response(ctx, digest, &transaction).await?); - } - - Ok(response) + Ok(transaction_response(ctx, digest, options) + .await + .with_internal_context(|| format!("Failed to get transaction {digest}"))?) } } @@ -181,26 +116,97 @@ impl Default for TransactionsConfig { } } +/// Fetch the necessary data from the stores in `ctx` and transform it to build a response for the +/// transaction identified by `digest`, according to the response `options`. +async fn transaction_response( + ctx: &Context, + digest: TransactionDigest, + options: SuiTransactionBlockResponseOptions, +) -> Result> { + let transaction = ctx.loader().load_one(TransactionKey(digest)); + let balance_changes: OptionFuture<_> = options + .show_balance_changes + .then(|| ctx.loader().load_one(TxBalanceChangeKey(digest))) + .into(); + + let (transaction, balance_changes) = join!(transaction, balance_changes); + + let transaction = transaction + .context("Failed to fetch transaction from store")? + .ok_or_else(|| invalid_params(Error::NotFound(digest)))?; + + // Balance changes might not be present because of pruning, in which case we return + // nothing, even if the changes were requested. + let balance_changes = match balance_changes + .transpose() + .context("Failed to fetch balance changes from store")? + { + Some(None) => return Err(invalid_params(Error::BalanceChangesPruned(digest))), + Some(changes) => changes, + None => None, + }; + + let digest = TransactionDigest::try_from(transaction.tx_digest.clone()) + .context("Failed to deserialize transaction digest")?; + + let mut response = SuiTransactionBlockResponse::new(digest); + + if options.show_input { + response.transaction = Some(input_response(ctx, &transaction).await?); + } + + if options.show_raw_input { + response.raw_transaction = transaction.raw_transaction.clone(); + } + + if options.show_effects { + response.effects = Some(effects_response(&transaction)?); + } + + if options.show_raw_effects { + response.raw_effects = transaction.raw_effects.clone(); + } + + if options.show_events { + response.events = Some(events_response(ctx, digest, &transaction).await?); + } + + if let Some(balance_changes) = balance_changes { + response.balance_changes = Some(balance_changes_response(balance_changes)?); + } + + if options.show_object_changes { + response.object_changes = Some(object_changes_response(ctx, digest, &transaction).await?); + } + + Ok(response) +} + /// Extract a representation of the transaction's input data from the stored form. async fn input_response( ctx: &Context, tx: &StoredTransaction, -) -> Result { - let data: TransactionData = bcs::from_bytes(&tx.raw_transaction)?; - let tx_signatures: Vec = bcs::from_bytes(&tx.user_signatures)?; +) -> Result> { + let data: TransactionData = + bcs::from_bytes(&tx.raw_transaction).context("Failed to deserialize TransactionData")?; + let tx_signatures: Vec = + bcs::from_bytes(&tx.user_signatures).context("Failed to deserialize user signatures")?; Ok(SuiTransactionBlock { data: SuiTransactionBlockData::try_from_with_package_resolver(data, ctx.package_resolver()) .await - .map_err(Error::Resolution)?, + .context("Failed to resolve types in transaction data")?, tx_signatures, }) } /// Extract a representation of the transaction's effects from the stored form. -fn effects_response(tx: &StoredTransaction) -> Result { - let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects)?; - effects.try_into().map_err(Error::Conversion) +fn effects_response(tx: &StoredTransaction) -> Result> { + let effects: TransactionEffects = + bcs::from_bytes(&tx.raw_effects).context("Failed to deserialize TransactionEffects")?; + Ok(effects + .try_into() + .context("Failed to convert Effects into response")?) } /// Extract the transaction's events from its stored form. @@ -208,10 +214,8 @@ async fn events_response( ctx: &Context, digest: TransactionDigest, tx: &StoredTransaction, -) -> Result { - use Error as E; - - let events: Vec = bcs::from_bytes(&tx.events)?; +) -> Result> { + let events: Vec = bcs::from_bytes(&tx.events).context("Failed to deserialize Events")?; let mut sui_events = Vec::with_capacity(events.len()); for (ix, event) in events.into_iter().enumerate() { @@ -219,16 +223,18 @@ async fn events_response( .package_resolver() .type_layout(event.type_.clone().into()) .await - .map_err(|e| E::Resolution(e.into()))? - { + .with_context(|| { + format!( + "Failed to resolve layout for {}", + event.type_.to_canonical_display(/* with_prefix */ true) + ) + })? { MoveTypeLayout::Struct(s) => MoveDatatypeLayout::Struct(s), MoveTypeLayout::Enum(e) => MoveDatatypeLayout::Enum(e), - _ => { - return Err(E::Resolution(anyhow!( - "Event {ix} from {digest} is not a struct or enum: {}", - event.type_.to_canonical_string(/* with_prefix */ true) - ))); - } + _ => rpc_bail!( + "Event {ix} is not a struct or enum: {}", + event.type_.to_canonical_string(/* with_prefix */ true) + ), }; let sui_event = SuiEvent::try_from( @@ -238,7 +244,7 @@ async fn events_response( Some(tx.timestamp_ms as u64), layout, ) - .map_err(E::Conversion)?; + .with_context(|| format!("Failed to convert Event {ix} into response"))?; sui_events.push(sui_event) } @@ -249,8 +255,9 @@ async fn events_response( /// Extract the transaction's balance changes from their stored form. fn balance_changes_response( balance_changes: StoredTxBalanceChange, -) -> Result, Error> { - let balance_changes: Vec = bcs::from_bytes(&balance_changes.balance_changes)?; +) -> Result, RpcError> { + let balance_changes: Vec = bcs::from_bytes(&balance_changes.balance_changes) + .context("Failed to deserialize BalanceChanges")?; let mut response = Vec::with_capacity(balance_changes.len()); for BalanceChange::V1 { @@ -259,7 +266,9 @@ fn balance_changes_response( amount, } in balance_changes { - let coin_type = TypeTag::from_str(&coin_type).map_err(Error::Resolution)?; + let coin_type = TypeTag::from_str(&coin_type) + .with_context(|| format!("Invalid coin type: {coin_type:?}"))?; + response.push(SuiBalanceChange { owner, coin_type, @@ -276,14 +285,11 @@ async fn object_changes_response( ctx: &Context, digest: TransactionDigest, tx: &StoredTransaction, -) -> RpcResult> { - let tx_data: TransactionData = bcs::from_bytes(&tx.raw_transaction) - .map_err(Error::from) - .map_err(internal_error)?; - - let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects) - .map_err(Error::from) - .map_err(internal_error)?; +) -> Result, RpcError> { + let tx_data: TransactionData = + bcs::from_bytes(&tx.raw_transaction).context("Failed to deserialize TransactionData")?; + let effects: TransactionEffects = + bcs::from_bytes(&tx.raw_effects).context("Failed to deserialize TransactionEffects")?; let mut keys = vec![]; let native_changes = effects.object_changes(); @@ -301,7 +307,7 @@ async fn object_changes_response( .loader() .load_many(keys) .await - .map_err(|e| internal_error(e.to_string()))?; + .context("Failed to fetch object contents")?; // Fetch and deserialize the contents of an object, based on its object ref. Assumes that all // object versions that will be fetched in this way have come from a valid transaction, and @@ -310,7 +316,7 @@ async fn object_changes_response( let fetch_object = |id: ObjectID, v: Option, d: Option| - -> RpcResult> { + -> Result, RpcError> { let Some(v) = v else { return Ok(None) }; let Some(d) = d else { return Ok(None) }; @@ -318,16 +324,15 @@ async fn object_changes_response( let stored = objects .get(&ObjectVersionKey(id, v)) - .ok_or_else(|| pruned(format!("Object {id} at version {v}")))?; + .ok_or_else(|| invalid_params(Error::ObjectPruned(digest, id, v)))?; let bytes = stored .serialized_object .as_ref() - .ok_or_else(|| internal_error("No content for object {id} at version {v}"))?; + .with_context(|| format!("No content for object {id} at version {v}"))?; let o = bcs::from_bytes(bytes) - .map_err(Error::from) - .map_err(internal_error)?; + .with_context(|| format!("Failed to deserialize object {id} at version {v}"))?; Ok(Some((o, d))) }; @@ -350,15 +355,15 @@ async fn object_changes_response( use IDOperation as ID; changes.push(match (id_operation, input, output) { - (ID::Created, Some((i, _)), _) => rpc_bail!(internal_error( + (ID::Created, Some((i, _)), _) => rpc_bail!( "Unexpected input version {} for object {object_id} created by transaction {digest}", i.version().value(), - )), + ), - (ID::Deleted, _, Some((o, _))) => rpc_bail!(internal_error( + (ID::Deleted, _, Some((o, _))) => rpc_bail!( "Unexpected output version {} for object {object_id} deleted by transaction {digest}", o.version().value(), - )), + ), // The following cases don't end up in the output: created and wrapped objects, // unwrapped objects (and by extension, unwrapped and deleted objects), system package @@ -387,7 +392,7 @@ async fn object_changes_response( owner: o.owner().clone(), object_type: o .struct_tag() - .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + .ok_or_else(|| internal_error!("No type for object {object_id}"))?, object_id, version: o.version(), digest: d, @@ -399,7 +404,7 @@ async fn object_changes_response( recipient: o.owner().clone(), object_type: o .struct_tag() - .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + .ok_or_else(|| internal_error!("No type for object {object_id}"))?, object_id, version: o.version(), digest: od, @@ -411,7 +416,7 @@ async fn object_changes_response( owner: o.owner().clone(), object_type: o .struct_tag() - .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + .ok_or_else(|| internal_error!("No type for object {object_id}"))?, object_id, version: o.version(), previous_version: i.version(), @@ -422,7 +427,7 @@ async fn object_changes_response( sender: tx_data.sender(), object_type: i .struct_tag() - .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + .ok_or_else(|| internal_error!("No type for object {object_id}"))?, object_id, version: effects.lamport_version(), }, @@ -431,7 +436,7 @@ async fn object_changes_response( sender: tx_data.sender(), object_type: i .struct_tag() - .ok_or_else(|| internal_error(format!("No type for object {object_id}")))?, + .ok_or_else(|| internal_error!("No type for object {object_id}"))?, object_id, version: effects.lamport_version(), }, diff --git a/crates/sui-indexer-alt-jsonrpc/src/error.rs b/crates/sui-indexer-alt-jsonrpc/src/error.rs index d219a9db9e065..97eebf5e851ef 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/error.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/error.rs @@ -2,43 +2,112 @@ // SPDX-License-Identifier: Apache-2.0 // -//! # Error -//! -//! Helper functions for propagating errors from within the service as JSON-RPC errors. Components -//! in the service may return errors in a variety of types. Bodies of JSON-RPC method handlers are -//! responsible for assigning an error code for these errors. - -use std::fmt::Display; +use std::{convert::Infallible, fmt::Display}; use jsonrpsee::types::{ error::{INTERNAL_ERROR_CODE, INVALID_PARAMS_CODE}, ErrorObject, }; -/// Macro wrapping a call to an RPC error constructor (above) which adds support for format -/// strings, and immediate early return: -/// -/// rpc_bail!(internal_error("hello, {}", "world")) -/// -/// Becomes -/// -/// return Err(internal_error(format!("hello, {}", "world"))) +/// Like anyhow's `bail!`, but for returning an internal error. macro_rules! rpc_bail { - ($kind:ident ( $fmt:literal $(,$x:expr)* $(,)? ) ) => { - return Err(crate::error::$kind(format!($fmt, $($x),*))) + ($($arg:tt)*) => { + return Err(crate::error::internal_error!($($arg)*)) }; } +/// Like anyhow's `anyhow!`, but for returning an internal error. +macro_rules! internal_error { + ($($arg:tt)*) => { + crate::error::RpcError::InternalError(anyhow::anyhow!($($arg)*)) + }; +} + +pub(crate) use internal_error; pub(crate) use rpc_bail; -pub(crate) fn internal_error(err: impl ToString) -> ErrorObject<'static> { - ErrorObject::owned(INTERNAL_ERROR_CODE, err.to_string(), None::<()>) +/// Behaves exactly like `anyhow::Context`, but only adds context to `RpcError::InternalError`. +pub(crate) trait InternalContext { + fn internal_context(self, c: C) -> Result> + where + C: Display + Send + Sync + 'static; + + fn with_internal_context(self, f: F) -> Result> + where + C: Display + Send + Sync + 'static, + F: FnOnce() -> C; +} + +/// This type represents two kinds of errors: Invalid Params (the user's fault), and Internal +/// Errors (the service's fault). Each RpcModule is responsible for defining its own structured +/// user errors, while internal errors are represented with anyhow everywhere. +/// +/// The internal error type defaults to `Infallible`, meaning there are no reasons the response +/// might fail because of user input. +/// +/// This representation was chosen to encourage a pattern where errors that are presented to users +/// have a single source of truth for how they should be displayed, while internal errors encourage +/// the addition of context (extra information to build a trace of why something went wrong). +/// +/// User errors must be explicitly wrapped with `invalid_params` while internal errors are +/// implicitly converted using the `?` operator. This asymmetry comes from the fact that we could +/// populate `E` with `anyhow::Error`, which would then cause `From` impls to overlap if we +/// supported conversion from both `E` and `anyhow::Error`. +#[derive(thiserror::Error, Debug)] +pub(crate) enum RpcError { + #[error("Invalid Params: {0}")] + InvalidParams(E), + + #[error("Internal Error: {0:#}")] + InternalError(#[from] anyhow::Error), } -pub(crate) fn invalid_params(err: impl ToString) -> ErrorObject<'static> { - ErrorObject::owned(INVALID_PARAMS_CODE, err.to_string(), None::<()>) +impl InternalContext for Result> { + /// Wrap an internal error with additional context. + fn internal_context(self, c: C) -> Result> + where + C: Display + Send + Sync + 'static, + { + use RpcError as E; + if let Err(E::InternalError(e)) = self { + Err(E::InternalError(e.context(c))) + } else { + self + } + } + + /// Wrap an internal error with additional context that is lazily evaluated only once an + /// internal error has occured. + fn with_internal_context(self, f: F) -> Result> + where + C: Display + Send + Sync + 'static, + F: FnOnce() -> C, + { + use RpcError as E; + if let Err(E::InternalError(e)) = self { + Err(E::InternalError(e.context(f()))) + } else { + self + } + } +} + +impl From> for ErrorObject<'static> { + fn from(err: RpcError) -> Self { + use RpcError as E; + match &err { + E::InvalidParams(_) => { + ErrorObject::owned(INVALID_PARAMS_CODE, err.to_string(), None::<()>) + } + + E::InternalError(_) => { + ErrorObject::owned(INTERNAL_ERROR_CODE, err.to_string(), None::<()>) + } + } + } } -pub(crate) fn pruned(what: impl Display) -> ErrorObject<'static> { - ErrorObject::owned(INVALID_PARAMS_CODE, format!("{what} pruned"), None::<()>) +/// Helper function to convert a user error into the `RpcError` type. +pub(crate) fn invalid_params(err: E) -> RpcError { + RpcError::InvalidParams(err) } From 18e50bf182d76dea8f211b5c4ad9208c211d00ef Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 00:10:05 +0000 Subject: [PATCH 13/30] test-adapter: support generating cursors for run-jsonrpc ## Description Add the ability to generate a Base64-encoded JSON cursor to the `run-jsonrpc` command, just like the feature on `run-graphql`. ## Test plan This feature will be used by a test in a future commit. --- crates/sui-transactional-test-runner/src/args.rs | 2 ++ crates/sui-transactional-test-runner/src/test_adapter.rs | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/sui-transactional-test-runner/src/args.rs b/crates/sui-transactional-test-runner/src/args.rs index c5b0ee74f991e..4cbbbb8fba54e 100644 --- a/crates/sui-transactional-test-runner/src/args.rs +++ b/crates/sui-transactional-test-runner/src/args.rs @@ -192,6 +192,8 @@ pub struct RunGraphqlCommand { pub struct RunJsonRpcCommand { #[clap(long = "show-headers")] pub show_headers: bool, + #[clap(long, num_args(1..))] + pub cursors: Vec, } #[derive(Debug, clap::Parser)] diff --git a/crates/sui-transactional-test-runner/src/test_adapter.rs b/crates/sui-transactional-test-runner/src/test_adapter.rs index 6801ed6d394c5..d32d21f2748c7 100644 --- a/crates/sui-transactional-test-runner/src/test_adapter.rs +++ b/crates/sui-transactional-test-runner/src/test_adapter.rs @@ -653,7 +653,10 @@ impl<'a> MoveTestAdapter<'a> for SuiTestAdapter { Ok(Some(output.join("\n"))) } - SuiSubcommand::RunJsonRpc(RunJsonRpcCommand { show_headers }) => { + SuiSubcommand::RunJsonRpc(RunJsonRpcCommand { + show_headers, + cursors, + }) => { let file = data.ok_or_else(|| anyhow::anyhow!("Missing JSON-RPC query"))?; let contents = std::fs::read_to_string(file.path())?; @@ -667,7 +670,8 @@ impl<'a> MoveTestAdapter<'a> for SuiTestAdapter { .wait_for_checkpoint_catchup(highest_checkpoint, Duration::from_secs(60)) .await; - let interpolated = self.interpolate_query(&contents, &[], highest_checkpoint)?; + let interpolated = + self.interpolate_query(&contents, &cursors, highest_checkpoint)?; #[derive(Deserialize)] struct Query { From a248d114d018c108aa290524733a4e3ca958e1aa Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 00:12:34 +0000 Subject: [PATCH 14/30] rpc-alt: queryTransactionBlocks ## Description Initial support for `queryTransactionBlocks`, but without supporting any of the filters (just paginating transactions). Pagination uses transaction sequence number as the cursor (rather than transaction digest, as it is in the fullnode implementation), to avoid an extra DB roundtrip to translate between digest and sequence number. This change also introduces an abstraction for pages and cursors that can be re-used for other paginated endpoints. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-jsonrpc -- transactions/query ``` --- Cargo.lock | 1 + .../tests/jsonrpc/transactions/query/all.exp | 334 ++++++++++++++++++ .../tests/jsonrpc/transactions/query/all.move | 117 ++++++ .../transactions/query/unsupported.exp | 16 + .../transactions/query/unsupported.move | 18 + crates/sui-indexer-alt-jsonrpc/Cargo.toml | 1 + .../src/api/transactions/error.rs | 25 ++ .../src/api/transactions/filter.rs | 132 +++++++ .../src/api/transactions/mod.rs | 159 +++++++++ .../response.rs} | 129 ++----- crates/sui-indexer-alt-jsonrpc/src/lib.rs | 6 +- .../sui-indexer-alt-jsonrpc/src/paginate.rs | 88 +++++ 12 files changed, 920 insertions(+), 106 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.move create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.move create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/transactions/error.rs create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/transactions/filter.rs create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/transactions/mod.rs rename crates/sui-indexer-alt-jsonrpc/src/api/{transactions.rs => transactions/response.rs} (75%) create mode 100644 crates/sui-indexer-alt-jsonrpc/src/paginate.rs diff --git a/Cargo.lock b/Cargo.lock index 684ef61b28deb..fcb3076cc407f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14143,6 +14143,7 @@ dependencies = [ "clap", "diesel", "diesel-async", + "fastcrypto", "futures", "jsonrpsee", "move-core-types", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.exp new file mode 100644 index 0000000000000..675f48e7ffb21 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.exp @@ -0,0 +1,334 @@ +processed 15 tasks + +init: +A: object(0,0) + +task 1, lines 16-18: +//# programmable --sender A --inputs 12 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(1,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 20-22: +//# programmable --sender A --inputs 34 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 3, lines 24-26: +//# programmable --sender A --inputs 56 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(3,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 4, lines 28-30: +//# programmable --sender A --inputs 78 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(4,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 5, lines 32-34: +//# programmable --sender A --inputs 90 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(5,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 6, line 36: +//# create-checkpoint +Checkpoint created: 1 + +task 7, lines 38-42: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "data": [ + { + "digest": "3FJ4fSrf7toVCANccxAbeJ5A1iSzwKLghCYcaz9atbCD" + }, + { + "digest": "Fx83wfghpUeiBQJ2C1Vt9WwY5rkGUWWgoXSCGfomqqnv" + }, + { + "digest": "4tTfhF9TpbEbJ1efxQbc6A4DWVbBzUYwNhXgt7zsmJsc" + }, + { + "digest": "BULsDepy775taHDviboyivQdnoWkB5QMDYiM1kGcfbQ9" + }, + { + "digest": "3H9FD5LGcHgFSQBfiziYa5f31b86iuQe9Cn5DVMenMAG" + }, + { + "digest": "8p2kdvQUf3TKihDPyC62ggc79intjzNW7TnfaA7an9At" + } + ], + "nextCursor": "NQ==", + "hasNextPage": false + } +} + +task 8, lines 44-52: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "data": [ + { + "digest": "3FJ4fSrf7toVCANccxAbeJ5A1iSzwKLghCYcaz9atbCD" + }, + { + "digest": "Fx83wfghpUeiBQJ2C1Vt9WwY5rkGUWWgoXSCGfomqqnv" + }, + { + "digest": "4tTfhF9TpbEbJ1efxQbc6A4DWVbBzUYwNhXgt7zsmJsc" + } + ], + "nextCursor": "Mg==", + "hasNextPage": true + } +} + +task 9, lines 54-62: +//# run-jsonrpc --cursors 2 +Response: { + "jsonrpc": "2.0", + "id": 2, + "result": { + "data": [ + { + "digest": "BULsDepy775taHDviboyivQdnoWkB5QMDYiM1kGcfbQ9" + }, + { + "digest": "3H9FD5LGcHgFSQBfiziYa5f31b86iuQe9Cn5DVMenMAG" + }, + { + "digest": "8p2kdvQUf3TKihDPyC62ggc79intjzNW7TnfaA7an9At" + } + ], + "nextCursor": "NQ==", + "hasNextPage": false + } +} + +task 10, lines 64-73: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 3, + "result": { + "data": [ + { + "digest": "8p2kdvQUf3TKihDPyC62ggc79intjzNW7TnfaA7an9At" + }, + { + "digest": "3H9FD5LGcHgFSQBfiziYa5f31b86iuQe9Cn5DVMenMAG" + }, + { + "digest": "BULsDepy775taHDviboyivQdnoWkB5QMDYiM1kGcfbQ9" + }, + { + "digest": "4tTfhF9TpbEbJ1efxQbc6A4DWVbBzUYwNhXgt7zsmJsc" + }, + { + "digest": "Fx83wfghpUeiBQJ2C1Vt9WwY5rkGUWWgoXSCGfomqqnv" + }, + { + "digest": "3FJ4fSrf7toVCANccxAbeJ5A1iSzwKLghCYcaz9atbCD" + } + ], + "nextCursor": "MA==", + "hasNextPage": false + } +} + +task 11, lines 75-84: +//# run-jsonrpc --cursors 3 +Response: { + "jsonrpc": "2.0", + "id": 4, + "result": { + "data": [ + { + "digest": "4tTfhF9TpbEbJ1efxQbc6A4DWVbBzUYwNhXgt7zsmJsc" + }, + { + "digest": "Fx83wfghpUeiBQJ2C1Vt9WwY5rkGUWWgoXSCGfomqqnv" + } + ], + "nextCursor": "MQ==", + "hasNextPage": true + } +} + +task 12, lines 86-93: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 5, + "error": { + "code": -32602, + "message": "Invalid Params: Pagination issue: Failed to decode Base64: Invalid value was given to the function" + } +} + +task 13, lines 95-103: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 6, + "error": { + "code": -32602, + "message": "Invalid Params: Pagination issue: Requested page size 10000 exceeds maximum 100" + } +} + +task 14, lines 105-117: +//# run-jsonrpc --cursors 1 +Response: { + "jsonrpc": "2.0", + "id": 7, + "result": { + "data": [ + { + "digest": "4tTfhF9TpbEbJ1efxQbc6A4DWVbBzUYwNhXgt7zsmJsc", + "transaction": { + "data": { + "messageVersion": "v1", + "transaction": { + "kind": "ProgrammableTransaction", + "inputs": [ + { + "type": "pure", + "valueType": "u64", + "value": "34" + }, + { + "type": "pure", + "valueType": "address", + "value": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + } + ], + "transactions": [ + { + "SplitCoins": [ + "GasCoin", + [ + { + "Input": 0 + } + ] + ] + }, + { + "TransferObjects": [ + [ + { + "Result": 0 + } + ], + { + "Input": 1 + } + ] + } + ] + }, + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "gasData": { + "payment": [ + { + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": 2, + "digest": "HPfMHJxiZ7ozY7jppcCGe2MwZ2TCFCCcmugtNvUgRuts" + } + ], + "owner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "price": "1000", + "budget": "5000000000" + } + }, + "txSignatures": [ + "APiQZonSLcnMfxma2YMk7AvdgkWSew2B3HWH98dbyf5Z67cR5ojet/acq516WO9rbD/hEJJMd51bgmjm3MOyTgN/UUY663bYjcm3XmNyULIgxJz1t5Z9vxfB+fp8WUoJKA==" + ] + } + }, + { + "digest": "BULsDepy775taHDviboyivQdnoWkB5QMDYiM1kGcfbQ9", + "transaction": { + "data": { + "messageVersion": "v1", + "transaction": { + "kind": "ProgrammableTransaction", + "inputs": [ + { + "type": "pure", + "valueType": "u64", + "value": "56" + }, + { + "type": "pure", + "valueType": "address", + "value": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + } + ], + "transactions": [ + { + "SplitCoins": [ + "GasCoin", + [ + { + "Input": 0 + } + ] + ] + }, + { + "TransferObjects": [ + [ + { + "Result": 0 + } + ], + { + "Input": 1 + } + ] + } + ] + }, + "sender": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "gasData": { + "payment": [ + { + "objectId": "0xbff5e96a4a5c0f7348259c7abfdfd999405c02b9e9c0d0d59ec669a53b104c7a", + "version": 3, + "digest": "CnQFnb8LS9rdRHjkWFo5GKnP5QBNkRAe6eoamKnxFWEQ" + } + ], + "owner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e", + "price": "1000", + "budget": "5000000000" + } + }, + "txSignatures": [ + "AEgcwTWMXQYfQwFRXqb3R26Z7LMv7ODaZHJ8zJd2LT88eg+2Y98zJwUUDxxmMj3tAadcwBgLTdcRFe6lzcO2jgl/UUY663bYjcm3XmNyULIgxJz1t5Z9vxfB+fp8WUoJKA==" + ] + } + } + ], + "nextCursor": "Mw==", + "hasNextPage": true + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.move new file mode 100644 index 0000000000000..44d561dd86aad --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/all.move @@ -0,0 +1,117 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses test=0x0 --simulator + +// 1. Default behavior of getTransactionBlock (no options) +// 2. Setting a limit +// 3. Setting a limit and cursor +// 4. Changing the order +// 5. Setting the order, cursor and limit +// 6. Providing a bad cursor +// 7. Page size too large +// 8. Unsupported filter +// 9. Supplying response options + +//# programmable --sender A --inputs 12 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 34 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 56 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 78 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 90 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [{}] +} + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + null, + 3 + ] +} + +//# run-jsonrpc --cursors 2 +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + "@{cursor_0}", + 3 + ] +} + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + null, + null, + true + ] +} + +//# run-jsonrpc --cursors 3 +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + "@{cursor_0}", + 2, + true + ] +} + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + "i_am_not_a_cursor" + ] +} + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [ + {}, + null, + 10000 + ] +} + +//# run-jsonrpc --cursors 1 +{ + "method": "suix_queryTransactionBlocks", + "params": [ + { + "options": { + "showInput": true + } + }, + "@{cursor_0}", + 2 + ] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.exp new file mode 100644 index 0000000000000..951a053a35aae --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.exp @@ -0,0 +1,16 @@ +processed 3 tasks + +task 1, line 6: +//# create-checkpoint +Checkpoint created: 1 + +task 2, lines 8-18: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "error": { + "code": -32602, + "message": "Invalid Params: TransactionKind filter is not supported" + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.move new file mode 100644 index 0000000000000..a785b205e81cb --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/transactions/query/unsupported.move @@ -0,0 +1,18 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --addresses test=0x0 --simulator + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "suix_queryTransactionBlocks", + "params": [ + { + "filter": { + "TransactionKind": "NotSupported" + } + } + ] +} diff --git a/crates/sui-indexer-alt-jsonrpc/Cargo.toml b/crates/sui-indexer-alt-jsonrpc/Cargo.toml index 735874400d606..b0a29a2c57ea3 100644 --- a/crates/sui-indexer-alt-jsonrpc/Cargo.toml +++ b/crates/sui-indexer-alt-jsonrpc/Cargo.toml @@ -19,6 +19,7 @@ bcs.workspace = true clap.workspace = true diesel = { workspace = true, features = ["chrono"] } diesel-async = { workspace = true, features = ["bb8", "postgres", "async-connection-wrapper"] } +fastcrypto.workspace = true futures.workspace = true jsonrpsee = { workspace = true, features = ["macros", "server"] } pin-project-lite.workspace = true diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions/error.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/error.rs new file mode 100644 index 0000000000000..82c5496880c7d --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/error.rs @@ -0,0 +1,25 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use sui_types::{base_types::ObjectID, digests::TransactionDigest}; + +#[derive(thiserror::Error, Debug)] +pub(super) enum Error { + #[error("Pagination issue: {0}")] + Pagination(#[from] crate::paginate::Error), + + #[error("Balance changes for transaction {0} have been pruned")] + PrunedBalanceChanges(TransactionDigest), + + #[error( + "Transaction {0} affected object {} pruned at version {2}", + .1.to_canonical_display(/* with_prefix */ true), + )] + PrunedObject(TransactionDigest, ObjectID, u64), + + #[error("Transaction {0} not found")] + NotFound(TransactionDigest), + + #[error("{0}")] + Unsupported(&'static str), +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions/filter.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/filter.rs new file mode 100644 index 0000000000000..e9b6b3d7fa723 --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/filter.rs @@ -0,0 +1,132 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Context as _; +use diesel::{ExpressionMethods, QueryDsl}; +use sui_indexer_alt_schema::schema::tx_digests; +use sui_json_rpc_types::{Page as PageResponse, TransactionFilter}; +use sui_types::digests::TransactionDigest; + +use crate::{ + error::{invalid_params, rpc_bail, RpcError}, + paginate::{Cursor, Page}, +}; + +use super::{error::Error, Context, TransactionsConfig}; + +/// Fetch the digests for a page of transactions that satisfy the given `filter` and pagination +/// parameters. Returns the digests and a cursor pointing to the last result (if there are any +/// results). +pub(super) async fn transactions( + ctx: &Context, + config: &TransactionsConfig, + filter: &Option, + cursor: Option, + limit: Option, + descending_order: Option, +) -> Result, RpcError> { + let page: Page = Page::from_params( + config.default_page_size, + config.max_page_size, + cursor, + limit, + descending_order, + )?; + + use TransactionFilter as F; + let mut refs = match filter { + None => all_transactions(ctx, &page).await?, + + Some(F::TransactionKind(_) | F::TransactionKindIn(_)) => { + return unsupported("TransactionKind filter is not supported") + } + + Some(F::InputObject(_)) => { + return unsupported( + "InputObject filter is not supported, please use AffectedObject instead.", + ) + } + + Some(F::ChangedObject(_)) => { + return unsupported( + "ChangedObject filter is not supported, please use AffectedObject instead.", + ) + } + + Some(F::ToAddress(_)) => { + return unsupported( + "ToAddress filter is not supported, please use FromOrToAddress instead.", + ) + } + + _ => rpc_bail!("Not implemented yet"), + }; + + let has_next_page = refs.len() > page.limit as usize; + if has_next_page { + refs.truncate(page.limit as usize); + } + + let digests = refs + .iter() + .map(|(_, digest)| TransactionDigest::try_from(digest.as_slice())) + .collect::, _>>() + .context("Failed to deserialize transaction digests")?; + + let cursor = refs + .last() + .map(|(last, _)| Cursor(*last).encode()) + .transpose() + .context("Failed to encode next cursor")?; + + Ok(PageResponse { + data: digests, + next_cursor: cursor, + has_next_page, + }) +} + +/// Fetch a page of transaction digests without filtering them. Fetches one more result than was +/// requested to detect a next page. +async fn all_transactions( + ctx: &Context, + page: &Page, +) -> Result)>, RpcError> { + use tx_digests::dsl as d; + + let mut query = d::tx_digests + .select((d::tx_sequence_number, d::tx_digest)) + .limit(page.limit + 1) + .into_boxed(); + + if let Some(Cursor(tx)) = page.cursor { + if page.descending { + query = query.filter(d::tx_sequence_number.lt(tx as i64)); + } else { + query = query.filter(d::tx_sequence_number.gt(tx as i64)); + } + } + + if page.descending { + query = query.order(d::tx_sequence_number.desc()); + } else { + query = query.order(d::tx_sequence_number.asc()); + } + + let mut conn = ctx + .reader() + .connect() + .await + .context("Failed to connect to database")?; + + let refs: Vec<(i64, Vec)> = conn + .results(query) + .await + .context("Failed to fetch matching transaction digests")?; + + Ok(refs) +} + +fn unsupported(msg: &'static str) -> Result> { + Err(invalid_params(Error::Unsupported(msg))) +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/mod.rs new file mode 100644 index 0000000000000..eb4d204be46cf --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/mod.rs @@ -0,0 +1,159 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use futures::future; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use serde::{Deserialize, Serialize}; +use sui_json_rpc_types::{ + Page, SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, + SuiTransactionBlockResponseQuery, +}; +use sui_open_rpc::Module; +use sui_open_rpc_macros::open_rpc; +use sui_types::digests::TransactionDigest; + +use crate::{context::Context, error::InternalContext}; + +use super::rpc_module::RpcModule; + +mod error; +mod filter; +mod response; + +#[open_rpc(namespace = "sui", tag = "Transactions API")] +#[rpc(server, namespace = "sui")] +trait TransactionsApi { + /// Fetch a transaction by its transaction digest. + #[method(name = "getTransactionBlock")] + async fn get_transaction_block( + &self, + /// The digest of the queried transaction. + digest: TransactionDigest, + /// Options controlling the output format. + options: SuiTransactionBlockResponseOptions, + ) -> RpcResult; +} + +#[open_rpc(namespace = "suix", tag = "Query Transactions API")] +#[rpc(server, namespace = "suix")] +trait QueryTransactionsApi { + /// Query transactions based on their properties (sender, affected addresses, function calls, + /// etc). Returns a paginated list of transactions. + /// + /// If a cursor is provided, the query will start from the transaction after the one pointed to + /// by this cursor, otherwise pagination starts from the first transaction that meets the query + /// criteria. + /// + /// The definition of "first" transaction is changed by the `descending_order` parameter, which + /// is optional, and defaults to false, meaning that the oldest transaction is shown first. + /// + /// The size of each page is controlled by the `limit` parameter. + #[method(name = "queryTransactionBlocks")] + async fn query_transaction_blocks( + &self, + /// The query criteria, and the output options. + query: SuiTransactionBlockResponseQuery, + /// Cursor to start paginating from. + cursor: Option, + /// Maximum number of transactions to return per page. + limit: Option, + /// Order of results, defaulting to ascending order (false), by sequence on-chain. + descending_order: Option, + ) -> RpcResult>; +} + +pub(crate) struct Transactions(pub Context); + +pub(crate) struct QueryTransactions(pub Context, pub TransactionsConfig); + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TransactionsConfig { + /// The default page size limit when querying transactions, if none is provided. + pub default_page_size: usize, + + /// The largest acceptable page size when querying transactions. Requesting a page larger than + /// this is a user error. + pub max_page_size: usize, +} + +#[async_trait::async_trait] +impl TransactionsApiServer for Transactions { + async fn get_transaction_block( + &self, + digest: TransactionDigest, + options: SuiTransactionBlockResponseOptions, + ) -> RpcResult { + let Self(ctx) = self; + Ok(response::transaction(ctx, digest, &options) + .await + .with_internal_context(|| format!("Failed to get transaction {digest}"))?) + } +} + +#[async_trait::async_trait] +impl QueryTransactionsApiServer for QueryTransactions { + async fn query_transaction_blocks( + &self, + query: SuiTransactionBlockResponseQuery, + cursor: Option, + limit: Option, + descending_order: Option, + ) -> RpcResult> { + let Self(ctx, config) = self; + + let Page { + data: digests, + next_cursor, + has_next_page, + } = filter::transactions(ctx, config, &query.filter, cursor, limit, descending_order) + .await?; + + let options = query.options.unwrap_or_default(); + + let tx_futures = digests + .iter() + .map(|d| response::transaction(ctx, *d, &options)); + + let data = future::join_all(tx_futures) + .await + .into_iter() + .zip(digests) + .map(|(r, d)| r.with_internal_context(|| format!("Failed to get transaction {d}"))) + .collect::, _>>()?; + + Ok(Page { + data, + next_cursor, + has_next_page, + }) + } +} + +impl RpcModule for Transactions { + fn schema(&self) -> Module { + TransactionsApiOpenRpc::module_doc() + } + + fn into_impl(self) -> jsonrpsee::RpcModule { + self.into_rpc() + } +} + +impl RpcModule for QueryTransactions { + fn schema(&self) -> Module { + QueryTransactionsApiOpenRpc::module_doc() + } + + fn into_impl(self) -> jsonrpsee::RpcModule { + self.into_rpc() + } +} + +impl Default for TransactionsConfig { + fn default() -> Self { + Self { + default_page_size: 50, + max_page_size: 100, + } + } +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs similarity index 75% rename from crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs rename to crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs index 51448cdc2486b..8273d4eb10921 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs @@ -5,9 +5,7 @@ use std::str::FromStr; use anyhow::Context as _; use futures::future::OptionFuture; -use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use move_core_types::annotated_value::{MoveDatatypeLayout, MoveTypeLayout}; -use serde::{Deserialize, Serialize}; use sui_indexer_alt_schema::transactions::{ BalanceChange, StoredTransaction, StoredTxBalanceChange, }; @@ -16,8 +14,6 @@ use sui_json_rpc_types::{ SuiTransactionBlock, SuiTransactionBlockData, SuiTransactionBlockEffects, SuiTransactionBlockEvents, SuiTransactionBlockResponse, SuiTransactionBlockResponseOptions, }; -use sui_open_rpc::Module; -use sui_open_rpc_macros::open_rpc; use sui_types::{ base_types::{ObjectID, SequenceNumber}, digests::{ObjectDigest, TransactionDigest}, @@ -36,154 +32,79 @@ use crate::{ objects::ObjectVersionKey, transactions::TransactionKey, tx_balance_changes::TxBalanceChangeKey, }, - error::{internal_error, invalid_params, rpc_bail, InternalContext, RpcError}, + error::{internal_error, invalid_params, rpc_bail, RpcError}, }; -use super::rpc_module::RpcModule; - -#[open_rpc(namespace = "sui", tag = "Transactions API")] -#[rpc(server, namespace = "sui")] -trait TransactionsApi { - /// Fetch a transaction by its transaction digest. - #[method(name = "getTransactionBlock")] - async fn get_transaction_block( - &self, - /// The digest of the queried transaction. - digest: TransactionDigest, - /// Options controlling the output format. - options: SuiTransactionBlockResponseOptions, - ) -> RpcResult; -} - -pub(crate) struct Transactions(pub Context); - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct TransactionsConfig { - /// The default page size limit when querying transactions, if none is provided. - pub default_page_size: usize, - - /// The largest acceptable page size when querying transactions. Requesting a page larger than - /// this is a user error. - pub max_page_size: usize, -} - -#[derive(thiserror::Error, Debug)] -pub(crate) enum Error { - #[error("Transaction {0} not found")] - NotFound(TransactionDigest), - - #[error("Balance changes for transaction {0} have been pruned")] - BalanceChangesPruned(TransactionDigest), - - #[error( - "Transaction {0} affected object {} pruned at version {2}", - .1.to_canonical_display(/* with_prefix */ true), - )] - ObjectPruned(TransactionDigest, ObjectID, u64), -} - -#[async_trait::async_trait] -impl TransactionsApiServer for Transactions { - async fn get_transaction_block( - &self, - digest: TransactionDigest, - options: SuiTransactionBlockResponseOptions, - ) -> RpcResult { - let Self(ctx) = self; - - Ok(transaction_response(ctx, digest, options) - .await - .with_internal_context(|| format!("Failed to get transaction {digest}"))?) - } -} - -impl RpcModule for Transactions { - fn schema(&self) -> Module { - TransactionsApiOpenRpc::module_doc() - } - - fn into_impl(self) -> jsonrpsee::RpcModule { - self.into_rpc() - } -} - -impl Default for TransactionsConfig { - fn default() -> Self { - Self { - default_page_size: 50, - max_page_size: 100, - } - } -} +use super::error::Error; /// Fetch the necessary data from the stores in `ctx` and transform it to build a response for the /// transaction identified by `digest`, according to the response `options`. -async fn transaction_response( +pub(super) async fn transaction( ctx: &Context, digest: TransactionDigest, - options: SuiTransactionBlockResponseOptions, + options: &SuiTransactionBlockResponseOptions, ) -> Result> { - let transaction = ctx.loader().load_one(TransactionKey(digest)); - let balance_changes: OptionFuture<_> = options + let stored_tx = ctx.loader().load_one(TransactionKey(digest)); + let stored_bc: OptionFuture<_> = options .show_balance_changes .then(|| ctx.loader().load_one(TxBalanceChangeKey(digest))) .into(); - let (transaction, balance_changes) = join!(transaction, balance_changes); + let (stored_tx, stored_bc) = join!(stored_tx, stored_bc); - let transaction = transaction + let stored_tx = stored_tx .context("Failed to fetch transaction from store")? .ok_or_else(|| invalid_params(Error::NotFound(digest)))?; // Balance changes might not be present because of pruning, in which case we return // nothing, even if the changes were requested. - let balance_changes = match balance_changes + let stored_bc = match stored_bc .transpose() .context("Failed to fetch balance changes from store")? { - Some(None) => return Err(invalid_params(Error::BalanceChangesPruned(digest))), + Some(None) => return Err(invalid_params(Error::PrunedBalanceChanges(digest))), Some(changes) => changes, None => None, }; - let digest = TransactionDigest::try_from(transaction.tx_digest.clone()) + let digest = TransactionDigest::try_from(stored_tx.tx_digest.clone()) .context("Failed to deserialize transaction digest")?; let mut response = SuiTransactionBlockResponse::new(digest); if options.show_input { - response.transaction = Some(input_response(ctx, &transaction).await?); + response.transaction = Some(input(ctx, &stored_tx).await?); } if options.show_raw_input { - response.raw_transaction = transaction.raw_transaction.clone(); + response.raw_transaction = stored_tx.raw_transaction.clone(); } if options.show_effects { - response.effects = Some(effects_response(&transaction)?); + response.effects = Some(effects(&stored_tx)?); } if options.show_raw_effects { - response.raw_effects = transaction.raw_effects.clone(); + response.raw_effects = stored_tx.raw_effects.clone(); } if options.show_events { - response.events = Some(events_response(ctx, digest, &transaction).await?); + response.events = Some(events(ctx, digest, &stored_tx).await?); } - if let Some(balance_changes) = balance_changes { - response.balance_changes = Some(balance_changes_response(balance_changes)?); + if let Some(changes) = stored_bc { + response.balance_changes = Some(balance_changes(changes)?); } if options.show_object_changes { - response.object_changes = Some(object_changes_response(ctx, digest, &transaction).await?); + response.object_changes = Some(object_changes(ctx, digest, &stored_tx).await?); } Ok(response) } /// Extract a representation of the transaction's input data from the stored form. -async fn input_response( +async fn input( ctx: &Context, tx: &StoredTransaction, ) -> Result> { @@ -201,7 +122,7 @@ async fn input_response( } /// Extract a representation of the transaction's effects from the stored form. -fn effects_response(tx: &StoredTransaction) -> Result> { +fn effects(tx: &StoredTransaction) -> Result> { let effects: TransactionEffects = bcs::from_bytes(&tx.raw_effects).context("Failed to deserialize TransactionEffects")?; Ok(effects @@ -210,7 +131,7 @@ fn effects_response(tx: &StoredTransaction) -> Result Result, RpcError> { let balance_changes: Vec = bcs::from_bytes(&balance_changes.balance_changes) @@ -281,7 +202,7 @@ fn balance_changes_response( /// Extract the transaction's object changes. Object IDs and versions are fetched from the stored /// transaction, and the object contents are fetched separately by a data loader. -async fn object_changes_response( +async fn object_changes( ctx: &Context, digest: TransactionDigest, tx: &StoredTransaction, @@ -324,7 +245,7 @@ async fn object_changes_response( let stored = objects .get(&ObjectVersionKey(id, v)) - .ok_or_else(|| invalid_params(Error::ObjectPruned(digest, id, v)))?; + .ok_or_else(|| invalid_params(Error::PrunedObject(digest, id, v)))?; let bytes = stored .serialized_object diff --git a/crates/sui-indexer-alt-jsonrpc/src/lib.rs b/crates/sui-indexer-alt-jsonrpc/src/lib.rs index 16ee86298163a..93b2fc91db795 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/lib.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/lib.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use anyhow::Context as _; use api::rpc_module::RpcModule; -use api::transactions::{Transactions, TransactionsConfig}; +use api::transactions::{QueryTransactions, Transactions, TransactionsConfig}; use config::RpcConfig; use data::system_package_task::{SystemPackageTask, SystemPackageTaskArgs}; use jsonrpsee::server::{RpcServiceBuilder, ServerBuilder}; @@ -31,6 +31,7 @@ mod context; pub mod data; mod error; mod metrics; +mod paginate; #[derive(clap::Args, Debug, Clone)] pub struct RpcArgs { @@ -206,7 +207,7 @@ pub async fn start_rpc( extra: _, } = rpc_config.finish(); - let _transactions_config = transactions.finish(TransactionsConfig::default()); + let transactions_config = transactions.finish(TransactionsConfig::default()); let mut rpc = RpcService::new(rpc_args, registry, cancel.child_token()) .context("Failed to create RPC service")?; @@ -220,6 +221,7 @@ pub async fn start_rpc( ); rpc.add_module(Governance(context.clone()))?; + rpc.add_module(QueryTransactions(context.clone(), transactions_config))?; rpc.add_module(Transactions(context.clone()))?; let h_rpc = rpc.run().await.context("Failed to start RPC service")?; diff --git a/crates/sui-indexer-alt-jsonrpc/src/paginate.rs b/crates/sui-indexer-alt-jsonrpc/src/paginate.rs new file mode 100644 index 0000000000000..ae88ae2ae93ba --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/paginate.rs @@ -0,0 +1,88 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use fastcrypto::{ + encoding::{Base64, Encoding}, + error::FastCryptoError, +}; +use serde::{de::DeserializeOwned, Serialize}; + +use crate::error::{invalid_params, RpcError}; + +/// This type wraps a value used as a cursor in a paginated request or response. Cursors are +/// serialized to JSON and then encoded as Base64. +pub(crate) struct Cursor(pub T); + +/// Description of a page to be fetched. +pub(crate) struct Page { + pub cursor: Option>, + pub limit: i64, + pub descending: bool, +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum Error { + #[error("Failed to decode Base64: {0}")] + DecodingBase64(FastCryptoError), + + #[error("Failed to decode JSON: {0}")] + DecodingJson(serde_json::error::Error), + + #[error("Failed to encode JSON: {0}")] + EncodingJson(serde_json::error::Error), + + #[error("Requested page size {requested} exceeds maximum {max}")] + ExceededMaxPageSize { requested: usize, max: usize }, +} + +impl Cursor { + /// Interpret the string as a cursor, Base64-decode it, and then deserialize it from JSON. A + /// failure to do so implies the cursor is invalid, which is treated as a user error. + pub(crate) fn decode(s: &str) -> Result { + let bytes = Base64::decode(s).map_err(Error::DecodingBase64)?; + let value: T = serde_json::from_slice(&bytes).map_err(Error::DecodingJson)?; + Ok(Cursor(value)) + } +} + +impl Cursor { + /// Represent the cursor in JSON, Base64-encoded. A failure implies the cursor is not properly + /// set-up, which is treated as an internal error. + pub(crate) fn encode(&self) -> Result { + let bytes = serde_json::to_vec(&self.0).map_err(Error::EncodingJson)?; + Ok(Base64::encode(&bytes)) + } +} + +impl Page { + /// Interpret RPC method parameters as a description of a page to fetch. + /// + /// This operation can fail if the Cursor cannot be decoded, or the requested page is too + /// large. These are all consider user errors. + pub(crate) fn from_params + std::error::Error>( + default_page_size: usize, + max_page_size: usize, + cursor: Option, + limit: Option, + descending: Option, + ) -> Result> { + let cursor = cursor + .map(|c| Cursor::decode(&c)) + .transpose() + .map_err(|e| invalid_params(E::from(e)))?; + + let limit = limit.unwrap_or(default_page_size); + if limit > max_page_size { + return Err(invalid_params(E::from(Error::ExceededMaxPageSize { + requested: limit, + max: max_page_size, + }))); + } + + Ok(Page { + cursor, + limit: limit as i64, + descending: descending.unwrap_or(false), + }) + } +} From 0663bebc9fc855bb5e985c87f642b0c47706f8e1 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 13:46:03 +0000 Subject: [PATCH 15/30] chore(rpc-alt): assorted renames ## Description Applying a number of suggested renames as a follow-up from code review. ## Test plan CI --- .../src/api/governance.rs | 19 ++++++++++--------- .../src/api/transactions/response.rs | 8 ++++---- .../src/data/object_versions.rs | 8 ++++---- .../src/data/objects.rs | 12 ++++++------ 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs index 8f182740bda6a..28659bfb393e5 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/governance.rs @@ -27,7 +27,7 @@ use sui_types::{ use crate::{ context::Context, - data::{object_versions::LatestObjectKey, objects::ObjectVersionKey}, + data::{object_versions::LatestObjectVersionKey, objects::VersionedObjectKey}, error::{internal_error, rpc_bail, InternalContext, RpcError}, }; @@ -94,9 +94,10 @@ async fn rgp_response(ctx: &Context) -> Result, RpcError> { async fn latest_sui_system_state_response( ctx: &Context, ) -> Result { - let wrapper: SuiSystemStateWrapper = fetch_latest(ctx, SUI_SYSTEM_STATE_OBJECT_ID) - .await - .internal_context("Failed to fetch system state wrapper object")?; + let wrapper: SuiSystemStateWrapper = + fetch_latest_for_system_state(ctx, SUI_SYSTEM_STATE_OBJECT_ID) + .await + .internal_context("Failed to fetch system state wrapper object")?; let inner_id = derive_dynamic_field_id( SUI_SYSTEM_STATE_OBJECT_ID, @@ -106,12 +107,12 @@ async fn latest_sui_system_state_response( .context("Failed to derive inner system state field ID")?; Ok(match wrapper.version { - 1 => fetch_latest::>(ctx, inner_id) + 1 => fetch_latest_for_system_state::>(ctx, inner_id) .await .internal_context("Failed to fetch inner system state object")? .value .into_sui_system_state_summary(), - 2 => fetch_latest::>(ctx, inner_id) + 2 => fetch_latest_for_system_state::>(ctx, inner_id) .await .internal_context("Failed to fetch inner system state object")? .value @@ -127,20 +128,20 @@ async fn latest_sui_system_state_response( /// API, but it does not generalize beyond that, because it assumes that the objects being loaded /// are never deleted or wrapped and have always existed (because it loads using `LatestObjectKey` /// directly without checking the live object set). -async fn fetch_latest( +async fn fetch_latest_for_system_state( ctx: &Context, object_id: ObjectID, ) -> Result { let loader = ctx.loader(); let latest_version = loader - .load_one(LatestObjectKey(object_id)) + .load_one(LatestObjectVersionKey(object_id)) .await .context("Failed to load latest version")? .ok_or_else(|| internal_error!("No latest version found"))?; let stored = loader - .load_one(ObjectVersionKey( + .load_one(VersionedObjectKey( object_id, latest_version.object_version as u64, )) diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs index 8273d4eb10921..bf4d541e1c4bb 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/transactions/response.rs @@ -29,7 +29,7 @@ use tokio::join; use crate::{ context::Context, data::{ - objects::ObjectVersionKey, transactions::TransactionKey, + objects::VersionedObjectKey, transactions::TransactionKey, tx_balance_changes::TxBalanceChangeKey, }, error::{internal_error, invalid_params, rpc_bail, RpcError}, @@ -217,10 +217,10 @@ async fn object_changes( for change in &native_changes { let id = change.id; if let Some(version) = change.input_version { - keys.push(ObjectVersionKey(id, version.value())); + keys.push(VersionedObjectKey(id, version.value())); } if let Some(version) = change.output_version { - keys.push(ObjectVersionKey(id, version.value())); + keys.push(VersionedObjectKey(id, version.value())); } } @@ -244,7 +244,7 @@ async fn object_changes( let v = v.value(); let stored = objects - .get(&ObjectVersionKey(id, v)) + .get(&VersionedObjectKey(id, v)) .ok_or_else(|| invalid_params(Error::PrunedObject(digest, id, v)))?; let bytes = stored diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs b/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs index bb66c5331f8e9..557ebc8450505 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/object_versions.rs @@ -16,17 +16,17 @@ use super::reader::{ReadError, Reader}; /// Key for fetching the latest version of an object, not accounting for deletions or wraps. If the /// object has been deleted or wrapped, the version before the delete/wrap is returned. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) struct LatestObjectKey(pub ObjectID); +pub(crate) struct LatestObjectVersionKey(pub ObjectID); #[async_trait::async_trait] -impl Loader for Reader { +impl Loader for Reader { type Value = StoredObjVersion; type Error = Arc; async fn load( &self, - keys: &[LatestObjectKey], - ) -> Result, Self::Error> { + keys: &[LatestObjectVersionKey], + ) -> Result, Self::Error> { use obj_versions::dsl as v; if keys.is_empty() { diff --git a/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs b/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs index 47021ca8d826d..252f244de8b9e 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/data/objects.rs @@ -10,19 +10,19 @@ use sui_types::base_types::ObjectID; use super::reader::{ReadError, Reader}; -/// Key for fetching a particular version of an object. +/// Key for fetching the contents a particular version of an object. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub(crate) struct ObjectVersionKey(pub ObjectID, pub u64); +pub(crate) struct VersionedObjectKey(pub ObjectID, pub u64); #[async_trait::async_trait] -impl Loader for Reader { +impl Loader for Reader { type Value = StoredObject; type Error = Arc; async fn load( &self, - keys: &[ObjectVersionKey], - ) -> Result, Self::Error> { + keys: &[VersionedObjectKey], + ) -> Result, Self::Error> { use kv_objects::dsl as o; if keys.is_empty() { @@ -33,7 +33,7 @@ impl Loader for Reader { let mut query = o::kv_objects.into_boxed(); - for ObjectVersionKey(id, version) in keys { + for VersionedObjectKey(id, version) in keys { query = query.or_filter( o::object_id .eq(id.into_bytes()) From cc1bf951b2113ae8365f3f43958429a6e6ab5c6d Mon Sep 17 00:00:00 2001 From: Tony Lee Date: Wed, 29 Jan 2025 10:00:13 -0500 Subject: [PATCH 16/30] Deepbook Indexer Trades Endpoint Update (#20998) ## Description 1. taker_balance_manager_id and maker_balance_manager_id included with each trade 2. taker_order_id and maker_order_id included with each trade Optional Filters: 1. limit: defaults to 1 (number of trades to be included in result). Most recent trades are always included first. 2. taker_balance_manager_id: defaults to no filter 3. maker_balance_manager_id: defaults to no filter 4. start_time: unix timestamp in seconds, defaults to 24hrs before end_time 5. end_time: unix timestamp in seconds, defaults to current time Sample query: `trades/SUI_USDC?start_time=1738093405&end_time=1738096485&limit=2&maker_balance_manager_id=0x344c2734b1d211bd15212bfb7847c66a3b18803f3f5ab00f5ff6f87b6fe6d27d&taker_balance_manager_id=0x47dcbbc8561fe3d52198336855f0983878152a12524749e054357ac2e3573d58` Sample Response: `[{"base_volume":405.9,"type":"sell","maker_order_id":"68160737799100866923792791","taker_order_id":"170141183460537392451039660509112362617","quote_volume":1499.8005,"maker_balance_manager_id":"0x344c2734b1d211bd15212bfb7847c66a3b18803f3f5ab00f5ff6f87b6fe6d27d","trade_id":"136321457151457660152049680","price":3.695,"timestamp":1738096392913,"taker_balance_manager_id":"0x47dcbbc8561fe3d52198336855f0983878152a12524749e054357ac2e3573d58"},{"trade_id":"137612729236617328765169261","taker_balance_manager_id":"0x47dcbbc8561fe3d52198336855f0983878152a12524749e054357ac2e3573d58","maker_order_id":"68806373841680701230353053","type":"sell","maker_balance_manager_id":"0x344c2734b1d211bd15212bfb7847c66a3b18803f3f5ab00f5ff6f87b6fe6d27d","quote_volume":1499.833,"timestamp":1738095493338,"taker_order_id":"170141183460538038087082240343418921936","price":3.73,"base_volume":402.1}]` ## Test plan How did you test the new or updated feature? Tested endpoint locally ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- crates/sui-deepbook-indexer/src/server.rs | 142 ++++++++++++++++------ 1 file changed, 103 insertions(+), 39 deletions(-) diff --git a/crates/sui-deepbook-indexer/src/server.rs b/crates/sui-deepbook-indexer/src/server.rs index ba9c9b5e35e2f..39ae32966c0a6 100644 --- a/crates/sui-deepbook-indexer/src/server.rs +++ b/crates/sui-deepbook-indexer/src/server.rs @@ -753,6 +753,7 @@ async fn price_change_24h( async fn trades( Path(pool_name): Path, + Query(params): Query>, State(state): State, ) -> Result>>, DeepBookError> { // Fetch all pools to map names to IDs and decimals @@ -768,14 +769,56 @@ async fn trades( .await .map_err(|_| DeepBookError::InternalError(format!("Pool '{}' not found", pool_name)))?; + // Parse start_time and end_time + let end_time = params + .get("end_time") + .and_then(|v| v.parse::().ok()) + .map(|t| t * 1000) // Convert to milliseconds + .unwrap_or_else(|| { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64 + }); + + let start_time = params + .get("start_time") + .and_then(|v| v.parse::().ok()) + .map(|t| t * 1000) // Convert to milliseconds + .unwrap_or_else(|| end_time - 24 * 60 * 60 * 1000); + + // Parse limit (default to 1 if not provided) + let limit = params + .get("limit") + .and_then(|v| v.parse::().ok()) + .unwrap_or(1); + + // Parse optional filters for balance managers + let maker_balance_manager_filter = params.get("maker_balance_manager_id").cloned(); + let taker_balance_manager_filter = params.get("taker_balance_manager_id").cloned(); + let (pool_id, base_decimals, quote_decimals) = pool_data; let base_decimals = base_decimals as u8; let quote_decimals = quote_decimals as u8; - // Fetch the last trade for the pool from the order_fills table - let last_trade = schema::order_fills::table + // Build the query dynamically + let mut query = schema::order_fills::table .filter(schema::order_fills::pool_id.eq(pool_id)) - .order_by(schema::order_fills::checkpoint_timestamp_ms.desc()) + .filter(schema::order_fills::checkpoint_timestamp_ms.between(start_time, end_time)) + .into_boxed(); + + // Apply optional filters if parameters are provided + if let Some(maker_id) = maker_balance_manager_filter { + query = query.filter(schema::order_fills::maker_balance_manager_id.eq(maker_id)); + } + if let Some(taker_id) = taker_balance_manager_filter { + query = query.filter(schema::order_fills::taker_balance_manager_id.eq(taker_id)); + } + + // Fetch latest trades (sorted by timestamp in descending order) within the time range, applying the limit + let trades = query + .order_by(schema::order_fills::checkpoint_timestamp_ms.desc()) // Ensures latest trades come first + .limit(limit) // Apply limit to get the most recent trades .select(( schema::order_fills::maker_order_id, schema::order_fills::taker_order_id, @@ -784,52 +827,73 @@ async fn trades( schema::order_fills::quote_quantity, schema::order_fills::checkpoint_timestamp_ms, schema::order_fills::taker_is_bid, + schema::order_fills::maker_balance_manager_id, + schema::order_fills::taker_balance_manager_id, )) - .first::<(String, String, i64, i64, i64, i64, bool)>(connection) + .load::<(String, String, i64, i64, i64, i64, bool, String, String)>(connection) .await .map_err(|_| { - DeepBookError::InternalError(format!("No trades found for pool '{}'", pool_name)) + DeepBookError::InternalError(format!( + "No trades found for pool '{}' in the specified time range", + pool_name + )) })?; - let ( - maker_order_id, - taker_order_id, - price, - base_quantity, - quote_quantity, - timestamp, - taker_is_bid, - ) = last_trade; - - // Calculate the `trade_id` using the external function - let trade_id = calculate_trade_id(&maker_order_id, &taker_order_id)?; - // Conversion factors for decimals let base_factor = 10u64.pow(base_decimals as u32); let quote_factor = 10u64.pow(quote_decimals as u32); let price_factor = 10u64.pow((9 - base_decimals + quote_decimals) as u32); - let trade_type = if taker_is_bid { "buy" } else { "sell" }; - // Prepare the trade data - let trade = HashMap::from([ - ("trade_id".to_string(), Value::from(trade_id.to_string())), // Computed from `maker_id` and `taker_id` - ( - "price".to_string(), - Value::from(price as f64 / price_factor as f64), - ), - ( - "base_volume".to_string(), - Value::from(base_quantity as f64 / base_factor as f64), - ), - ( - "quote_volume".to_string(), - Value::from(quote_quantity as f64 / quote_factor as f64), - ), - ("timestamp".to_string(), Value::from(timestamp as u64)), - ("type".to_string(), Value::from(trade_type)), // Trade type (buy/sell) - ]); - - Ok(Json(vec![trade])) + // Map trades to JSON format + let trade_data: Vec> = trades + .into_iter() + .map( + |( + maker_order_id, + taker_order_id, + price, + base_quantity, + quote_quantity, + timestamp, + taker_is_bid, + maker_balance_manager_id, + taker_balance_manager_id, + )| { + let trade_id = calculate_trade_id(&maker_order_id, &taker_order_id).unwrap_or(0); + let trade_type = if taker_is_bid { "buy" } else { "sell" }; + + HashMap::from([ + ("trade_id".to_string(), Value::from(trade_id.to_string())), + ("maker_order_id".to_string(), Value::from(maker_order_id)), + ("taker_order_id".to_string(), Value::from(taker_order_id)), + ( + "maker_balance_manager_id".to_string(), + Value::from(maker_balance_manager_id), + ), + ( + "taker_balance_manager_id".to_string(), + Value::from(taker_balance_manager_id), + ), + ( + "price".to_string(), + Value::from(price as f64 / price_factor as f64), + ), + ( + "base_volume".to_string(), + Value::from(base_quantity as f64 / base_factor as f64), + ), + ( + "quote_volume".to_string(), + Value::from(quote_quantity as f64 / quote_factor as f64), + ), + ("timestamp".to_string(), Value::from(timestamp as u64)), + ("type".to_string(), Value::from(trade_type)), + ]) + }, + ) + .collect(); + + Ok(Json(trade_data)) } async fn trade_count( From cfae4ac3d65dfacb1ca706bccc21dabd60689783 Mon Sep 17 00:00:00 2001 From: Jort Date: Wed, 29 Jan 2025 12:01:12 -0800 Subject: [PATCH 17/30] [cli] upgrade errors capitalize letter on declarations, remove unneeded format var = var (#21001) ## Description - capitalize letter on declarations and notes - remove unneeded var = var ## Test plan snapshots --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- ...upgrade_compatibility_tests__additive.snap | 14 +++---- ..._compatibility_tests__addresses_first.snap | 2 +- ...ity__upgrade_compatibility_tests__all.snap | 10 ++--- ...atibility_tests__declarations_missing.snap | 14 +++---- ...y__upgrade_compatibility_tests__emoji.snap | 2 +- ...y__upgrade_compatibility_tests__empty.snap | 2 +- ..._compatibility_tests__package_no_name.snap | 2 +- ...mpatibility_tests__starts_second_line.snap | 2 +- ...grade_compatibility_tests__whitespace.snap | 2 +- crates/sui/src/upgrade_compatibility/mod.rs | 39 ++++++++++++------- 10 files changed, 49 insertions(+), 40 deletions(-) diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__additive.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__additive.snap index 0402b496262cf..ffce25130f5d3 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__additive.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__additive.snap @@ -1,5 +1,5 @@ --- -source: crates/sui/src/unit_tests/upgrade_compatibility_tests.rs +source: crates/sui/src/upgrade_compatibility/../unit_tests/upgrade_compatibility_tests.rs expression: normalize_path(err.to_string()) --- error[Compatibility E01007]: missing declaration @@ -8,8 +8,8 @@ error[Compatibility E01007]: missing declaration 4 │ module upgrades::upgrades { │ ^^^^^^^^ enum 'EnumToRemove' is missing │ - = enums cannot be removed or changed during an 'additive' or 'dependency only' upgrade. - = add missing enum 'EnumToRemove' back to the module 'upgrades'. + = Enums cannot be removed or changed during an 'additive' or 'dependency only' upgrade. + = Add missing enum 'EnumToRemove' back to the module 'upgrades'. error[Compatibility E01007]: missing declaration ┌─ /fixtures/upgrade_errors/additive_errors_v2/sources/UpgradeErrors.move:4:18 @@ -17,8 +17,8 @@ error[Compatibility E01007]: missing declaration 4 │ module upgrades::upgrades { │ ^^^^^^^^ function 'function_to_remove' is missing │ - = functions cannot be removed or changed during an 'additive' or 'dependency only' upgrade. - = add missing function 'function_to_remove' back to the module 'upgrades'. + = Functions cannot be removed or changed during an 'additive' or 'dependency only' upgrade. + = Add missing function 'function_to_remove' back to the module 'upgrades'. error[Compatibility E01007]: missing declaration ┌─ /fixtures/upgrade_errors/additive_errors_v2/sources/UpgradeErrors.move:4:18 @@ -26,8 +26,8 @@ error[Compatibility E01007]: missing declaration 4 │ module upgrades::upgrades { │ ^^^^^^^^ struct 'StructToRemove' is missing │ - = structs cannot be removed or changed during an 'additive' or 'dependency only' upgrade. - = add missing struct 'StructToRemove' back to the module 'upgrades'. + = Structs cannot be removed or changed during an 'additive' or 'dependency only' upgrade. + = Add missing struct 'StructToRemove' back to the module 'upgrades'. error[Compatibility E01002]: type mismatch ┌─ /fixtures/upgrade_errors/additive_errors_v2/sources/UpgradeErrors.move:7:9 diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__addresses_first.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__addresses_first.snap index 96b7a840b0ff8..cb13ce806b047 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__addresses_first.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__addresses_first.snap @@ -10,4 +10,4 @@ error[Compatibility E01006]: module missing │ ╰─────────────────^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__all.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__all.snap index e61275713863c..77c9cd22fcead 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__all.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__all.snap @@ -1,5 +1,5 @@ --- -source: crates/sui/src/unit_tests/upgrade_compatibility_tests.rs +source: crates/sui/src/upgrade_compatibility/../unit_tests/upgrade_compatibility_tests.rs expression: normalize_path(err.to_string()) --- error[Compatibility E01001]: missing public declaration @@ -8,8 +8,8 @@ error[Compatibility E01001]: missing public declaration 6 │ module upgrades::upgrades { │ ^^^^^^^^ enum 'EnumToBeRemoved' is missing │ - = enums are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. - = add missing enum 'EnumToBeRemoved' back to the module 'upgrades'. + = Enums are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. + = Add missing enum 'EnumToBeRemoved' back to the module 'upgrades'. error[Compatibility E01001]: missing public declaration ┌─ /fixtures/upgrade_errors/all_v2/sources/UpgradeErrors.move:6:18 @@ -17,8 +17,8 @@ error[Compatibility E01001]: missing public declaration 6 │ module upgrades::upgrades { │ ^^^^^^^^ struct 'StructToBeRemoved' is missing │ - = structs are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. - = add missing struct 'StructToBeRemoved' back to the module 'upgrades'. + = Structs are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. + = Add missing struct 'StructToBeRemoved' back to the module 'upgrades'. error[Compatibility E01003]: ability mismatch ┌─ /fixtures/upgrade_errors/all_v2/sources/UpgradeErrors.move:11:19 diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__declarations_missing.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__declarations_missing.snap index dbab1edf61c5b..4ca8bc94eb4f0 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__declarations_missing.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__declarations_missing.snap @@ -8,8 +8,8 @@ error[Compatibility E01001]: missing public declaration 4 │ module upgrades::enum_ { │ ^^^^^ enum 'EnumToBeRemoved' is missing │ - = enums are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. - = add missing enum 'EnumToBeRemoved' back to the module 'enum_'. + = Enums are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. + = Add missing enum 'EnumToBeRemoved' back to the module 'enum_'. error[Compatibility E01001]: missing public declaration ┌─ /fixtures/upgrade_errors/declaration_errors_v2/sources/func.move:4:18 @@ -17,8 +17,8 @@ error[Compatibility E01001]: missing public declaration 4 │ module upgrades::func_ { │ ^^^^^ public function 'fun_to_be_removed' is missing │ - = public functions are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. - = add missing public function 'fun_to_be_removed' back to the module 'func_'. + = Public functions are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. + = Add missing public function 'fun_to_be_removed' back to the module 'func_'. error[Compatibility E01001]: missing public declaration ┌─ /fixtures/upgrade_errors/declaration_errors_v2/sources/func.move:7:9 @@ -35,8 +35,8 @@ error[Compatibility E01001]: missing public declaration 4 │ module upgrades::struct_ { │ ^^^^^^^ struct 'StructToBeRemoved' is missing │ - = structs are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. - = add missing struct 'StructToBeRemoved' back to the module 'struct_'. + = Structs are part of a module's public interface and cannot be removed or changed during a 'compatible' upgrade. + = Add missing struct 'StructToBeRemoved' back to the module 'struct_'. error[Compatibility E01006]: module missing ┌─ /fixtures/upgrade_errors/declaration_errors_v2/Move.toml:1:1 @@ -47,7 +47,7 @@ error[Compatibility E01006]: module missing │ ╰────────────────────────────────────────────────────────────────────────^ Package is missing module 'missing_module' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'missing_module' back to the package. + = Add missing module 'missing_module' back to the package. Upgrade failed, this package requires changes to be compatible with the existing package. Its upgrade policy is set to 'compatible'. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__emoji.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__emoji.snap index 5a5674d2208ba..e109d0fe1edc6 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__emoji.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__emoji.snap @@ -12,4 +12,4 @@ error[Compatibility E01006]: module missing │ ╰──────────────────^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__empty.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__empty.snap index 68ce2b68f1658..5acea1da7822f 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__empty.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__empty.snap @@ -9,4 +9,4 @@ error[Compatibility E01006]: module missing │ ^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__package_no_name.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__package_no_name.snap index 23157a8904b02..b91b9966ad203 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__package_no_name.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__package_no_name.snap @@ -9,4 +9,4 @@ error[Compatibility E01006]: module missing │ ^^^^^^^^^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__starts_second_line.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__starts_second_line.snap index 24757e70124d1..dfbcb6c6608ea 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__starts_second_line.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__starts_second_line.snap @@ -11,4 +11,4 @@ error[Compatibility E01006]: module missing │ ╰────────────────────────────────────────────────────────────────────────^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__whitespace.snap b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__whitespace.snap index f6fd0b30619f0..8435927e8ea82 100644 --- a/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__whitespace.snap +++ b/crates/sui/src/unit_tests/snapshots/sui__upgrade_compatibility__upgrade_compatibility_tests__whitespace.snap @@ -10,4 +10,4 @@ error[Compatibility E01006]: module missing │ ╰──^ Package is missing module 'identifier' │ = Modules which are part package cannot be removed during an upgrade. - = add missing module 'identifier' back to the package. + = Add missing module 'identifier' back to the package. diff --git a/crates/sui/src/upgrade_compatibility/mod.rs b/crates/sui/src/upgrade_compatibility/mod.rs index 2cfd834cdde14..f9da6d38137bc 100644 --- a/crates/sui/src/upgrade_compatibility/mod.rs +++ b/crates/sui/src/upgrade_compatibility/mod.rs @@ -1031,11 +1031,11 @@ fn missing_module_diag( diags.add(Diagnostic::new( Declarations::ModuleMissing, - (loc, format!("Package is missing module '{module_name}'",)), + (loc, format!("Package is missing module '{module_name}'")), Vec::<(Loc, String)>::new(), vec![ "Modules which are part package cannot be removed during an upgrade.".to_string(), - format!("add missing module '{module_name}' back to the package."), + format!("Add missing module '{module_name}' back to the package."), ], )); @@ -1051,6 +1051,19 @@ fn missing_definition_diag( ) -> Result { let mut diags = Diagnostics::new(); + // capitalize the first letter + let capital_declaration_kind = declaration_kind + .chars() + .enumerate() + .map(|(i, c)| { + if i == 0 { + c.to_uppercase().next().unwrap_or(c) + } else { + c + } + }) + .collect::(); + let module_name = compiled_unit_with_source.unit.name.as_str(); let loc = compiled_unit_with_source .unit @@ -1062,11 +1075,11 @@ fn missing_definition_diag( Declarations::PublicMissing, vec![ format!( - "{declaration_kind}s are part of a module's public interface \ + "{capital_declaration_kind}s are part of a module's public interface \ and cannot be removed or changed during a 'compatible' upgrade.", ), format!( - "add missing {declaration_kind} '{identifier_name}' \ + "Add missing {declaration_kind} '{identifier_name}' \ back to the module '{module_name}'.", ), ], @@ -1076,11 +1089,11 @@ fn missing_definition_diag( Declarations::Missing, vec![ format!( - "{declaration_kind}s cannot be removed or changed during an 'additive' or \ + "{capital_declaration_kind}s cannot be removed or changed during an 'additive' or \ 'dependency only' upgrade.", ), format!( - "add missing {declaration_kind} '{identifier_name}' \ + "Add missing {declaration_kind} '{identifier_name}' \ back to the module '{module_name}'.", ), ], @@ -1091,11 +1104,7 @@ fn missing_definition_diag( code, ( loc, - format!( - "{declaration_kind} '{identifier_name}' is missing", - declaration_kind = declaration_kind, - identifier_name = identifier_name, - ), + format!("{declaration_kind} '{identifier_name}' is missing"), ), std::iter::empty::<(Loc, String)>(), [reason_notes].concat(), @@ -1129,7 +1138,7 @@ fn function_lost_public( Declarations::PublicMissing, ( def_loc, - format!("Function '{function_name}' has lost its public visibility",), + format!("Function '{function_name}' has lost its public visibility"), ), Vec::<(Loc, String)>::new(), vec![ @@ -2040,7 +2049,7 @@ fn enum_variant_missing_diag( Enums::VariantMismatch, ( enum_sourcemap.definition_location, - format!("Missing variant '{variant_name}'.",), + format!("Missing variant '{variant_name}'."), ), Vec::<(Loc, String)>::new(), vec![ @@ -2085,7 +2094,7 @@ fn struct_new_diag( Vec::<(Loc, String)>::new(), vec![ "Structs cannot be added during a 'dependency only' upgrade.".to_string(), - format!("Remove the struct '{struct_name}' from its module.",), + format!("Remove the struct '{struct_name}' from its module."), ], )); @@ -2156,7 +2165,7 @@ fn enum_new_diag( Vec::<(Loc, String)>::new(), vec![ "Enums cannot be added during a 'dependency only' upgrade.".to_string(), - format!("Remove the enum '{enum_name}' from its module.",), + format!("Remove the enum '{enum_name}' from its module."), ], )); From 1ce2726ead2169ecca883cd51b50e9dfcfd06b36 Mon Sep 17 00:00:00 2001 From: "Michael D. George" Date: Wed, 29 Jan 2025 16:44:10 -0500 Subject: [PATCH 18/30] Migrate sui-move shell tests to sui crate (#20995) ## Description This PR migrates the shell-script-based tests from `sui-move` to `sui`, and extends the test runner to create a TestCluster for tests in a certain subdirectory. Some of the existing tests have been migrated from `cli_tests.rs` to the new infrastructure as well. ## Test plan There are some dummy tests to make sure the infrastructure is working properly. I will manually check that CI is running them! --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- Cargo.lock | 2 + crates/sui-move/Cargo.toml | 5 +- crates/sui-move/tests/cli_tests.rs | 79 --------------- .../snapshots/cli_tests__dummy__dummy.sh.snap | 11 --- ...tests__new_tests__gitignore_exists.sh.snap | 11 --- ...ts__new_tests__gitignore_has_build.sh.snap | 18 ---- .../cli_tests__new_tests__new_files.sh.snap | 18 ---- ...i_tests__new_tests__new_then_build.sh.snap | 12 --- ...li_tests__new_tests__new_then_test.sh.snap | 14 --- crates/sui/Cargo.toml | 7 +- crates/sui/tests/cli_tests.rs | 90 ------------------ crates/sui/tests/shell_tests.rs | 95 +++++++++++++++++++ .../tests/shell_tests}/dummy/data/data.txt | 0 .../tests/shell_tests}/dummy/dummy.sh | 2 +- .../new_tests/gitignore_exists.sh | 4 +- .../new_tests/gitignore_has_build.sh | 4 +- .../new_tests/manifest_template.sh | 2 +- .../tests/shell_tests}/new_tests/new_files.sh | 4 +- .../shell_tests}/new_tests/new_then_build.sh | 8 +- .../shell_tests}/new_tests/new_then_test.sh | 8 +- .../tests/shell_tests/with_network/dummy.sh | 5 + .../depends_on_simple/Move.toml | 0 .../sources/depends_on_simple.move | 0 ..._build_bytecode_with_address_resolution.sh | 9 ++ .../simple/Move.toml | 0 .../simple/sources/simple.move | 0 .../with_network/regression_6546.sh | 12 +++ .../shell_tests__dummy__dummy.sh.snap | 21 ++++ ...tests__new_tests__gitignore_exists.sh.snap | 22 +++++ ...ts__new_tests__gitignore_has_build.sh.snap | 34 +++++++ ...sts__new_tests__manifest_template.sh.snap} | 12 ++- .../shell_tests__new_tests__new_files.sh.snap | 32 +++++++ ...l_tests__new_tests__new_then_build.sh.snap | 33 +++++++ ...ll_tests__new_tests__new_then_test.sh.snap | 34 +++++++ .../shell_tests__with_network__dummy.sh.snap | 18 ++++ ...d_bytecode_with_address_resolution.sh.snap | 28 ++++++ ...sts__with_network__regression_6546.sh.snap | 27 ++++++ 37 files changed, 405 insertions(+), 276 deletions(-) delete mode 100644 crates/sui-move/tests/cli_tests.rs delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__dummy__dummy.sh.snap delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_exists.sh.snap delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_has_build.sh.snap delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__new_tests__new_files.sh.snap delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_build.sh.snap delete mode 100644 crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_test.sh.snap create mode 100644 crates/sui/tests/shell_tests.rs rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/dummy/data/data.txt (100%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/dummy/dummy.sh (89%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/gitignore_exists.sh (64%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/gitignore_has_build.sh (78%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/manifest_template.sh (81%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/new_files.sh (75%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/new_then_build.sh (72%) rename crates/{sui-move/tests/tests => sui/tests/shell_tests}/new_tests/new_then_test.sh (72%) create mode 100644 crates/sui/tests/shell_tests/with_network/dummy.sh rename crates/sui/tests/{data => shell_tests/with_network/move_build_bytecode_with_address_resolution}/depends_on_simple/Move.toml (100%) rename crates/sui/tests/{data => shell_tests/with_network/move_build_bytecode_with_address_resolution}/depends_on_simple/sources/depends_on_simple.move (100%) create mode 100644 crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/move_build_bytecode_with_address_resolution.sh rename crates/sui/tests/{data => shell_tests/with_network/move_build_bytecode_with_address_resolution}/simple/Move.toml (100%) rename crates/sui/tests/{data => shell_tests/with_network/move_build_bytecode_with_address_resolution}/simple/sources/simple.move (100%) create mode 100644 crates/sui/tests/shell_tests/with_network/regression_6546.sh create mode 100644 crates/sui/tests/snapshots/shell_tests__dummy__dummy.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_exists.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_has_build.sh.snap rename crates/{sui-move/tests/snapshots/cli_tests__new_tests__manifest_template.sh.snap => sui/tests/snapshots/shell_tests__new_tests__manifest_template.sh.snap} (85%) create mode 100644 crates/sui/tests/snapshots/shell_tests__new_tests__new_files.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__new_tests__new_then_build.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__new_tests__new_then_test.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__with_network__dummy.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__with_network__move_build_bytecode_with_address_resolution__move_build_bytecode_with_address_resolution.sh.snap create mode 100644 crates/sui/tests/snapshots/shell_tests__with_network__regression_6546.sh.snap diff --git a/Cargo.lock b/Cargo.lock index fcb3076cc407f..d9f02b3fcfbda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12742,6 +12742,7 @@ dependencies = [ "im", "inquire", "insta", + "insta-cmd", "jemalloc-ctl", "json_to_table", "miette", @@ -14521,6 +14522,7 @@ dependencies = [ "clap", "colored", "datatest-stable", + "fs_extra", "futures", "insta", "insta-cmd", diff --git a/crates/sui-move/Cargo.toml b/crates/sui-move/Cargo.toml index c4c6c3dc73ef6..e3b6e81573777 100644 --- a/crates/sui-move/Cargo.toml +++ b/crates/sui-move/Cargo.toml @@ -49,6 +49,7 @@ tempfile.workspace = true walkdir.workspace = true insta-cmd.workspace = true insta.workspace = true +fs_extra.workspace = true move-package.workspace = true @@ -61,9 +62,5 @@ sui-simulator.workspace = true [package.metadata.cargo-udeps.ignore] normal = ["jemalloc-ctl"] -[[test]] -name = "cli_tests" -harness = false - [lints] workspace = true diff --git a/crates/sui-move/tests/cli_tests.rs b/crates/sui-move/tests/cli_tests.rs deleted file mode 100644 index 990940433dd18..0000000000000 --- a/crates/sui-move/tests/cli_tests.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Mysten Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use insta_cmd::get_cargo_bin; -use std::fs; -use std::path::Path; -use std::process::Command; -use walkdir::WalkDir; - -// [test_shell_snapshot] is run on every file matching [TEST_PATTERN] in [TEST_DIR]; this runs the -// files as shell scripts and compares their output to the snapshots; use `cargo insta test -// --review` to update the snapshots. - -const TEST_DIR: &str = "tests/tests"; -const TEST_PATTERN: &str = r"^test.*\.sh$"; - -/// run the bash script at [path], comparing its output to the insta snapshot of the same name. -/// The script is run in a temporary working directory that contains a copy of the parent directory -/// of [path], with the `sui-move` binary on the path. -fn test_shell_snapshot(path: &Path) -> datatest_stable::Result<()> { - // copy files into temporary directory - let srcdir = path.parent().unwrap(); - let tmpdir = tempfile::tempdir()?; - let sandbox = tmpdir.path().join("sandbox"); - - for entry in WalkDir::new(srcdir) { - let entry = entry.unwrap(); - let srcfile = entry.path(); - let dstfile = sandbox.join(srcfile.strip_prefix(srcdir)?); - if srcfile.is_dir() { - fs::create_dir_all(dstfile)?; - } else { - fs::copy(srcfile, dstfile)?; - } - } - - // set up command - let mut shell = Command::new("bash"); - shell - .env("PATH", format!("/bin:/usr/bin:{}", get_sui_move_path())) - .current_dir(sandbox) - .arg(path.file_name().unwrap()); - - // run it; snapshot test output - let output = shell.output()?; - let result = format!( - "success: {:?}\nexit_code: {}\n----- stdout -----\n{}\n----- stderr -----\n{}", - output.status.success(), - output.status.code().unwrap_or(!0), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - - let snapshot_name: String = path - .strip_prefix("tests/tests")? - .to_string_lossy() - .to_string(); - - insta::with_settings!({description => path.to_string_lossy(), omit_expression => true}, { - insta::assert_snapshot!(snapshot_name, result); - }); - - Ok(()) -} - -fn get_sui_move_path() -> String { - get_cargo_bin("sui-move") - .parent() - .unwrap() - .to_str() - .expect("directory name is valid UTF-8") - .to_owned() -} - -#[cfg(not(msim))] -datatest_stable::harness!(test_shell_snapshot, TEST_DIR, TEST_PATTERN); - -#[cfg(msim)] -fn main() {} diff --git a/crates/sui-move/tests/snapshots/cli_tests__dummy__dummy.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__dummy__dummy.sh.snap deleted file mode 100644 index 15f146522559f..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__dummy__dummy.sh.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/dummy/dummy.sh ---- -success: true -exit_code: 0 ------ stdout ----- -dummy test -some dummy data - ------ stderr ----- diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_exists.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_exists.sh.snap deleted file mode 100644 index 98273f8ed0c71..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_exists.sh.snap +++ /dev/null @@ -1,11 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/gitignore_exists.sh ---- -success: true -exit_code: 0 ------ stdout ----- -existing_ignore -build/* - ------ stderr ----- diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_has_build.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_has_build.sh.snap deleted file mode 100644 index 5af9fb5d7b873..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__gitignore_has_build.sh.snap +++ /dev/null @@ -1,18 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/gitignore_has_build.sh ---- -success: true -exit_code: 0 ------ stdout ----- -ignore1 -build/* -ignore2 - -==== files in example/ ==== -.gitignore -Move.toml -sources -tests - ------ stderr ----- diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_files.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_files.sh.snap deleted file mode 100644 index 2f8a7212059dd..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_files.sh.snap +++ /dev/null @@ -1,18 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/new_files.sh ---- -success: true -exit_code: 0 ------ stdout ----- -==== files in project ==== -.gitignore -Move.toml -sources -tests -==== files in sources ==== -example.move -==== files in tests ===== -example_tests.move - ------ stderr ----- diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_build.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_build.sh.snap deleted file mode 100644 index ceff62071690e..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_build.sh.snap +++ /dev/null @@ -1,12 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/new_then_build.sh ---- -success: true -exit_code: 0 ------ stdout ----- - ------ stderr ----- -INCLUDING DEPENDENCY Sui -INCLUDING DEPENDENCY MoveStdlib -BUILDING example diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_test.sh.snap b/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_test.sh.snap deleted file mode 100644 index 63ed656458958..0000000000000 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__new_then_test.sh.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/new_then_test.sh ---- -success: true -exit_code: 0 ------ stdout ----- -INCLUDING DEPENDENCY Sui -INCLUDING DEPENDENCY MoveStdlib -BUILDING example -Running Move unit tests -Test result: OK. Total tests: 0; passed: 0; failed: 0 - ------ stderr ----- diff --git a/crates/sui/Cargo.toml b/crates/sui/Cargo.toml index 08e35e0639baf..0c043560dd70b 100644 --- a/crates/sui/Cargo.toml +++ b/crates/sui/Cargo.toml @@ -26,7 +26,6 @@ futures.workspace = true http.workspace = true im.workspace = true inquire.workspace = true -insta.workspace = true json_to_table.workspace = true miette.workspace = true num-bigint.workspace = true @@ -110,6 +109,8 @@ fs_extra.workspace = true expect-test.workspace = true assert_cmd.workspace = true toml.workspace = true +insta.workspace = true +insta-cmd.workspace = true test-cluster.workspace = true sui-macros.workspace = true @@ -132,6 +133,10 @@ test = false name = "ptb_files_tests" harness = false +[[test]] +name = "shell_tests" +harness = false + [features] tracing = [ "sui-types/tracing", diff --git a/crates/sui/tests/cli_tests.rs b/crates/sui/tests/cli_tests.rs index aa60fc2305b45..8a9bd44335f37 100644 --- a/crates/sui/tests/cli_tests.rs +++ b/crates/sui/tests/cli_tests.rs @@ -355,43 +355,6 @@ async fn test_ptb_publish() -> Result<(), anyhow::Error> { Ok(()) } -// fixing issue https://github.com/MystenLabs/sui/issues/6546 -#[tokio::test] -async fn test_regression_6546() -> Result<(), anyhow::Error> { - let mut test_cluster = TestClusterBuilder::new().build().await; - let address = test_cluster.get_address_0(); - let context = &mut test_cluster.wallet; - - let SuiClientCommandResult::Objects(coins) = SuiClientCommands::Objects { - address: Some(KeyIdentity::Address(address)), - } - .execute(context) - .await? - else { - panic!() - }; - let config_path = test_cluster.swarm.dir().join(SUI_CLIENT_CONFIG); - - test_with_sui_binary(&[ - "client", - "--client.config", - config_path.to_str().unwrap(), - "call", - "--package", - "0x2", - "--module", - "sui", - "--function", - "transfer", - "--args", - &coins.first().unwrap().object()?.object_id.to_string(), - &test_cluster.get_address_1().to_string(), - "--gas-budget", - "100000000", - ]) - .await -} - #[sim_test] async fn test_custom_genesis() -> Result<(), anyhow::Error> { // Create and save genesis config file @@ -3944,59 +3907,6 @@ async fn test_clever_errors() -> Result<(), anyhow::Error> { Ok(()) } -#[tokio::test] -async fn test_move_build_bytecode_with_address_resolution() -> Result<(), anyhow::Error> { - let test_cluster = TestClusterBuilder::new().build().await; - let config_path = test_cluster.swarm.dir().join(SUI_CLIENT_CONFIG); - - // Package setup: a simple package depends on another and copied to tmpdir - let mut simple_package_path = PathBuf::from(TEST_DATA_DIR); - simple_package_path.push("simple"); - - let mut depends_on_simple_package_path = PathBuf::from(TEST_DATA_DIR); - depends_on_simple_package_path.push("depends_on_simple"); - - let tmp_dir = tempfile::tempdir().unwrap(); - - fs_extra::dir::copy( - &simple_package_path, - &tmp_dir, - &fs_extra::dir::CopyOptions::default(), - )?; - - fs_extra::dir::copy( - &depends_on_simple_package_path, - &tmp_dir, - &fs_extra::dir::CopyOptions::default(), - )?; - - // Publish simple package. - let simple_tmp_dir = tmp_dir.path().join("simple"); - test_with_sui_binary(&[ - "client", - "--client.config", - config_path.to_str().unwrap(), - "publish", - simple_tmp_dir.to_str().unwrap(), - ]) - .await?; - - // Build the package that depends on 'simple' package. Addresses must resolve successfully - // from the `Move.lock` for this command to succeed at all. - let depends_on_simple_tmp_dir = tmp_dir.path().join("depends_on_simple"); - test_with_sui_binary(&[ - "move", - "--client.config", - config_path.to_str().unwrap(), - "build", - "--dump-bytecode-as-base64", - "--path", - depends_on_simple_tmp_dir.to_str().unwrap(), - ]) - .await?; - Ok(()) -} - #[tokio::test] async fn test_parse_host_port() { let input = "127.0.0.0"; diff --git a/crates/sui/tests/shell_tests.rs b/crates/sui/tests/shell_tests.rs new file mode 100644 index 0000000000000..c899913138294 --- /dev/null +++ b/crates/sui/tests/shell_tests.rs @@ -0,0 +1,95 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use fs_extra::dir::CopyOptions; +use insta_cmd::get_cargo_bin; +use std::fs; +use std::path::Path; +use std::process::Command; +use sui_config::SUI_CLIENT_CONFIG; +use test_cluster::TestClusterBuilder; + +// [test_shell_snapshot] is run on every file matching [TEST_PATTERN] in [TEST_DIR]. +// Files in [TEST_NET_DIR] will be run with a [TestCluster] configured. +// +// These run the files as shell scripts and compares their output to the snapshots; use `cargo +// insta test --review` to update the snapshots. + +const TEST_DIR: &str = "tests/shell_tests"; +const TEST_NET_DIR: &str = "tests/shell_tests/with_network"; +const TEST_PATTERN: &str = r"\.sh$"; + +/// run the bash script at [path], comparing its output to the insta snapshot of the same name. +/// The script is run in a temporary working directory that contains a copy of the parent directory +/// of [path], with the `sui` binary on the path. +/// +/// If [cluster] is provided, the config file for the cluster is passed as the `CONFIG` environment +/// variable. +#[tokio::main] +async fn test_shell_snapshot(path: &Path) -> datatest_stable::Result<()> { + // set up test cluster + let cluster = if path.starts_with(TEST_NET_DIR) { + Some(TestClusterBuilder::new().build().await) + } else { + None + }; + + // copy files into temporary directory + let srcdir = path.parent().unwrap(); + let tmpdir = tempfile::tempdir()?; + let sandbox = tmpdir.path(); + + fs_extra::dir::copy(srcdir, sandbox, &CopyOptions::new().content_only(true))?; + + // set up command + let mut shell = Command::new("bash"); + shell + .env( + "PATH", + format!("{}:{}", get_sui_bin_path(), std::env::var("PATH")?), + ) + .current_dir(sandbox) + .arg(path.file_name().unwrap()); + + if let Some(ref cluster) = cluster { + shell.env("CONFIG", cluster.swarm.dir().join(SUI_CLIENT_CONFIG)); + } + + // run it; snapshot test output + let output = shell.output()?; + let result = format!( + "----- script -----\n{}\n----- results -----\nsuccess: {:?}\nexit_code: {}\n----- stdout -----\n{}\n----- stderr -----\n{}", + fs::read_to_string(path)?, + output.status.success(), + output.status.code().unwrap_or(!0), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let snapshot_name: String = path + .strip_prefix("tests/shell_tests")? + .to_string_lossy() + .to_string(); + + insta::with_settings!({description => path.to_string_lossy(), omit_expression => true}, { + insta::assert_snapshot!(snapshot_name, result); + }); + + Ok(()) +} + +/// return the path to the `sui` binary that is currently under test +fn get_sui_bin_path() -> String { + get_cargo_bin("sui") + .parent() + .unwrap() + .to_str() + .expect("directory name is valid UTF-8") + .to_owned() +} + +#[cfg(not(msim))] +datatest_stable::harness!(test_shell_snapshot, TEST_DIR, TEST_PATTERN); + +#[cfg(msim)] +fn main() {} diff --git a/crates/sui-move/tests/tests/dummy/data/data.txt b/crates/sui/tests/shell_tests/dummy/data/data.txt similarity index 100% rename from crates/sui-move/tests/tests/dummy/data/data.txt rename to crates/sui/tests/shell_tests/dummy/data/data.txt diff --git a/crates/sui-move/tests/tests/dummy/dummy.sh b/crates/sui/tests/shell_tests/dummy/dummy.sh similarity index 89% rename from crates/sui-move/tests/tests/dummy/dummy.sh rename to crates/sui/tests/shell_tests/dummy/dummy.sh index 55dc94f21fbc1..c6359b0c9bc45 100644 --- a/crates/sui-move/tests/tests/dummy/dummy.sh +++ b/crates/sui/tests/shell_tests/dummy/dummy.sh @@ -4,4 +4,4 @@ # simple test just to make sure the test runner works echo "dummy test" cat data/data.txt -sui-move new dummy +sui move new dummy diff --git a/crates/sui-move/tests/tests/new_tests/gitignore_exists.sh b/crates/sui/tests/shell_tests/new_tests/gitignore_exists.sh similarity index 64% rename from crates/sui-move/tests/tests/new_tests/gitignore_exists.sh rename to crates/sui/tests/shell_tests/new_tests/gitignore_exists.sh index 90bc5f8cf9d7b..1777a7051a217 100644 --- a/crates/sui-move/tests/tests/new_tests/gitignore_exists.sh +++ b/crates/sui/tests/shell_tests/new_tests/gitignore_exists.sh @@ -1,8 +1,8 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -# check that sui-move new correctly updates existing .gitignore +# check that sui move new correctly updates existing .gitignore mkdir example echo "existing_ignore" > example/.gitignore -sui-move new example +sui move new example cat example/.gitignore diff --git a/crates/sui-move/tests/tests/new_tests/gitignore_has_build.sh b/crates/sui/tests/shell_tests/new_tests/gitignore_has_build.sh similarity index 78% rename from crates/sui-move/tests/tests/new_tests/gitignore_has_build.sh rename to crates/sui/tests/shell_tests/new_tests/gitignore_has_build.sh index 4ef0230b0fee6..a00401b70ad1b 100644 --- a/crates/sui-move/tests/tests/new_tests/gitignore_has_build.sh +++ b/crates/sui/tests/shell_tests/new_tests/gitignore_has_build.sh @@ -1,12 +1,12 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -# sui-move new example when `example/.gitignore` already contains build/*; it should be unchanged +# sui move new example when `example/.gitignore` already contains build/*; it should be unchanged mkdir example echo "ignore1" >> example/.gitignore echo "build/*" >> example/.gitignore echo "ignore2" >> example/.gitignore -sui-move new example +sui move new example cat example/.gitignore echo echo ==== files in example/ ==== diff --git a/crates/sui-move/tests/tests/new_tests/manifest_template.sh b/crates/sui/tests/shell_tests/new_tests/manifest_template.sh similarity index 81% rename from crates/sui-move/tests/tests/new_tests/manifest_template.sh rename to crates/sui/tests/shell_tests/new_tests/manifest_template.sh index e8544500ab64b..d3613ca78f5d5 100644 --- a/crates/sui-move/tests/tests/new_tests/manifest_template.sh +++ b/crates/sui/tests/shell_tests/new_tests/manifest_template.sh @@ -1,5 +1,5 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -sui-move new example +sui move new example cat example/Move.toml diff --git a/crates/sui-move/tests/tests/new_tests/new_files.sh b/crates/sui/tests/shell_tests/new_tests/new_files.sh similarity index 75% rename from crates/sui-move/tests/tests/new_tests/new_files.sh rename to crates/sui/tests/shell_tests/new_tests/new_files.sh index 3f36222121448..4dcd4e4762c7a 100644 --- a/crates/sui-move/tests/tests/new_tests/new_files.sh +++ b/crates/sui/tests/shell_tests/new_tests/new_files.sh @@ -1,8 +1,8 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -# basic test that sui-move new outputs correct files -sui-move new example +# basic test that sui move new outputs correct files +sui move new example echo ==== files in project ==== ls -A example echo ==== files in sources ==== diff --git a/crates/sui-move/tests/tests/new_tests/new_then_build.sh b/crates/sui/tests/shell_tests/new_tests/new_then_build.sh similarity index 72% rename from crates/sui-move/tests/tests/new_tests/new_then_build.sh rename to crates/sui/tests/shell_tests/new_tests/new_then_build.sh index 13b2df54a9caf..26ef628b91151 100644 --- a/crates/sui-move/tests/tests/new_tests/new_then_build.sh +++ b/crates/sui/tests/shell_tests/new_tests/new_then_build.sh @@ -1,12 +1,12 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -# tests that sui-move new followed by sui-move build succeeds +# tests that sui move new followed by sui move build succeeds -sui-move new example +sui move new example # we mangle the generated toml file to replace the framework dependency with a local dependency -FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui-move##g') +FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui##g') cat example/Move.toml \ | sed 's#\(Sui = .*\)git = "[^"]*", \(.*\)#\1\2#' \ | sed 's#\(Sui = .*\), rev = "[^"]*"\(.*\)#\1\2#' \ @@ -15,4 +15,4 @@ cat example/Move.toml \ > Move.toml mv Move.toml example/Move.toml -cd example && sui-move build +cd example && sui move build diff --git a/crates/sui-move/tests/tests/new_tests/new_then_test.sh b/crates/sui/tests/shell_tests/new_tests/new_then_test.sh similarity index 72% rename from crates/sui-move/tests/tests/new_tests/new_then_test.sh rename to crates/sui/tests/shell_tests/new_tests/new_then_test.sh index d6ba83403a7b4..051d2945589a5 100644 --- a/crates/sui-move/tests/tests/new_tests/new_then_test.sh +++ b/crates/sui/tests/shell_tests/new_tests/new_then_test.sh @@ -1,11 +1,11 @@ # Copyright (c) Mysten Labs, Inc. # SPDX-License-Identifier: Apache-2.0 -# check that sui-move new followed by sui-move test succeeds -sui-move new example +# check that sui move new followed by sui move test succeeds +sui move new example # we mangle the generated toml file to replace the framework dependency with a local dependency -FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui-move##g') +FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui##g') cat example/Move.toml \ | sed 's#\(Sui = .*\)git = "[^"]*", \(.*\)#\1\2#' \ | sed 's#\(Sui = .*\), rev = "[^"]*"\(.*\)#\1\2#' \ @@ -14,4 +14,4 @@ cat example/Move.toml \ > Move.toml mv Move.toml example/Move.toml -cd example && sui-move test +cd example && sui move test diff --git a/crates/sui/tests/shell_tests/with_network/dummy.sh b/crates/sui/tests/shell_tests/with_network/dummy.sh new file mode 100644 index 0000000000000..b7da468061edc --- /dev/null +++ b/crates/sui/tests/shell_tests/with_network/dummy.sh @@ -0,0 +1,5 @@ +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# simple test just to make sure the test runner works with the network +sui client --client.config $CONFIG objects --json | jq 'length' diff --git a/crates/sui/tests/data/depends_on_simple/Move.toml b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/depends_on_simple/Move.toml similarity index 100% rename from crates/sui/tests/data/depends_on_simple/Move.toml rename to crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/depends_on_simple/Move.toml diff --git a/crates/sui/tests/data/depends_on_simple/sources/depends_on_simple.move b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/depends_on_simple/sources/depends_on_simple.move similarity index 100% rename from crates/sui/tests/data/depends_on_simple/sources/depends_on_simple.move rename to crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/depends_on_simple/sources/depends_on_simple.move diff --git a/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/move_build_bytecode_with_address_resolution.sh b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/move_build_bytecode_with_address_resolution.sh new file mode 100644 index 0000000000000..71a79d247fa6d --- /dev/null +++ b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/move_build_bytecode_with_address_resolution.sh @@ -0,0 +1,9 @@ +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +sui client --client.config $CONFIG \ + publish simple \ + --json | jq '.effects.status' + +sui move --client.config $CONFIG \ + build --path depends_on_simple diff --git a/crates/sui/tests/data/simple/Move.toml b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/simple/Move.toml similarity index 100% rename from crates/sui/tests/data/simple/Move.toml rename to crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/simple/Move.toml diff --git a/crates/sui/tests/data/simple/sources/simple.move b/crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/simple/sources/simple.move similarity index 100% rename from crates/sui/tests/data/simple/sources/simple.move rename to crates/sui/tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/simple/sources/simple.move diff --git a/crates/sui/tests/shell_tests/with_network/regression_6546.sh b/crates/sui/tests/shell_tests/with_network/regression_6546.sh new file mode 100644 index 0000000000000..392edb20b48a3 --- /dev/null +++ b/crates/sui/tests/shell_tests/with_network/regression_6546.sh @@ -0,0 +1,12 @@ +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# fixing issue https://github.com/MystenLabs/sui/issues/6546 + +COIN=$(sui client --client.config $CONFIG objects --json | jq '.[0].data.objectId') +ADDR=$(sui client --client.config $CONFIG addresses --json | jq '.addresses[0][1]') + +sui client --client.config $CONFIG \ + call --package 0x2 --module sui --function transfer --args $COIN $ADDR \ + --gas-budget 100000000 \ + --json | jq '.effects.status' diff --git a/crates/sui/tests/snapshots/shell_tests__dummy__dummy.sh.snap b/crates/sui/tests/snapshots/shell_tests__dummy__dummy.sh.snap new file mode 100644 index 0000000000000..3efc740d07d5c --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__dummy__dummy.sh.snap @@ -0,0 +1,21 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/dummy/dummy.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# simple test just to make sure the test runner works +echo "dummy test" +cat data/data.txt +sui move new dummy + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +dummy test +some dummy data + +----- stderr ----- diff --git a/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_exists.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_exists.sh.snap new file mode 100644 index 0000000000000..8cef1cdd2fbca --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_exists.sh.snap @@ -0,0 +1,22 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/gitignore_exists.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# check that sui move new correctly updates existing .gitignore +mkdir example +echo "existing_ignore" > example/.gitignore +sui move new example +cat example/.gitignore + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +existing_ignore +build/* + +----- stderr ----- diff --git a/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_has_build.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_has_build.sh.snap new file mode 100644 index 0000000000000..798662ea1855d --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__gitignore_has_build.sh.snap @@ -0,0 +1,34 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/gitignore_has_build.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# sui move new example when `example/.gitignore` already contains build/*; it should be unchanged +mkdir example +echo "ignore1" >> example/.gitignore +echo "build/*" >> example/.gitignore +echo "ignore2" >> example/.gitignore +sui move new example +cat example/.gitignore +echo +echo ==== files in example/ ==== +ls -A example + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +ignore1 +build/* +ignore2 + +==== files in example/ ==== +.gitignore +Move.toml +sources +tests + +----- stderr ----- diff --git a/crates/sui-move/tests/snapshots/cli_tests__new_tests__manifest_template.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__manifest_template.sh.snap similarity index 85% rename from crates/sui-move/tests/snapshots/cli_tests__new_tests__manifest_template.sh.snap rename to crates/sui/tests/snapshots/shell_tests__new_tests__manifest_template.sh.snap index b60ec26122809..64f1a3cae2ce4 100644 --- a/crates/sui-move/tests/snapshots/cli_tests__new_tests__manifest_template.sh.snap +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__manifest_template.sh.snap @@ -1,7 +1,15 @@ --- -source: crates/sui-move/tests/cli_tests.rs -description: tests/tests/new_tests/manifest_template.sh +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/manifest_template.sh --- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +sui move new example +cat example/Move.toml + +----- results ----- success: true exit_code: 0 ----- stdout ----- diff --git a/crates/sui/tests/snapshots/shell_tests__new_tests__new_files.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__new_files.sh.snap new file mode 100644 index 0000000000000..147b4a0d89e5b --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__new_files.sh.snap @@ -0,0 +1,32 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/new_files.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# basic test that sui move new outputs correct files +sui move new example +echo ==== files in project ==== +ls -A example +echo ==== files in sources ==== +ls -A example/sources +echo ==== files in tests ===== +ls -A example/tests + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +==== files in project ==== +.gitignore +Move.toml +sources +tests +==== files in sources ==== +example.move +==== files in tests ===== +example_tests.move + +----- stderr ----- diff --git a/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_build.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_build.sh.snap new file mode 100644 index 0000000000000..06501ff6f3e5a --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_build.sh.snap @@ -0,0 +1,33 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/new_then_build.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# tests that sui move new followed by sui move build succeeds + +sui move new example + +# we mangle the generated toml file to replace the framework dependency with a local dependency +FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui##g') +cat example/Move.toml \ + | sed 's#\(Sui = .*\)git = "[^"]*", \(.*\)#\1\2#' \ + | sed 's#\(Sui = .*\), rev = "[^"]*"\(.*\)#\1\2#' \ + | sed 's#\(Sui = .*\)subdir = "\([^"]*\)"\(.*\)#\1local = "FRAMEWORK/\2"\3#' \ + | sed "s#\(Sui = .*\)FRAMEWORK\(.*\)#\1$FRAMEWORK_DIR\2#" \ + > Move.toml +mv Move.toml example/Move.toml + +cd example && sui move build + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- + +----- stderr ----- +INCLUDING DEPENDENCY Sui +INCLUDING DEPENDENCY MoveStdlib +BUILDING example diff --git a/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_test.sh.snap b/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_test.sh.snap new file mode 100644 index 0000000000000..e70031835eb4f --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__new_tests__new_then_test.sh.snap @@ -0,0 +1,34 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/new_tests/new_then_test.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# check that sui move new followed by sui move test succeeds +sui move new example + +# we mangle the generated toml file to replace the framework dependency with a local dependency +FRAMEWORK_DIR=$(echo $CARGO_MANIFEST_DIR | sed 's#/crates/sui##g') +cat example/Move.toml \ + | sed 's#\(Sui = .*\)git = "[^"]*", \(.*\)#\1\2#' \ + | sed 's#\(Sui = .*\), rev = "[^"]*"\(.*\)#\1\2#' \ + | sed 's#\(Sui = .*\)subdir = "\([^"]*\)"\(.*\)#\1local = "FRAMEWORK/\2"\3#' \ + | sed "s#\(Sui = .*\)FRAMEWORK\(.*\)#\1$FRAMEWORK_DIR\2#" \ + > Move.toml +mv Move.toml example/Move.toml + +cd example && sui move test + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +INCLUDING DEPENDENCY Sui +INCLUDING DEPENDENCY MoveStdlib +BUILDING example +Running Move unit tests +Test result: OK. Total tests: 0; passed: 0; failed: 0 + +----- stderr ----- diff --git a/crates/sui/tests/snapshots/shell_tests__with_network__dummy.sh.snap b/crates/sui/tests/snapshots/shell_tests__with_network__dummy.sh.snap new file mode 100644 index 0000000000000..d443b5b10876b --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__with_network__dummy.sh.snap @@ -0,0 +1,18 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/with_network/dummy.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# simple test just to make sure the test runner works with the network +sui client --client.config $CONFIG objects --json | jq 'length' + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +5 + +----- stderr ----- diff --git a/crates/sui/tests/snapshots/shell_tests__with_network__move_build_bytecode_with_address_resolution__move_build_bytecode_with_address_resolution.sh.snap b/crates/sui/tests/snapshots/shell_tests__with_network__move_build_bytecode_with_address_resolution__move_build_bytecode_with_address_resolution.sh.snap new file mode 100644 index 0000000000000..d3afab6c23ec4 --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__with_network__move_build_bytecode_with_address_resolution__move_build_bytecode_with_address_resolution.sh.snap @@ -0,0 +1,28 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/with_network/move_build_bytecode_with_address_resolution/move_build_bytecode_with_address_resolution.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +sui client --client.config $CONFIG \ + publish simple \ + --json | jq '.effects.status' + +sui move --client.config $CONFIG \ + build --path depends_on_simple + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +{ + "status": "success" +} + +----- stderr ----- +BUILDING simple +Successfully verified dependencies on-chain against source. +INCLUDING DEPENDENCY simple +BUILDING depends_on_simple diff --git a/crates/sui/tests/snapshots/shell_tests__with_network__regression_6546.sh.snap b/crates/sui/tests/snapshots/shell_tests__with_network__regression_6546.sh.snap new file mode 100644 index 0000000000000..f31b5968fbee0 --- /dev/null +++ b/crates/sui/tests/snapshots/shell_tests__with_network__regression_6546.sh.snap @@ -0,0 +1,27 @@ +--- +source: crates/sui/tests/shell_tests.rs +description: tests/shell_tests/with_network/regression_6546.sh +--- +----- script ----- +# Copyright (c) Mysten Labs, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# fixing issue https://github.com/MystenLabs/sui/issues/6546 + +COIN=$(sui client --client.config $CONFIG objects --json | jq '.[0].data.objectId') +ADDR=$(sui client --client.config $CONFIG addresses --json | jq '.addresses[0][1]') + +sui client --client.config $CONFIG \ + call --package 0x2 --module sui --function transfer --args $COIN $ADDR \ + --gas-budget 100000000 \ + --json | jq '.effects.status' + +----- results ----- +success: true +exit_code: 0 +----- stdout ----- +{ + "status": "success" +} + +----- stderr ----- From 37743e3d3d264b382181aa1ecf0807a1a254b2ed Mon Sep 17 00:00:00 2001 From: Adam Welc Date: Wed, 29 Jan 2025 14:24:28 -0800 Subject: [PATCH 19/30] [trace-view] Start debugging in disassembly (#20963) ## Description This PR adds the ability to start trace debugging a unit test with an assembly file (rather than a source file) in active editor ## Test plan Tested manually that one can start debugging with a disassembly file opened --- .../trace-adapter/src/runtime.ts | 18 +- .../move-analyzer/trace-debug/package.json | 15 +- .../trace-debug/src/extension.ts | 208 +++++++++++++++--- 3 files changed, 200 insertions(+), 41 deletions(-) diff --git a/external-crates/move/crates/move-analyzer/trace-adapter/src/runtime.ts b/external-crates/move/crates/move-analyzer/trace-adapter/src/runtime.ts index 932ae73407402..ba991f5aaaf1b 100644 --- a/external-crates/move/crates/move-analyzer/trace-adapter/src/runtime.ts +++ b/external-crates/move/crates/move-analyzer/trace-adapter/src/runtime.ts @@ -268,15 +268,16 @@ export class Runtime extends EventEmitter { /** * Start a trace viewing session and set up the initial state of the runtime. * - * @param source path to the Move source file whose traces are to be viewed. + * @param openedFilePath path to the Move source file (or disassembled bytecode file) + * whose traces are to be viewed. * @param traceInfo trace selected for viewing. * @throws Error with a descriptive error message if starting runtime has failed. * */ - public async start(source: string, traceInfo: string, stopOnEntry: boolean): Promise { - const pkgRoot = await findPkgRoot(source); + public async start(openedFilePath: string, traceInfo: string, stopOnEntry: boolean): Promise { + const pkgRoot = await findPkgRoot(openedFilePath); if (!pkgRoot) { - throw new Error(`Cannot find package root for file: ${source}`); + throw new Error(`Cannot find package root for file: ${openedFilePath}`); } const manifest_path = path.join(pkgRoot, 'Move.toml'); @@ -287,6 +288,14 @@ export class Runtime extends EventEmitter { throw Error(`Cannot find package name in manifest file: ${manifest_path}`); } + const openedFileExt = path.extname(openedFilePath); + if (openedFileExt !== MOVE_FILE_EXT + && openedFileExt !== BCODE_FILE_EXT + && openedFileExt !== JSON_FILE_EXT) { + throw new Error(`File extension: ${openedFileExt} is not supported by trace debugger`); + } + const showDisassembly = openedFileExt === BCODE_FILE_EXT; + // create file maps for all files in the `sources` directory, including both package source // files and source files for dependencies hashToFileMap(path.join(pkgRoot, 'build', pkg_name, 'sources'), this.filesMap, MOVE_FILE_EXT); @@ -335,6 +344,7 @@ export class Runtime extends EventEmitter { currentEvent.optimizedSrcLines, currentEvent.optimizedBcodeLines ); + newFrame.showDisassembly = showDisassembly; this.frameStack = { frames: [newFrame], globals: new Map() diff --git a/external-crates/move/crates/move-analyzer/trace-debug/package.json b/external-crates/move/crates/move-analyzer/trace-debug/package.json index 9c043870680a4..ce6ecd84ab8d5 100644 --- a/external-crates/move/crates/move-analyzer/trace-debug/package.json +++ b/external-crates/move/crates/move-analyzer/trace-debug/package.json @@ -5,7 +5,7 @@ "publisher": "mysten", "icon": "images/move.png", "license": "Apache-2.0", - "version": "0.0.4", + "version": "0.0.5", "preview": true, "repository": { "url": "https://github.com/MystenLabs/sui.git", @@ -55,6 +55,16 @@ "extensions": [ ".mvb" ] + }, + { + "id": "mtrace", + "aliases": [ + "mtrace" + ], + "extensions": [ + ".json", + ".JSON" + ] } ], "breakpoints": [{ "language": "move" }, { "language": "mvb" }], @@ -69,7 +79,8 @@ ], "languages": [ "move", - "mvb" + "mvb", + "mtrace" ], "configurationAttributes": { "launch": { diff --git a/external-crates/move/crates/move-analyzer/trace-debug/src/extension.ts b/external-crates/move/crates/move-analyzer/trace-debug/src/extension.ts index 9edf2167c1126..0c8a189f29a5e 100644 --- a/external-crates/move/crates/move-analyzer/trace-debug/src/extension.ts +++ b/external-crates/move/crates/move-analyzer/trace-debug/src/extension.ts @@ -23,6 +23,10 @@ const LOG_LEVEL = 'log'; */ const DEBUGGER_TYPE = 'move-debug'; +const MOVE_FILE_EXT = ".move"; +const BCODE_FILE_EXT = ".mvb"; + + /** * Provider of on-hover information during debug session. */ @@ -36,6 +40,15 @@ class MoveEvaluatableExpressionProvider { } } +/** + * Information about a traced function. + */ +interface TracedFunctionInfo { + pkgAddr: number; + module: string; + function: string; +} + /** * Called when the extension is activated. */ @@ -75,10 +88,6 @@ export function activate(context: vscode.ExtensionContext) { const stackFrame: StackFrame = stackTraceResponse.stackFrames[0]; if (stackFrame && stackFrame.source && stackFrame.source.path !== previousSourcePath) { previousSourcePath = stackFrame.source.path; - const source = stackFrame.source; - const line = stackFrame.line; - console.log(`Frame details: ${source?.name} at line ${line}`); - const editor = vscode.window.activeTextEditor; if (editor) { const optimized_lines = stackTraceResponse.optimizedLines; @@ -171,7 +180,9 @@ class MoveConfigurationProvider implements vscode.DebugConfigurationProvider { // if launch.json is missing or empty if (!config.type && !config.request && !config.name) { const editor = vscode.window.activeTextEditor; - if (editor && editor.document.languageId === 'move') { + if (editor && (editor.document.languageId === 'move' + || editor.document.languageId === 'mvb' + || editor.document.languageId === 'mtrace')) { try { let traceInfo = await findTraceInfo(editor); @@ -206,7 +217,7 @@ class MoveConfigurationProvider implements vscode.DebugConfigurationProvider { * Finds the trace information for the current active editor. * * @param editor active text editor. - * @returns trace information of the form `::::`. + * @returns trace information of the form `::::`. * @throws Error with a descriptive error message if the trace information cannot be found. */ async function findTraceInfo(editor: vscode.TextEditor): Promise { @@ -215,14 +226,34 @@ async function findTraceInfo(editor: vscode.TextEditor): Promise { throw new Error(`Cannot find package root for file '${editor.document.uri.fsPath}'`); } - const pkgModules = findModules(editor.document.getText()); - if (pkgModules.length === 0) { - throw new Error(`Cannot find any modules in file '${editor.document.uri.fsPath}'`); + let tracedFunctions: string[] = []; + if (path.extname(editor.document.uri.fsPath) === MOVE_FILE_EXT) { + const pkgModules = findSrcModules(editor.document.getText()); + if (pkgModules.length === 0) { + throw new Error(`Cannot find any modules in file '${editor.document.uri.fsPath}'`); + } + tracedFunctions = findTracedFunctionsFromPath(pkgRoot, pkgModules); + } else if (path.extname(editor.document.uri.fsPath) === BCODE_FILE_EXT) { + const modulePattern = /\bmodule\s+\d+\.\w+\b/g; + const moduleSequences = editor.document.getText().match(modulePattern); + if (!moduleSequences || moduleSequences.length === 0) { + throw new Error(`Cannot find module declaration in disassembly file '${editor.document.uri.fsPath}'`); + } + // there should be only one module declaration in a disassembly file + const [pkgAddrStr, module] = moduleSequences[0].substring('module'.length).trim().split('.'); + const pkgAddr = parseInt(pkgAddrStr); + if (isNaN(pkgAddr)) { + throw new Error(`Cannot parse package address from '${pkgAddrStr}' in disassembly file '${editor.document.uri.fsPath}'`); + } + tracedFunctions = findTracedFunctionsFromTrace(pkgRoot, pkgAddr, module); + } else { + // this is a JSON (hopefully) trace as this function is only called if + // the active file is either a .move, .mvb, or .json file + const fpath = editor.document.uri.fsPath; + const tracedFunctionInfo = getTracedFunctionInfo(fpath); + tracedFunctions = [constructTraceInfo(fpath, tracedFunctionInfo)]; } - - const tracedFunctions = findTracedFunctions(pkgRoot, pkgModules); - - if (tracedFunctions.length === 0) { + if (!tracedFunctions || tracedFunctions.length === 0) { throw new Error(`No traced functions found for package at '${pkgRoot}'`); } @@ -233,7 +264,6 @@ async function findTraceInfo(editor: vscode.TextEditor): Promise { if (!fun) { throw new Error(`No function to be trace-debugged selected from\n` + tracedFunctions.join('\n')); } - return fun; } @@ -268,7 +298,7 @@ async function findPkgRoot(active_file_path: string): Promise::`. * We cannot rely on the directory structure to find modules because * trace info is generated based on module names in the source files. @@ -276,7 +306,7 @@ async function findPkgRoot(active_file_path: string): Promise::`. */ -function findModules(file_content: string): string[] { +function findSrcModules(file_content: string): string[] { const modulePattern = /\bmodule\s+\w+::\w+\b/g; const moduleSequences = file_content.match(modulePattern); return moduleSequences @@ -285,28 +315,19 @@ function findModules(file_content: string): string[] { } /** - * Find all functions that have a corresponding trace file. + * Find all functions that have a corresponding trace file by looking at + * the trace file names that have the following format and extracting all + * function names that match: + * `____.json`. * * @param pkgRoot root directory of the package. * @param pkgModules modules in the package of the form `::`. - * @returns list of functions of the form `::::`. - * @throws Error (containing a descriptive message) if no trace files are found for the package. + * @returns list of functions of the form `::::`. + * @throws Error (containing a descriptive message) if no traced functions are found for the package. */ -function findTracedFunctions(pkgRoot: string, pkgModules: string[]): string[] { +function findTracedFunctionsFromPath(pkgRoot: string, pkgModules: string[]): string[] { - function getFiles(tracesDir: string): string[] { - try { - return fs.readdirSync(tracesDir); - } catch (err) { - throw new Error(`Error accessing 'traces' directory for package at '${pkgRoot}'`); - } - } - const tracesDir = path.join(pkgRoot, 'traces'); - - const filePaths = getFiles(tracesDir); - if (filePaths.length === 0) { - throw new Error(`No trace files for package at ${pkgRoot}`); - } + const filePaths = getTraceFiles(pkgRoot); const result: [string, string[]][] = []; pkgModules.forEach((module) => { @@ -327,11 +348,128 @@ function findTracedFunctions(pkgRoot: string, pkgModules: string[]): string[] { }).flat(); } +/** + * Find all functions that have a corresponding trace file by looking at + * the content of the trace file and its name (`____.json`). + * We need to match the package address, module name, and function name in the trace + * file itself as this is the only place where we can find the (potentially matching) + * package address (module name and function name could be extracted from the trace + * file name). + * + * @param pkgRoot root directory of the package. + * @param pkgAddr package address. + * @param module module name. + * @returns list of functions of the form `::::`. + * @throws Error (containing a descriptive message) if no traced functions are found for the package. + */ +function findTracedFunctionsFromTrace(pkgRoot: string, pkgAddr: number, module: string): string[] { + const filePaths = getTraceFiles(pkgRoot); + const result: string[] = []; + for (const p of filePaths) { + const tracePath = path.join(pkgRoot, 'traces', p); + const tracedFunctionInfo = getTracedFunctionInfo(tracePath); + if (tracedFunctionInfo.pkgAddr === pkgAddr && tracedFunctionInfo.module === module) { + result.push(constructTraceInfo(tracePath, tracedFunctionInfo)); + } + } + return result; +} + +/** + * Retrieves traced function info from the trace file. + * + * @param tracePath path to the trace file. + * @returns traced function info containing package address, module, and function itself. + */ +function getTracedFunctionInfo(tracePath: string): TracedFunctionInfo { + let traceContent = undefined; + try { + traceContent = fs.readFileSync(tracePath, 'utf-8'); + } catch { + throw new Error(`Error reading trace file '${tracePath}'`); + } + + const trace = JSON.parse(traceContent); + if (!trace) { + throw new Error(`Error parsing trace file '${tracePath}'`); + } + if (trace.events.length === 0) { + throw new Error(`Empty trace file '${tracePath}'`); + } + const frame = trace.events[0]?.OpenFrame?.frame; + const pkgAddrStrInTrace = frame?.module?.address; + if (!pkgAddrStrInTrace) { + throw new Error(`No package address for the initial frame in trace file '${tracePath}'`); + } + const pkgAddrInTrace = parseInt(pkgAddrStrInTrace); + if (isNaN(pkgAddrInTrace)) { + throw new Error('Cannot parse package address ' + + pkgAddrStrInTrace + + ' for the initial frame in trace file ' + + tracePath); + } + const moduleInTrace = frame?.module?.name; + if (!moduleInTrace) { + throw new Error(`No module name for the initial frame in trace file '${tracePath}'`); + } + const functionInTrace = frame?.function_name; + if (!functionInTrace) { + throw new Error(`No function name for the initial frame in trace file '${tracePath}'`); + } + return { + pkgAddr: pkgAddrInTrace, + module: moduleInTrace, + function: functionInTrace + }; +} + +/** + * Given trace file path and traced function, constructs a string of the form + * `::::`, taking package from the trace file name + * (module name and function are the same in the file name and in the trace itself). + * + * @param tracePath path to the trace file. + * @param tracedFunctionInfo traced function info. + * @returns string of the form `::::`. + */ +function constructTraceInfo(tracePath: string, tracedFunctionInfo: TracedFunctionInfo): string { + const tracedFileBaseName = path.basename(tracePath, path.extname(tracePath)); + const fileBaseNameSuffix = '__' + tracedFunctionInfo.module + '__' + tracedFunctionInfo.function; + if (!tracedFileBaseName.endsWith(fileBaseNameSuffix)) { + throw new Error('Trace file name (' + tracedFileBaseName + ')' + + 'does not end with expected suffix (' + fileBaseNameSuffix + ')' + + ' obtained from concateneting module and entry function found in the trace'); + } + const pkgName = tracedFileBaseName.substring(0, tracedFileBaseName.length - fileBaseNameSuffix.length); + return pkgName + '::' + tracedFunctionInfo.module + '::' + tracedFunctionInfo.function; +} + +/** + * Return list of trace files for a given package. + * + * @param pkgRoot root directory of the package. + * @returns list of trace files for the package. + * @throws Error (containing a descriptive message) if no trace files are found for the package. + */ +function getTraceFiles(pkgRoot: string): string[] { + const tracesDir = path.join(pkgRoot, 'traces'); + let filePaths = []; + try { + filePaths = fs.readdirSync(tracesDir); + } catch (err) { + throw new Error(`Error accessing 'traces' directory for package at '${pkgRoot}'`); + } + if (filePaths.length === 0) { + throw new Error(`No trace files for package at ${pkgRoot}`); + } + return filePaths; +} + /** * Prompts the user to select a function to debug from a list of traced functions. * - * @param tracedFunctions list of traced functions of the form `::::`. - * @returns single function to debug of the form `::::`. + * @param tracedFunctions list of traced functions of the form `::::`. + * @returns single function to debug of the form `::::`. */ async function pickFunctionToDebug(tracedFunctions: string[]): Promise { const selectedFunction = await vscode.window.showQuickPick(tracedFunctions.map(pkgFun => { From 8e6047e06119d3d0b16a88603e321d29697d18a9 Mon Sep 17 00:00:00 2001 From: Todd Nowacki Date: Wed, 29 Jan 2025 14:36:14 -0800 Subject: [PATCH 20/30] [test-infra][1/n] Replace Move's `.exp` tests with cargo-insta (#21003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description - This PR adds a wrapper around `insta::assert_snapshot` to ease migration from `.exp` files - Utilize this for `move-docgen-testsuite` (which was already using snapshot) - Moves snapshot files next to the `toml` files ## Test plan - 👀 --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- Cargo.lock | 11 +- external-crates/move/Cargo.lock | 23 +- .../move-command-line-common/Cargo.toml | 1 + .../move-command-line-common/src/testing.rs | 129 ++++++ .../a__m.md@collapsed_sections.snap} | 10 +- .../annotation/a__m.md@default.snap} | 10 +- .../a__m.md@collapsed_sections.snap} | 10 +- .../code_block/a__m.md@default.snap} | 10 +- .../a__m.md@collapsed_sections.snap} | 10 +- .../const_string/a__m.md@default.snap} | 10 +- .../a__m.md@collapsed_sections.snap} | 10 +- .../a__m.md@default.snap} | 10 +- .../enums/a__m.md@collapsed_sections.snap} | 10 +- .../enums/a__m.md@default.snap} | 10 +- .../root.md@collapsed_sections.snap} | 10 +- .../root_template/root.md@default.snap} | 10 +- ..._annotation__collapsed_sections__m.md.snap | 105 ----- ...ests__move__annotation__default__m.md.snap | 113 ----- ..._code_block__collapsed_sections__m.md.snap | 43 -- ...ests__move__code_block__default__m.md.snap | 48 --- ...onst_string__collapsed_sections__m.md.snap | 59 --- ...ts__move__const_string__default__m.md.snap | 59 --- ...isibilities__collapsed_sections__m.md.snap | 76 ---- ...different_visibilities__default__m.md.snap | 87 ---- ...move__enums__collapsed_sections__m.md.snap | 357 ---------------- ...te__tests__move__enums__default__m.md.snap | 392 ------------------ .../move-docgen-tests/tests/testsuite.rs | 75 ++-- 27 files changed, 308 insertions(+), 1390 deletions(-) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__annotation__collapsed_sections__a__m.md.snap => move/annotation/a__m.md@collapsed_sections.snap} (87%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__annotation__default__a__m.md.snap => move/annotation/a__m.md@default.snap} (87%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__code_block__collapsed_sections__a__m.md.snap => move/code_block/a__m.md@collapsed_sections.snap} (70%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__code_block__default__a__m.md.snap => move/code_block/a__m.md@default.snap} (71%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__const_string__default__a__m.md.snap => move/const_string/a__m.md@collapsed_sections.snap} (81%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__const_string__collapsed_sections__a__m.md.snap => move/const_string/a__m.md@default.snap} (81%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__a__m.md.snap => move/different_visibilities/a__m.md@collapsed_sections.snap} (85%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__different_visibilities__default__a__m.md.snap => move/different_visibilities/a__m.md@default.snap} (86%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__enums__collapsed_sections__a__m.md.snap => move/enums/a__m.md@collapsed_sections.snap} (94%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__enums__default__a__m.md.snap => move/enums/a__m.md@default.snap} (95%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__root_template__collapsed_sections__root.md.snap => move/root_template/root.md@collapsed_sections.snap} (89%) rename external-crates/move/crates/move-docgen-tests/tests/{snapshots/testsuite__tests__move__root_template__default__root.md.snap => move/root_template/root.md@default.snap} (89%) delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__m.md.snap delete mode 100644 external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__m.md.snap diff --git a/Cargo.lock b/Cargo.lock index d9f02b3fcfbda..e5d00e35adce9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6344,13 +6344,13 @@ dependencies = [ [[package]] name = "insta" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8" +checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5" dependencies = [ "console", - "lazy_static", "linked-hash-map", + "once_cell", "pest", "pest_derive", "serde", @@ -7714,6 +7714,7 @@ dependencies = [ "difference", "dirs-next", "hex", + "insta", "move-binary-format", "move-core-types", "once_cell", @@ -9010,9 +9011,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" diff --git a/external-crates/move/Cargo.lock b/external-crates/move/Cargo.lock index 134b9750058a6..9ce38f4ee41c5 100644 --- a/external-crates/move/Cargo.lock +++ b/external-crates/move/Cargo.lock @@ -1019,6 +1019,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-discriminant" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f74784b51403f16c5fa6dc667488389e629811329c1c6719c25874da2ba4f" +dependencies = [ + "typeid", +] + [[package]] name = "errno" version = "0.3.9" @@ -1332,6 +1341,7 @@ dependencies = [ "console", "linked-hash-map", "once_cell", + "serde", "similar", ] @@ -1838,6 +1848,7 @@ dependencies = [ "difference", "dirs-next", "hex", + "insta", "move-binary-format", "move-core-types", "once_cell", @@ -3360,13 +3371,15 @@ dependencies = [ [[package]] name = "serde-reflection" -version = "0.3.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05a5f801ac62a51a49d378fdb3884480041b99aced450b28990673e8ff99895" +checksum = "e5bef77b40d103fda6c10d29c21f5c78c980e8570e1a290a648a9ff5011f96e1" dependencies = [ + "erased-discriminant", "once_cell", "serde", "thiserror", + "typeid", ] [[package]] @@ -4007,6 +4020,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "typeid" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e13db2e0ccd5e14a544e8a246ba2312cd25223f616442d7f2cb0e3db614236e" + [[package]] name = "typenum" version = "1.17.0" diff --git a/external-crates/move/crates/move-command-line-common/Cargo.toml b/external-crates/move/crates/move-command-line-common/Cargo.toml index 83aa969a204af..c17caa3b09f83 100644 --- a/external-crates/move/crates/move-command-line-common/Cargo.toml +++ b/external-crates/move/crates/move-command-line-common/Cargo.toml @@ -20,6 +20,7 @@ serde.workspace = true dirs-next.workspace = true vfs.workspace = true bcs.workspace = true +insta = { workspace = true, features = ["serde"] } move-core-types.workspace = true move-binary-format.workspace = true diff --git a/external-crates/move/crates/move-command-line-common/src/testing.rs b/external-crates/move/crates/move-command-line-common/src/testing.rs index db0f98c982294..620dd78c2260c 100644 --- a/external-crates/move/crates/move-command-line-common/src/testing.rs +++ b/external-crates/move/crates/move-command-line-common/src/testing.rs @@ -63,3 +63,132 @@ pub fn format_diff(expected: impl AsRef, actual: impl AsRef) -> String } ret } + +/// See `insta_assert!` for documentation. +pub struct InstaOptions> { + pub info: Option, + pub suffix: Option, +} + +impl> InstaOptions { + /// See `insta_assert!` for documentation. + pub fn new() -> Self { + Self { + info: None, + suffix: None, + } + } +} + +impl InstaOptions<(), String> { + /// See `insta_assert!` for documentation. + pub fn none() -> Self { + Self { + info: None, + suffix: None, + } + } +} + +#[macro_export] +/// A wrapper around `insta::assert_snapshort` to promote uniformity in the Move codebase, intended +/// to be used with datatest-stable and as a replacement for the hand-rolled baseline tests. +/// The snapshot file will be saved in the same directory as the input file with the name specified. +/// In essence, it will be saved at the path `{input_path}/{name}.snap` (and +/// `{input_path}/{name}@{suffix}.snap` if `suffix` is specified). +/// +/// For ease of use and reviewing, `insta_assert` should be used at most once per test. When it +/// fails, it will stop the test. So if there are multiple snapshots in a given test, it would +/// require multiple test runs to review all the failures. +/// If you do need multiple snapshots in a test, you may want to disable assertions for your test +/// run by setting `INSTA_FORCE_PASS=1` see +/// https://insta.rs/docs/advanced/#disabling-assertion-failure for more information. +/// +/// # Arguments +/// The macro has three required arguments: +/// +/// - `name`: The name of the test. This will be used to name the snapshot file. For datatest this +/// should likely be the file name. +/// - `input_path`: The path to the input file. This is used to determine the snapshot path. +/// - `contents`: The contents to snapshot. +/// +/// +/// The macro also accepts an optional arguments to that are used with `InstaOptions` to customize +/// the snapshot. If needed the `InstaOptions` struct can be used directly by specifying the +/// `options` argument. Options include: +/// +/// - `info`: Additional information to include in the header of the snapshot file. This can be +/// useful for debugging tests. The value can be any type that implements +/// `serde::Serialize`. +/// - `suffix`: A suffix to append to the snapshot file name. This changes the snapshot path to +/// `{input_path}/{name}@{suffix}.snap`. +/// +/// # Updating snapshots +/// +/// After running the test, the `.snap` files can be updated in two ways: +/// +/// 1. By using `cargo insta review`, which will open an interactive UI to review the changes. +/// 2. Running the tests with the environment variable `INSTA_UPDATE=alawys` +/// +/// See https://docs.rs/insta/latest/insta/#updating-snapshots for more information. +macro_rules! insta_assert { + { + name: $name:expr, + input_path: $input:expr, + contents: $contents:expr, + options: $options:expr + $(,)? + } => {{ + let name: String = $name.into(); + let i: &std::path::Path = $input.as_ref(); + let i = i.canonicalize().unwrap(); + let c = $contents; + let $crate::testing::InstaOptions { info, suffix } = $options; + let mut settings = insta::Settings::clone_current(); + settings.set_input_file(&i); + settings.set_snapshot_path(i.parent().unwrap()); + if let Some(info) = info { + settings.set_info(info); + } + if let Some(suffix) = suffix { + settings.set_snapshot_suffix(suffix); + } + settings.set_prepend_module_to_snapshot(false); + settings.set_omit_expression(true); + settings.bind(|| { + insta::assert_snapshot!(name, c); + }); + }}; + { + name: $name:expr, + input_path: $input:expr, + contents: $contents:expr + $(,)? + } => {{ + insta_assert! { + name: $name, + input_path: $input, + output_path: $output, + contents: $contents, + options: $crate::testing::InstaOptions::none(), + } + }}; + { + name: $name:expr, + input_path: $input:expr, + contents: $contents:expr, + $($k:ident: $v:expr),+$(,)? + } => {{ + let mut opts = $crate::testing::InstaOptions::new(); + $( + opts.$k = Some($v); + )+ + insta_assert! { + name: $name, + input_path: $input, + contents: $contents, + options: opts + } + }}; +} +pub use insta_assert; diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@collapsed_sections.snap similarity index 87% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@collapsed_sections.snap index 1d4b17a4e618a..32d4600341323 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/annotation/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@default.snap similarity index 87% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@default.snap index 1371d53d83e10..cc800cf3001f6 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/annotation/a__m.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/annotation/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@collapsed_sections.snap similarity index 70% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@collapsed_sections.snap index a44fe725b3e9a..72319518783dd 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/code_block/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@default.snap similarity index 71% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@default.snap index e16c3379b575a..8cb2c4d342921 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/code_block/a__m.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/code_block/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@collapsed_sections.snap similarity index 81% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@collapsed_sections.snap index 12711220197a2..09d9d8a968a07 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/const_string/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@default.snap similarity index 81% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@default.snap index 12711220197a2..625aba8e2c078 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/const_string/a__m.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/const_string/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@collapsed_sections.snap similarity index 85% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@collapsed_sections.snap index a1e2a4ea55c9a..996d78f69ac61 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/different_visibilities/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@default.snap similarity index 86% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@default.snap index 471f435af65da..96b87dbc774bf 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/different_visibilities/a__m.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/different_visibilities/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@collapsed_sections.snap similarity index 94% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@collapsed_sections.snap index f089d0c204600..a39e65e413143 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/enums/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__a__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@default.snap similarity index 95% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__a__m.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@default.snap index 608314e24b0c9..05c8f5be89464 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__a__m.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/enums/a__m.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/enums/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__collapsed_sections__root.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@collapsed_sections.snap similarity index 89% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__collapsed_sections__root.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@collapsed_sections.snap index 7775c49c51e09..bc230225f5dfb 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__collapsed_sections__root.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@collapsed_sections.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: true + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/root_template/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__default__root.md.snap b/external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@default.snap similarity index 89% rename from external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__default__root.md.snap rename to external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@default.snap index 2d90b75e0511a..38e58e43df1c6 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__root_template__default__root.md.snap +++ b/external-crates/move/crates/move-docgen-tests/tests/move/root_template/root.md@default.snap @@ -1,6 +1,14 @@ --- source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents +info: + section_level_start: 1 + exclude_private_fun: false + exclude_impl: false + toc_depth: 3 + no_collapsed_sections: false + include_dep_diagrams: false + include_call_diagrams: false +input_file: crates/move-docgen-tests/tests/move/root_template/Move.toml --- diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__m.md.snap deleted file mode 100644 index b0bf3e00da4ea..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__collapsed_sections__m.md.snap +++ /dev/null @@ -1,105 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - -This is a doc comment above an annotation. - - -- [Constants](#@Constants_0) -- [Function `test`](#a_m_test) -- [Function `test1`](#a_m_test1) - - -
- - - - - -## Constants - - - - -This is the top doc comment -This is the middle doc comment - - -
const Cool: u32 = 0;
-
- - - - - -This is a doc comment on a constant with an annotation. Below the annotation. - - -
const Error: u32 = 0;
-
- - - - - -This is a doc comment on a constant with an annotation. Above the annotation. - - -
const OtherError: u32 = 0;
-
- - - - - -This is the top doc comment -This is the middle doc comment -This is the bottom doc comment - - -
const Woah: u32 = 0;
-
- - - - - -## Function `test` - -This is a doc comment above a function with an annotation. Above the annotation. - - -
fun test()
-
- - - -##### Implementation - - -
fun test() { }
-
- - - - - -## Function `test1` - -This is a doc comment above a function with an annotation. Below the annotation. - - -
fun test1()
-
- - - -##### Implementation - - -
fun test1() { }
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__m.md.snap deleted file mode 100644 index 32acba75f1dde..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__annotation__default__m.md.snap +++ /dev/null @@ -1,113 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - -This is a doc comment above an annotation. - - -- [Constants](#@Constants_0) -- [Function `test`](#a_m_test) -- [Function `test1`](#a_m_test1) - - -
- - - - - -## Constants - - - - -This is the top doc comment -This is the middle doc comment - - -
const Cool: u32 = 0;
-
- - - - - -This is a doc comment on a constant with an annotation. Below the annotation. - - -
const Error: u32 = 0;
-
- - - - - -This is a doc comment on a constant with an annotation. Above the annotation. - - -
const OtherError: u32 = 0;
-
- - - - - -This is the top doc comment -This is the middle doc comment -This is the bottom doc comment - - -
const Woah: u32 = 0;
-
- - - - - -## Function `test` - -This is a doc comment above a function with an annotation. Above the annotation. - - -
fun test()
-
- - - -
-Implementation - - -
fun test() { }
-
- - - -
- - - -## Function `test1` - -This is a doc comment above a function with an annotation. Below the annotation. - - -
fun test1()
-
- - - -
-Implementation - - -
fun test1() { }
-
- - - -
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__m.md.snap deleted file mode 100644 index bdfa7055c609e..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__collapsed_sections__m.md.snap +++ /dev/null @@ -1,43 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - - - -- [Function `main`](#a_m_main) - - [Explanation of the algorithm](#@Explanation_of_the_algorithm_0) - - -
- - - - - -## Function `main` - - - - -### Explanation of the algorithm - -``` -code block -``` -then inline code - - -
entry fun main()
-
- - - -##### Implementation - - -
entry fun main() { }
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__m.md.snap deleted file mode 100644 index db1caf798ea5d..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__code_block__default__m.md.snap +++ /dev/null @@ -1,48 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - - - -- [Function `main`](#a_m_main) - - [Explanation of the algorithm](#@Explanation_of_the_algorithm_0) - - -
- - - - - -## Function `main` - - - - -### Explanation of the algorithm - -``` -code block -``` -then inline code - - -
entry fun main()
-
- - - -
-Implementation - - -
entry fun main() { }
-
- - - -
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__m.md.snap deleted file mode 100644 index 8b4a2e6dacedb..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__collapsed_sections__m.md.snap +++ /dev/null @@ -1,59 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - - - -- [Constants](#@Constants_0) - - -
- - - - - -## Constants - - - - -This is a doc comment above an error constant that should be rendered as a string - - -
#[error]
-const AString: vector<u8> = b"Hello, world  🦀   ";
-
- - - - - - - -
const AStringNotError: vector<u8> = vector[72, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100, 32, 32, 240, 159, 166, 128, 32, 32, 32];
-
- - - - - -This is a doc comment above an error constant that should not be rendered as a string - - -
#[error]
-const ErrorNotString: u64 = 10;
-
- - - - - - - -
const NotAString: vector<u8> = vector[1, 2, 3];
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__m.md.snap deleted file mode 100644 index 8b4a2e6dacedb..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__const_string__default__m.md.snap +++ /dev/null @@ -1,59 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - - - -- [Constants](#@Constants_0) - - -
- - - - - -## Constants - - - - -This is a doc comment above an error constant that should be rendered as a string - - -
#[error]
-const AString: vector<u8> = b"Hello, world  🦀   ";
-
- - - - - - - -
const AStringNotError: vector<u8> = vector[72, 101, 108, 108, 111, 44, 32, 119, 111, 114, 108, 100, 32, 32, 240, 159, 166, 128, 32, 32, 32];
-
- - - - - -This is a doc comment above an error constant that should not be rendered as a string - - -
#[error]
-const ErrorNotString: u64 = 10;
-
- - - - - - - -
const NotAString: vector<u8> = vector[1, 2, 3];
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__m.md.snap deleted file mode 100644 index 7228666eea890..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__collapsed_sections__m.md.snap +++ /dev/null @@ -1,76 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::TestViz` - - - -- [Function `this_is_a_public_fun`](#a_TestViz_this_is_a_public_fun) -- [Function `this_is_a_public_script_fun`](#a_TestViz_this_is_a_public_script_fun) -- [Function `this_is_a_private_fun`](#a_TestViz_this_is_a_private_fun) - - -
- - - - - -## Function `this_is_a_public_fun` - -This is a public function - - -
publicfun this_is_a_public_fun()
-
- - - -##### Implementation - - -
public fun this_is_a_public_fun() { }
-
- - - - - -## Function `this_is_a_public_script_fun` - -This is a public entry function - - -
publicentry fun this_is_a_public_script_fun()
-
- - - -##### Implementation - - -
public entry fun this_is_a_public_script_fun() {}
-
- - - - - -## Function `this_is_a_private_fun` - -This is a private function - - -
fun this_is_a_private_fun()
-
- - - -##### Implementation - - -
fun this_is_a_private_fun() {}
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__m.md.snap deleted file mode 100644 index 8470e7fd7d122..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__different_visibilities__default__m.md.snap +++ /dev/null @@ -1,87 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::TestViz` - - - -- [Function `this_is_a_public_fun`](#a_TestViz_this_is_a_public_fun) -- [Function `this_is_a_public_script_fun`](#a_TestViz_this_is_a_public_script_fun) -- [Function `this_is_a_private_fun`](#a_TestViz_this_is_a_private_fun) - - -
- - - - - -## Function `this_is_a_public_fun` - -This is a public function - - -
publicfun this_is_a_public_fun()
-
- - - -
-Implementation - - -
public fun this_is_a_public_fun() { }
-
- - - -
- - - -## Function `this_is_a_public_script_fun` - -This is a public entry function - - -
publicentry fun this_is_a_public_script_fun()
-
- - - -
-Implementation - - -
public entry fun this_is_a_public_script_fun() {}
-
- - - -
- - - -## Function `this_is_a_private_fun` - -This is a private function - - -
fun this_is_a_private_fun()
-
- - - -
-Implementation - - -
fun this_is_a_private_fun() {}
-
- - - -
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__m.md.snap deleted file mode 100644 index 299d3efadf692..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__collapsed_sections__m.md.snap +++ /dev/null @@ -1,357 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - -This is a doc comment above an annotation. - - -- [Struct `X`](#a_m_X) -- [Struct `Y`](#a_m_Y) -- [Struct `XG`](#a_m_XG) -- [Struct `YG`](#a_m_YG) -- [Struct `XGG`](#a_m_XGG) -- [Struct `YGG`](#a_m_YGG) -- [Struct `VecMap`](#a_m_VecMap) -- [Struct `Entry`](#a_m_Entry) -- [Enum `Enum`](#a_m_Enum) -- [Enum `GenericEnum`](#a_m_GenericEnum) -- [Function `f`](#a_m_f) - - -
- - - - - -## Struct `X` - - - -
public struct X has drop
-
- - - -##### Fields - - -
-
-x: a::m::Enum -
-
-
-
- - - - -## Struct `Y` - - - -
public struct Y
-
- - - -##### Fields - - -
-
-0: a::m::Enum -
-
-
-
- - - - -## Struct `XG` - - - -
public struct XG
-
- - - -##### Fields - - -
-
-x: a::m::GenericEnum<a::m::Enum> -
-
-
-
- - - - -## Struct `YG` - - - -
public struct YG
-
- - - -##### Fields - - -
-
-0: a::m::GenericEnum<a::m::Enum> -
-
-
-
- - - - -## Struct `XGG` - - - -
public struct XGGT
-
- - - -##### Fields - - -
-
-x: a::m::GenericEnum<T> -
-
-
-
- - - - -## Struct `YGG` - - - -
public struct YGGT
-
- - - -##### Fields - - -
-
-0: a::m::GenericEnum<T> -
-
-
-
- - - - -## Struct `VecMap` - - - -
public struct VecMapK, V has copy, drop, store
-
- - - -##### Fields - - -
-
-contents: vector<a::m::Entry<K, V>> -
-
-
-
- - - - -## Struct `Entry` - -An entry in the map - - -
public struct EntryK, V has copy, drop, store
-
- - - -##### Fields - - -
-
-key: K -
-
-
-
-value: V -
-
-
-
- - - - -## Enum `Enum` - -This is a doc comment above an enum - - -
public enum Enum has drop
-
- - - -##### Variants - - -
-
-Variant A -
-
- This is a doc comment above a variant -
-
-Variant B -
-
-
-
-Variant C -
-
-
- -
-
-0: u64 -
-
-
-
- -
-Variant D -
-
- Another doc comment -
- -
-
-x: u64 -
-
- Doc text on variant field -
-
- -
-Variant E -
-
-
- -
-
-x: u64 -
-
-
-
- - -
-
-y: u64 -
-
-
-
- -
- - - - -## Enum `GenericEnum` - - - -
public enum GenericEnum<T>
-
- - - -##### Variants - - -
-
-Variant A -
-
-
- -
-
-0: T -
-
-
-
- -
-Variant B -
-
-
-
- - - - -## Function `f` - -Doc comments type_: VecMap<u64, X> - - -
fun f(x: a::m::VecMap<u64, a::m::X>): u64
-
- - - -##### Implementation - - -
fun f(x: VecMap<u64, X>): u64 {
-    0
-}
-
diff --git a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__m.md.snap b/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__m.md.snap deleted file mode 100644 index 78207e1636578..0000000000000 --- a/external-crates/move/crates/move-docgen-tests/tests/snapshots/testsuite__tests__move__enums__default__m.md.snap +++ /dev/null @@ -1,392 +0,0 @@ ---- -source: crates/move-docgen-tests/tests/testsuite.rs -expression: contents ---- - - -# Module `a::m` - -This is a doc comment above an annotation. - - -- [Struct `X`](#a_m_X) -- [Struct `Y`](#a_m_Y) -- [Struct `XG`](#a_m_XG) -- [Struct `YG`](#a_m_YG) -- [Struct `XGG`](#a_m_XGG) -- [Struct `YGG`](#a_m_YGG) -- [Struct `VecMap`](#a_m_VecMap) -- [Struct `Entry`](#a_m_Entry) -- [Enum `Enum`](#a_m_Enum) -- [Enum `GenericEnum`](#a_m_GenericEnum) -- [Function `f`](#a_m_f) - - -
- - - - - -## Struct `X` - - - -
public struct X has drop
-
- - - -
-Fields - - -
-
-x: a::m::Enum -
-
-
-
- - -
- - - -## Struct `Y` - - - -
public struct Y
-
- - - -
-Fields - - -
-
-0: a::m::Enum -
-
-
-
- - -
- - - -## Struct `XG` - - - -
public struct XG
-
- - - -
-Fields - - -
-
-x: a::m::GenericEnum<a::m::Enum> -
-
-
-
- - -
- - - -## Struct `YG` - - - -
public struct YG
-
- - - -
-Fields - - -
-
-0: a::m::GenericEnum<a::m::Enum> -
-
-
-
- - -
- - - -## Struct `XGG` - - - -
public struct XGGT
-
- - - -
-Fields - - -
-
-x: a::m::GenericEnum<T> -
-
-
-
- - -
- - - -## Struct `YGG` - - - -
public struct YGGT
-
- - - -
-Fields - - -
-
-0: a::m::GenericEnum<T> -
-
-
-
- - -
- - - -## Struct `VecMap` - - - -
public struct VecMapK, V has copy, drop, store
-
- - - -
-Fields - - -
-
-contents: vector<a::m::Entry<K, V>> -
-
-
-
- - -
- - - -## Struct `Entry` - -An entry in the map - - -
public struct EntryK, V has copy, drop, store
-
- - - -
-Fields - - -
-
-key: K -
-
-
-
-value: V -
-
-
-
- - -
- - - -## Enum `Enum` - -This is a doc comment above an enum - - -
public enum Enum has drop
-
- - - -
-Variants - - -
-
-Variant A -
-
- This is a doc comment above a variant -
-
-Variant B -
-
-
-
-Variant C -
-
-
- -
-
-0: u64 -
-
-
-
- -
-Variant D -
-
- Another doc comment -
- -
-
-x: u64 -
-
- Doc text on variant field -
-
- -
-Variant E -
-
-
- -
-
-x: u64 -
-
-
-
- - -
-
-y: u64 -
-
-
-
- -
- - -
- - - -## Enum `GenericEnum` - - - -
public enum GenericEnum<T>
-
- - - -
-Variants - - -
-
-Variant A -
-
-
- -
-
-0: T -
-
-
-
- -
-Variant B -
-
-
-
- - -
- - - -## Function `f` - -Doc comments type_: VecMap<u64, X> - - -
fun f(x: a::m::VecMap<u64, a::m::X>): u64
-
- - - -
-Implementation - - -
fun f(x: VecMap<u64, X>): u64 {
-    0
-}
-
- - - -
diff --git a/external-crates/move/crates/move-docgen-tests/tests/testsuite.rs b/external-crates/move/crates/move-docgen-tests/tests/testsuite.rs index 27c27f5194029..bc53060983757 100644 --- a/external-crates/move/crates/move-docgen-tests/tests/testsuite.rs +++ b/external-crates/move/crates/move-docgen-tests/tests/testsuite.rs @@ -1,8 +1,8 @@ // Copyright (c) The Move Contributors // SPDX-License-Identifier: Apache-2.0 -use move_docgen::{Docgen, DocgenOptions}; -use move_model_2::source_model; +use move_command_line_common::testing::insta_assert; +use move_docgen::{Docgen, DocgenFlags, DocgenOptions}; use move_package::compilation::model_builder; use move_package::BuildConfig; use std::path::Path; @@ -11,7 +11,7 @@ use tempfile::TempDir; const ROOT_DOC_TEMPLATE_NAME: &str = "root_template.md"; -fn options(root_doc_template: Option<&Path>) -> DocgenOptions { +fn options(root_doc_template: Option<&Path>, flags: DocgenFlags) -> DocgenOptions { DocgenOptions { output_directory: "output".to_string(), root_doc_templates: root_doc_template @@ -19,11 +19,26 @@ fn options(root_doc_template: Option<&Path>) -> DocgenOptions { .map(|p| p.to_string_lossy().to_string()) .collect(), compile_relative_to_output_dir: true, + flags, ..DocgenOptions::default() } } -fn test_move(toml_path: &Path) -> datatest_stable::Result<()> { +fn test_default(toml_path: &Path) -> datatest_stable::Result<()> { + let flags = DocgenFlags::default(); + assert!(!flags.exclude_impl); + assert!(!flags.no_collapsed_sections); + test_impl(toml_path, flags, "default") +} + +fn test_collapsed_sections(toml_path: &Path) -> datatest_stable::Result<()> { + let mut flags = DocgenFlags::default(); + assert!(!flags.exclude_impl); + flags.no_collapsed_sections = true; + test_impl(toml_path, flags, "collapsed_sections") +} + +fn test_impl(toml_path: &Path, flags: DocgenFlags, test_case: &str) -> datatest_stable::Result<()> { let test_dir = toml_path.parent().unwrap(); let output_dir = TempDir::new()?; let config = BuildConfig { @@ -42,34 +57,30 @@ fn test_move(toml_path: &Path) -> datatest_stable::Result<()> { } else { None }; - let mut options = options(root_doc_template); - - assert!(!options.flags.exclude_impl); - assert!(!options.flags.exclude_impl); - test_move_one(&test_dir.join("default"), &model, &options)?; - - assert!(!options.flags.no_collapsed_sections); - options.flags.no_collapsed_sections = true; - test_move_one(&test_dir.join("collapsed_sections"), &model, &options)?; - - Ok(()) -} - -fn test_move_one( - out_dir: &Path, - model: &source_model::Model, - doc_options: &DocgenOptions, -) -> anyhow::Result<()> { - let docgen = Docgen::new(model, doc_options); - let file_contents = docgen.gen(model)?; - for (path, contents) in file_contents { - if path.contains("dependencies") { - continue; - } - let out_path = out_dir.join(&path).to_string_lossy().to_string(); - insta::assert_snapshot!(out_path, contents); - } + let options = options(root_doc_template, flags); + let docgen = Docgen::new(&model, &options); + let file_contents = docgen.gen(&model)?; + let [(path, contents)] = file_contents + .iter() + .filter(|(path, _contents)| !path.contains("dependencies")) + .collect::>() + .try_into() + .expect("Test infra supports only one output file currently"); + insta_assert! { + name: path, + input_path: toml_path, + contents: contents, + info: &options.flags, + suffix: test_case, + }; Ok(()) } -datatest_stable::harness!(test_move, "tests/move/", r".*\.toml",); +datatest_stable::harness!( + test_default, + "tests/move/", + r".*\.toml", + test_collapsed_sections, + "tests/move/", + r".*\.toml" +); From 8a72297feda65a20cdff2c6aac2393f8ddbb76a2 Mon Sep 17 00:00:00 2001 From: Mark Logan <103447440+mystenmark@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:02:47 -0800 Subject: [PATCH 21/30] Re-execution of pending certs must happen concurrently with consensus handling, since there may be dependencies in either direction. (#21000) --- crates/sui-core/src/authority.rs | 4 +- .../checkpoints/checkpoint_executor/mod.rs | 19 ++-- crates/sui-node/src/lib.rs | 107 +++++++++++------- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index dafddeffcc750..e9f18ae21b367 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -1255,7 +1255,9 @@ impl AuthorityState { ) .await .tap_err(|e| info!("process_certificate failed: {e}")) - .tap_ok(|_| debug!("process_certificate succeeded")) + .tap_ok( + |(fx, _)| debug!(?tx_digest, fx_digest=?fx.digest(), "process_certificate succeeded"), + ) } pub fn read_objects_for_execution( diff --git a/crates/sui-core/src/checkpoints/checkpoint_executor/mod.rs b/crates/sui-core/src/checkpoints/checkpoint_executor/mod.rs index 6030006dc4fc3..c6204b6299003 100644 --- a/crates/sui-core/src/checkpoints/checkpoint_executor/mod.rs +++ b/crates/sui-core/src/checkpoints/checkpoint_executor/mod.rs @@ -436,7 +436,7 @@ impl CheckpointExecutor { /// Post processing and plumbing after we executed a checkpoint. This function is guaranteed /// to be called in the order of checkpoint sequence number. - #[instrument(level = "debug", skip_all)] + #[instrument(level = "info", skip_all, fields(seq = ?checkpoint.sequence_number()))] async fn process_executed_checkpoint( &self, epoch_store: &AuthorityPerEpochStore, @@ -447,7 +447,7 @@ impl CheckpointExecutor { ) { // Commit all transaction effects to disk let cache_commit = self.state.get_cache_commit(); - debug!(seq = ?checkpoint.sequence_number, "committing checkpoint transactions to disk"); + debug!("committing checkpoint transactions to disk"); cache_commit .commit_transaction_outputs( epoch_store.epoch(), @@ -1040,8 +1040,8 @@ fn extract_end_of_epoch_tx( let change_epoch_tx = VerifiedExecutableTransaction::new_from_checkpoint( (*change_epoch_tx.unwrap_or_else(|| panic!( - "state-sync should have ensured that transaction with digest {:?} exists for checkpoint: {checkpoint:?}", - digests.transaction, + "state-sync should have ensured that transaction with digests {:?} exists for checkpoint: {checkpoint:?}", + digests ) )).clone(), epoch_store.epoch(), @@ -1100,16 +1100,15 @@ fn get_unexecuted_transactions( // Remove the change epoch transaction so that we can special case its execution. checkpoint.end_of_epoch_data.as_ref().tap_some(|_| { - let change_epoch_tx_digest = execution_digests + let digests = execution_digests .pop() - .expect("Final checkpoint must have at least one transaction") - .transaction; + .expect("Final checkpoint must have at least one transaction"); let change_epoch_tx = cache_reader - .get_transaction_block(&change_epoch_tx_digest) + .get_transaction_block(&digests.transaction) .unwrap_or_else(|| panic!( - "state-sync should have ensured that transaction with digest {change_epoch_tx_digest:?} exists for checkpoint: {}", + "state-sync should have ensured that transaction with digests {digests:?} exists for checkpoint: {}", checkpoint.sequence_number() ) ); @@ -1138,7 +1137,7 @@ fn get_unexecuted_transactions( let maybe_randomness_tx = cache_reader.get_transaction_block(&first_digest.transaction) .unwrap_or_else(|| panic!( - "state-sync should have ensured that transaction with digest {first_digest:?} exists for checkpoint: {}", + "state-sync should have ensured that transaction with digests {first_digest:?} exists for checkpoint: {}", checkpoint.sequence_number() ) ); diff --git a/crates/sui-node/src/lib.rs b/crates/sui-node/src/lib.rs index 59e14a435f073..df5812084bd97 100644 --- a/crates/sui-node/src/lib.rs +++ b/crates/sui-node/src/lib.rs @@ -12,12 +12,14 @@ use anyhow::Result; use arc_swap::ArcSwap; use fastcrypto_zkp::bn254::zk_login::JwkId; use fastcrypto_zkp::bn254::zk_login::OIDCProvider; +use futures::future::BoxFuture; use futures::TryFutureExt; use mysten_common::debug_fatal; use mysten_network::server::SUI_TLS_SERVER_NAME; use prometheus::Registry; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt; +use std::future::Future; use std::path::PathBuf; use std::str::FromStr; #[cfg(msim)] @@ -151,7 +153,7 @@ mod handle; pub mod metrics; pub struct ValidatorComponents { - validator_server_handle: JoinHandle>, + validator_server_handle: SpawnOnce, validator_overload_monitor_handle: Option>, consensus_manager: ConsensusManager, consensus_store_pruner: ConsensusStorePruner, @@ -836,26 +838,30 @@ impl SuiNode { let sui_node_metrics = Arc::new(SuiNodeMetrics::new(®istry_service.default_registry())); let validator_components = if state.is_validator(&epoch_store) { - Self::reexecute_pending_consensus_certs(&epoch_store, &state).await; + let (components, _) = futures::join!( + Self::construct_validator_components( + config.clone(), + state.clone(), + committee, + epoch_store.clone(), + checkpoint_store.clone(), + state_sync_handle.clone(), + randomness_handle.clone(), + Arc::downgrade(&accumulator), + backpressure_manager.clone(), + connection_monitor_status.clone(), + ®istry_service, + sui_node_metrics.clone(), + ), + Self::reexecute_pending_consensus_certs(&epoch_store, &state,) + ); + let mut components = components?; - let components = Self::construct_validator_components( - config.clone(), - state.clone(), - committee, - epoch_store.clone(), - checkpoint_store.clone(), - state_sync_handle.clone(), - randomness_handle.clone(), - Arc::downgrade(&accumulator), - backpressure_manager.clone(), - connection_monitor_status.clone(), - ®istry_service, - sui_node_metrics.clone(), - ) - .await?; - // This is only needed during cold start. components.consensus_adapter.submit_recovered(&epoch_store); + // Start the gRPC server + components.validator_server_handle = components.validator_server_handle.start(); + Some(components) } else { None @@ -1325,7 +1331,7 @@ impl SuiNode { consensus_store_pruner: ConsensusStorePruner, accumulator: Weak, backpressure_manager: Arc, - validator_server_handle: JoinHandle>, + validator_server_handle: SpawnOnce, validator_overload_monitor_handle: Option>, checkpoint_metrics: Arc, sui_node_metrics: Arc, @@ -1505,7 +1511,7 @@ impl SuiNode { state: Arc, consensus_adapter: Arc, prometheus_registry: &Registry, - ) -> Result>> { + ) -> Result { let validator_service = ValidatorService::new( state.clone(), consensus_adapter, @@ -1533,9 +1539,8 @@ impl SuiNode { .map_err(|err| anyhow!(err.to_string()))?; let local_addr = server.local_addr(); info!("Listening to traffic on {local_addr}"); - let grpc_server = spawn_monitored_task!(server.serve().map_err(Into::into)); - Ok(grpc_server) + Ok(SpawnOnce::new(server.serve().map_err(Into::into))) } async fn reexecute_pending_consensus_certs( @@ -1898,23 +1903,25 @@ impl SuiNode { if self.state.is_validator(&new_epoch_store) { info!("Promoting the node from fullnode to validator, starting grpc server"); - Some( - Self::construct_validator_components( - self.config.clone(), - self.state.clone(), - Arc::new(next_epoch_committee.clone()), - new_epoch_store.clone(), - self.checkpoint_store.clone(), - self.state_sync_handle.clone(), - self.randomness_handle.clone(), - weak_accumulator, - self.backpressure_manager.clone(), - self.connection_monitor_status.clone(), - &self.registry_service, - self.metrics.clone(), - ) - .await?, + let mut components = Self::construct_validator_components( + self.config.clone(), + self.state.clone(), + Arc::new(next_epoch_committee.clone()), + new_epoch_store.clone(), + self.checkpoint_store.clone(), + self.state_sync_handle.clone(), + self.randomness_handle.clone(), + weak_accumulator, + self.backpressure_manager.clone(), + self.connection_monitor_status.clone(), + &self.registry_service, + self.metrics.clone(), ) + .await?; + + components.validator_server_handle = components.validator_server_handle.start(); + + Some(components) } else { None } @@ -2042,6 +2049,30 @@ impl SuiNode { } } +enum SpawnOnce { + // Mutex is only needed to make SpawnOnce Send + Unstarted(Mutex>>), + #[allow(unused)] + Started(JoinHandle>), +} + +impl SpawnOnce { + pub fn new(future: impl Future> + Send + 'static) -> Self { + Self::Unstarted(Mutex::new(Box::pin(future))) + } + + pub fn start(self) -> Self { + match self { + Self::Unstarted(future) => { + let future = future.into_inner(); + let handle = tokio::spawn(future); + Self::Started(handle) + } + Self::Started(_) => self, + } + } +} + /// Notify state-sync that a new list of trusted peers are now available. fn send_trusted_peer_change( config: &NodeConfig, From 8ba733ea284168a1dd2c26478731b6059e5a3f82 Mon Sep 17 00:00:00 2001 From: Jort Date: Wed, 29 Jan 2025 18:06:48 -0800 Subject: [PATCH 22/30] [tests] create dry run helper for sui transactional test runner framework (#20920) ## Description motivation: #20109 Mirroring the existing `--dev-inspect` flag, this change adds `--dry-run` for sui transactional test runner framework. Pulls out common code into `tx_summary_from_effects`. ## Test plan Added new tests for reference return types. Dev inspect flag allows reference return while dry run does not. --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- .../dev_inspect/reference_return_value.exp | 16 +++ .../dev_inspect/reference_return_value.move | 17 +++ .../tests/dry_run/reference_return_value.exp | 16 +++ .../tests/dry_run/reference_return_value.move | 17 +++ .../sui-transactional-test-runner/src/args.rs | 2 + .../sui-transactional-test-runner/src/lib.rs | 29 +++- .../src/test_adapter.rs | 125 ++++++++++++------ 7 files changed, 183 insertions(+), 39 deletions(-) create mode 100644 crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.exp create mode 100644 crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.move create mode 100644 crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.exp create mode 100644 crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.move diff --git a/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.exp b/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.exp new file mode 100644 index 0000000000000..ddb71637c7693 --- /dev/null +++ b/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.exp @@ -0,0 +1,16 @@ +processed 3 tasks + +init: +A: object(0,0) + +task 1, lines 8-14: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 3465600, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 16-17: +//# programmable --sender A --inputs 0 --dev-inspect +//> 0: test::m::return_ref(Input(0)); +mutated: object(_) +gas summary: computation_cost: 500000, storage_cost: 988000, storage_rebate: 0, non_refundable_storage_fee: 0 diff --git a/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.move b/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.move new file mode 100644 index 0000000000000..841260fb5b465 --- /dev/null +++ b/crates/sui-adapter-transactional-tests/tests/dev_inspect/reference_return_value.move @@ -0,0 +1,17 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// attempt to return a reference from a function + +//# init --addresses test=0x0 --accounts A + +//# publish + +module test::m { + public fun return_ref(n: &u64): &u64 { + n + } +} + +//# programmable --sender A --inputs 0 --dev-inspect +//> 0: test::m::return_ref(Input(0)); diff --git a/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.exp b/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.exp new file mode 100644 index 0000000000000..858cbad483f9f --- /dev/null +++ b/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.exp @@ -0,0 +1,16 @@ +processed 3 tasks + +init: +A: object(0,0) + +task 1, lines 8-14: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 3465600, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 16-17: +//# programmable --sender A --inputs 0 --dry-run +//> 0: test::m::return_ref(Input(0)); +Error: Transaction Effects Status: InvalidPublicFunctionReturnType { idx: 0 } in command 0 +Execution Error: InvalidPublicFunctionReturnType { idx: 0 } in command 0 diff --git a/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.move b/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.move new file mode 100644 index 0000000000000..3535f3c9515a1 --- /dev/null +++ b/crates/sui-adapter-transactional-tests/tests/dry_run/reference_return_value.move @@ -0,0 +1,17 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// attempt to return a reference from a function + +//# init --addresses test=0x0 --accounts A + +//# publish + +module test::m { + public fun return_ref(n: &u64): &u64 { + n + } +} + +//# programmable --sender A --inputs 0 --dry-run +//> 0: test::m::return_ref(Input(0)); diff --git a/crates/sui-transactional-test-runner/src/args.rs b/crates/sui-transactional-test-runner/src/args.rs index 4cbbbb8fba54e..1882ca80fffe5 100644 --- a/crates/sui-transactional-test-runner/src/args.rs +++ b/crates/sui-transactional-test-runner/src/args.rs @@ -124,6 +124,8 @@ pub struct ProgrammableTransactionCommand { pub gas_payment: Option, #[clap(long = "dev-inspect")] pub dev_inspect: bool, + #[clap(long = "dry-run")] + pub dry_run: bool, #[clap( long = "inputs", value_parser = ParsedValue::::parse, diff --git a/crates/sui-transactional-test-runner/src/lib.rs b/crates/sui-transactional-test-runner/src/lib.rs index 760c65d1f3c4c..949b98e735c0c 100644 --- a/crates/sui-transactional-test-runner/src/lib.rs +++ b/crates/sui-transactional-test-runner/src/lib.rs @@ -22,8 +22,8 @@ use sui_core::authority::authority_per_epoch_store::CertLockGuard; use sui_core::authority::authority_test_utils::send_and_confirm_transaction_with_execution_error; use sui_core::authority::AuthorityState; use sui_json_rpc::authority_state::StateRead; -use sui_json_rpc_types::DevInspectResults; use sui_json_rpc_types::EventFilter; +use sui_json_rpc_types::{DevInspectResults, DryRunTransactionBlockResponse}; use sui_storage::key_value_store::TransactionKeyValueStore; use sui_types::base_types::ObjectID; use sui_types::base_types::SuiAddress; @@ -45,10 +45,10 @@ use sui_types::storage::ObjectStore; use sui_types::storage::ReadStore; use sui_types::sui_system_state::epoch_start_sui_system_state::EpochStartSystemStateTrait; use sui_types::sui_system_state::SuiSystemStateTrait; -use sui_types::transaction::InputObjects; use sui_types::transaction::Transaction; use sui_types::transaction::TransactionDataAPI; use sui_types::transaction::TransactionKind; +use sui_types::transaction::{InputObjects, TransactionData}; use test_adapter::{SuiTestAdapter, PRE_COMPILED}; #[cfg_attr(not(msim), tokio::main)] @@ -99,6 +99,12 @@ pub trait TransactionalAdapter: Send + Sync + ReadStore { amount: u64, ) -> anyhow::Result; + async fn dry_run_transaction_block( + &self, + transaction_block: TransactionData, + transaction_digest: TransactionDigest, + ) -> SuiResult; + async fn dev_inspect_transaction_block( &self, sender: SuiAddress, @@ -172,6 +178,17 @@ impl TransactionalAdapter for ValidatorWithFullnode { Ok((effects, error)) } + async fn dry_run_transaction_block( + &self, + transaction_block: TransactionData, + transaction_digest: TransactionDigest, + ) -> SuiResult { + self.fullnode + .dry_exec_transaction(transaction_block, transaction_digest) + .await + .map(|result| result.0) + } + async fn dev_inspect_transaction_block( &self, sender: SuiAddress, @@ -409,6 +426,14 @@ impl TransactionalAdapter for Simulacrum { unimplemented!("dev_inspect_transaction_block not supported in simulator mode") } + async fn dry_run_transaction_block( + &self, + _transaction_block: TransactionData, + _transaction_digest: TransactionDigest, + ) -> SuiResult { + unimplemented!("dry_run_transaction_block not supported in simulator mode") + } + async fn query_tx_events_asc( &self, tx_digest: &TransactionDigest, diff --git a/crates/sui-transactional-test-runner/src/test_adapter.rs b/crates/sui-transactional-test-runner/src/test_adapter.rs index d32d21f2748c7..deb0f3b270cdf 100644 --- a/crates/sui-transactional-test-runner/src/test_adapter.rs +++ b/crates/sui-transactional-test-runner/src/test_adapter.rs @@ -54,7 +54,10 @@ use sui_core::authority::AuthorityState; use sui_framework::DEFAULT_FRAMEWORK_PATH; use sui_graphql_rpc::test_infra::cluster::{RetentionConfig, SnapshotLagConfig}; use sui_json_rpc_api::QUERY_MAX_RESULT_LIMIT; -use sui_json_rpc_types::{DevInspectResults, SuiExecutionStatus, SuiTransactionBlockEffectsAPI}; +use sui_json_rpc_types::{ + DevInspectResults, DryRunTransactionBlockResponse, SuiExecutionStatus, + SuiTransactionBlockEffects, SuiTransactionBlockEffectsAPI, SuiTransactionBlockEvents, +}; use sui_protocol_config::{Chain, ProtocolConfig}; use sui_storage::{ key_value_store::TransactionKeyValueStore, key_value_store_metrics::KeyValueStoreMetrics, @@ -832,12 +835,17 @@ impl<'a> MoveTestAdapter<'a> for SuiTestAdapter { gas_price, gas_payment, dev_inspect, + dry_run, inputs, }) => { if dev_inspect && self.is_simulator() { bail!("Dev inspect is not supported on simulator mode"); } + if dry_run && dev_inspect { + bail!("Cannot set both dev-inspect and dry-run"); + } + let inputs = self.compiled_state().resolve_args(inputs)?; let inputs: Vec = inputs .into_iter() @@ -873,7 +881,8 @@ impl<'a> MoveTestAdapter<'a> for SuiTestAdapter { ) }) .collect::>>()?; - let summary = if !dev_inspect { + + let summary = if !dev_inspect && !dry_run { let gas_budget = gas_budget.unwrap_or(DEFAULT_GAS_BUDGET); let gas_price = gas_price.unwrap_or(self.gas_price); let transaction = self.sign_sponsor_txn( @@ -892,6 +901,22 @@ impl<'a> MoveTestAdapter<'a> for SuiTestAdapter { }, ); self.execute_txn(transaction).await? + } else if dry_run { + let gas_budget = gas_budget.unwrap_or(DEFAULT_GAS_BUDGET); + let gas_price = gas_price.unwrap_or(self.gas_price); + let sender = self.get_sender(sender); + let sponsor = sponsor.map_or(sender, |a| self.get_sender(Some(a))); + + let payment = self.get_payment(sponsor, gas_payment); + + let transaction = TransactionData::new_programmable( + sender.address, + vec![payment], + ProgrammableTransaction { inputs, commands }, + gas_budget, + gas_price, + ); + self.dry_run(transaction).await? } else { assert!( gas_budget.is_none(), @@ -1380,6 +1405,7 @@ impl<'a> SuiTestAdapter { gas_budget: Option, policy: u8, gas_price: u64, + // dry_run: bool, ) -> anyhow::Result> { let modules_bytes = modules .iter() @@ -1473,6 +1499,19 @@ impl<'a> SuiTestAdapter { }) } + fn get_payment(&self, sponsor: &TestAccount, payment: Option) -> ObjectRef { + let payment = if let Some(payment) = payment { + self.fake_to_real_object_id(payment) + .expect("Could not find specified payment object") + } else { + sponsor.gas + }; + + self.get_object(&payment, None) + .unwrap() + .compute_object_reference() + } + fn sign_sponsor_txn( &self, sender: Option, @@ -1487,17 +1526,7 @@ impl<'a> SuiTestAdapter { let sender = self.get_sender(sender); let sponsor = sponsor.map_or(sender, |a| self.get_sender(Some(a))); - let payment = if let Some(payment) = payment { - self.fake_to_real_object_id(payment) - .expect("Could not find specified payment object") - } else { - sponsor.gas - }; - - let payment_ref = self - .get_object(&payment, None) - .unwrap() - .compute_object_reference(); + let payment_ref = self.get_payment(sponsor, payment); let data = txn_data(sender.address, sponsor.address, payment_ref); if sender.address == sponsor.address { @@ -1663,6 +1692,19 @@ impl<'a> SuiTestAdapter { } } + async fn dry_run(&mut self, transaction: TransactionData) -> anyhow::Result { + let digest = transaction.digest(); + let results = self + .executor + .dry_run_transaction_block(transaction, digest) + .await?; + let DryRunTransactionBlockResponse { + effects, events, .. + } = results; + + self.tx_summary_from_effects(effects, events) + } + async fn dev_inspect( &mut self, sender: SuiAddress, @@ -1676,6 +1718,21 @@ impl<'a> SuiTestAdapter { let DevInspectResults { effects, events, .. } = results; + + self.tx_summary_from_effects(effects, events) + } + + fn tx_summary_from_effects( + &mut self, + effects: SuiTransactionBlockEffects, + events: SuiTransactionBlockEvents, + ) -> anyhow::Result { + if let SuiExecutionStatus::Failure { error } = effects.status() { + return Err(anyhow::anyhow!(self.stabilize_str(format!( + "Transaction Effects Status: {error}\nExecution Error: {error}", + )))); + } + let mut created_ids: Vec<_> = effects.created().iter().map(|o| o.object_id()).collect(); let mut mutated_ids: Vec<_> = effects.mutated().iter().map(|o| o.object_id()).collect(); let mut unwrapped_ids: Vec<_> = effects.unwrapped().iter().map(|o| o.object_id()).collect(); @@ -1712,30 +1769,24 @@ impl<'a> SuiTestAdapter { unwrapped_then_deleted_ids.sort_by_key(|id| self.real_to_fake_object_id(id)); wrapped_ids.sort_by_key(|id| self.real_to_fake_object_id(id)); - match effects.status() { - SuiExecutionStatus::Success { .. } => { - let events = events - .data - .into_iter() - .map(|sui_event| sui_event.into()) - .collect(); - Ok(TxnSummary { - events, - gas_summary: gas_summary.clone(), - created: created_ids, - mutated: mutated_ids, - unwrapped: unwrapped_ids, - deleted: deleted_ids, - unwrapped_then_deleted: unwrapped_then_deleted_ids, - wrapped: wrapped_ids, - // TODO: Properly propagate unchanged shared objects in dev_inspect. - unchanged_shared: vec![], - }) - } - SuiExecutionStatus::Failure { error } => Err(anyhow::anyhow!(self.stabilize_str( - format!("Transaction Effects Status: {error}\nExecution Error: {error}",) - ))), - } + let events = events + .data + .into_iter() + .map(|sui_event| sui_event.into()) + .collect(); + + Ok(TxnSummary { + events, + gas_summary: gas_summary.clone(), + created: created_ids, + mutated: mutated_ids, + unwrapped: unwrapped_ids, + deleted: deleted_ids, + unwrapped_then_deleted: unwrapped_then_deleted_ids, + wrapped: wrapped_ids, + // TODO: Properly propagate unchanged shared objects in dev_inspect. + unchanged_shared: vec![], + }) } fn get_object(&self, id: &ObjectID, version: Option) -> anyhow::Result { From b6aed4f09a38aae4a76ea3889401327ceceaabd7 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 15:45:24 +0000 Subject: [PATCH 23/30] rpc-alt: tryGetPastObject ## Description Initial support for `tryGetPastObject`. Support for its response options, and multi-gets will come in follow-up PRs. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- objects ``` --- .../jsonrpc/objects/try_get_past_object.exp | 89 +++++++++++++++++++ .../jsonrpc/objects/try_get_past_object.move | 46 ++++++++++ crates/sui-indexer-alt-jsonrpc/src/api/mod.rs | 1 + .../src/api/objects/mod.rs | 60 +++++++++++++ .../src/api/objects/response.rs | 67 ++++++++++++++ crates/sui-indexer-alt-jsonrpc/src/lib.rs | 6 +- 6 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp new file mode 100644 index 0000000000000..a6fc4824df420 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp @@ -0,0 +1,89 @@ +processed 9 tasks + +init: +A: object(0,0), B: object(0,1) + +task 1, lines 11-13: +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(1,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 15-17: +//# programmable --sender A --inputs 43 object(1,0) +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: MergeCoins(Input(1), [Result(0)]) +mutated: object(0,0), object(1,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 + +task 3, lines 19-20: +//# programmable --sender A --inputs object(1,0) +//> 0: MergeCoins(Gas, [Input(0)]) +mutated: object(0,0) +deleted: object(1,0) +gas summary: computation_cost: 1000000, storage_cost: 988000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 + +task 4, line 22: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 24-28: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "status": "VersionNotFound", + "details": [ + "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + 1 + ] + } +} + +task 6, lines 30-34: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "2", + "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745" + } + } +} + +task 7, lines 36-40: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 2, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "3", + "digest": "5p2rFja4CnVp4TWPg2VuRYpDS6d2AUSTQL5mfWBoiVjh" + } + } +} + +task 8, lines 42-46: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 3, + "result": { + "status": "ObjectDeleted", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": 4, + "digest": "7gyGAp71YXQRoxmFBaHxofQXAipvgHyBKPyxmdSJxyvz" + } + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move new file mode 100644 index 0000000000000..d77195ac7faac --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move @@ -0,0 +1,46 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A B --addresses test=0x0 --simulator + +// 1. Trying to fetch an object at too low a version +// 2. ...at its first version +// 3. ...after it has been modified +// 4. ...after it has been deleted + +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 43 object(1,0) +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: MergeCoins(Input(1), [Result(0)]) + +//# programmable --sender A --inputs object(1,0) +//> 0: MergeCoins(Gas, [Input(0)]) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 1] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 2] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 3] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 4] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs index 885cd3b02da68..f5fda76c56463 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/mod.rs @@ -2,5 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 pub(crate) mod governance; +pub(crate) mod objects; pub(crate) mod rpc_module; pub(crate) mod transactions; diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs new file mode 100644 index 0000000000000..887243438e07d --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs @@ -0,0 +1,60 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use sui_json_rpc_types::{SuiObjectDataOptions, SuiPastObjectResponse}; +use sui_open_rpc::Module; +use sui_open_rpc_macros::open_rpc; +use sui_types::base_types::{ObjectID, SequenceNumber}; + +use crate::context::Context; + +use super::rpc_module::RpcModule; + +mod response; + +#[open_rpc(namespace = "sui", tag = "Objects API")] +#[rpc(server, namespace = "sui")] +trait ObjectsApi { + /// Return the object information for a specified version. + /// + /// Note that past versions of an object may be pruned from the system, even if they once + /// existed. Different RPC services may return different responses for the same request as a + /// result, based on their pruning policies. + #[method(name = "tryGetPastObject")] + async fn try_get_past_object( + &self, + /// The ID of the queried object + object_id: ObjectID, + /// The version of the queried object. + version: SequenceNumber, + /// Options for specifying the content to be returned + options: Option, + ) -> RpcResult; +} + +pub(crate) struct Objects(pub Context); + +#[async_trait::async_trait] +impl ObjectsApiServer for Objects { + async fn try_get_past_object( + &self, + object_id: ObjectID, + version: SequenceNumber, + options: Option, + ) -> RpcResult { + let Self(ctx) = self; + let options = options.unwrap_or_default(); + Ok(response::past_object(ctx, object_id, version, &options).await?) + } +} + +impl RpcModule for Objects { + fn schema(&self) -> Module { + ObjectsApiOpenRpc::module_doc() + } + + fn into_impl(self) -> jsonrpsee::RpcModule { + self.into_rpc() + } +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs new file mode 100644 index 0000000000000..e0de0a230deee --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -0,0 +1,67 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Context as _; +use sui_json_rpc_types::{ + SuiObjectData, SuiObjectDataOptions, SuiObjectRef, SuiPastObjectResponse, +}; +use sui_types::{ + base_types::{ObjectID, SequenceNumber}, + digests::ObjectDigest, + object::Object, +}; + +use crate::{context::Context, data::objects::VersionedObjectKey, error::RpcError}; + +/// Fetch the necessary data from the stores in `ctx` and transform it to build a response for a +/// past object identified by its ID and version, according to the response `options`. +pub(super) async fn past_object( + ctx: &Context, + object_id: ObjectID, + version: SequenceNumber, + options: &SuiObjectDataOptions, +) -> Result { + let Some(stored) = ctx + .loader() + .load_one(VersionedObjectKey(object_id, version.value())) + .await + .context("Failed to load object from store")? + else { + return Ok(SuiPastObjectResponse::VersionNotFound(object_id, version)); + }; + + let Some(bytes) = &stored.serialized_object else { + return Ok(SuiPastObjectResponse::ObjectDeleted(SuiObjectRef { + object_id, + version, + digest: ObjectDigest::OBJECT_DIGEST_DELETED, + })); + }; + + Ok(SuiPastObjectResponse::VersionFound(object( + object_id, version, bytes, options, + )?)) +} + +/// Extract a representation of the object from its stored form, according to its response options. +fn object( + object_id: ObjectID, + version: SequenceNumber, + bytes: &[u8], + _options: &SuiObjectDataOptions, +) -> Result { + let object: Object = bcs::from_bytes(bytes).context("Failed to deserialize object")?; + + Ok(SuiObjectData { + object_id, + version, + digest: object.digest(), + type_: None, + owner: None, + previous_transaction: None, + storage_rebate: None, + display: None, + content: None, + bcs: None, + }) +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/lib.rs b/crates/sui-indexer-alt-jsonrpc/src/lib.rs index 93b2fc91db795..6f2fb8ab8e775 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/lib.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/lib.rs @@ -5,6 +5,7 @@ use std::net::SocketAddr; use std::sync::Arc; use anyhow::Context as _; +use api::objects::Objects; use api::rpc_module::RpcModule; use api::transactions::{QueryTransactions, Transactions, TransactionsConfig}; use config::RpcConfig; @@ -192,8 +193,8 @@ impl Default for RpcArgs { /// command-line). The service will continue to run until the cancellation token is triggered, and /// will signal cancellation on the token when it is shutting down. /// -/// The service may spin up auxilliary services (such as the system package task) to support -/// itself, and will clean these up on shutdown as well. +/// The service may spin up auxiliary services (such as the system package task) to support itself, +/// and will clean these up on shutdown as well. pub async fn start_rpc( db_args: DbArgs, rpc_args: RpcArgs, @@ -221,6 +222,7 @@ pub async fn start_rpc( ); rpc.add_module(Governance(context.clone()))?; + rpc.add_module(Objects(context.clone()))?; rpc.add_module(QueryTransactions(context.clone(), transactions_config))?; rpc.add_module(Transactions(context.clone()))?; From ee29d0458275b14aad393638fa66571e813378db Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 15:59:23 +0000 Subject: [PATCH 24/30] rpc-alt: tryGetPastObject showType --- .../jsonrpc/objects/try_get_past_object.exp | 63 ++++++++++++++++--- .../jsonrpc/objects/try_get_past_object.move | 40 ++++++++++++ .../src/api/objects/response.rs | 8 ++- 3 files changed, 99 insertions(+), 12 deletions(-) diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp index a6fc4824df420..11e3aee3d9706 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp @@ -1,9 +1,9 @@ -processed 9 tasks +processed 12 tasks init: A: object(0,0), B: object(0,1) -task 1, lines 11-13: +task 1, lines 15-17: //# programmable --sender A --inputs 42 @A //> 0: SplitCoins(Gas, [Input(0)]); //> 1: TransferObjects([Result(0)], Input(1)) @@ -11,25 +11,25 @@ created: object(1,0) mutated: object(0,0) gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 -task 2, lines 15-17: +task 2, lines 19-21: //# programmable --sender A --inputs 43 object(1,0) //> 0: SplitCoins(Gas, [Input(0)]); //> 1: MergeCoins(Input(1), [Result(0)]) mutated: object(0,0), object(1,0) gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 -task 3, lines 19-20: +task 3, lines 23-24: //# programmable --sender A --inputs object(1,0) //> 0: MergeCoins(Gas, [Input(0)]) mutated: object(0,0) deleted: object(1,0) gas summary: computation_cost: 1000000, storage_cost: 988000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 -task 4, line 22: +task 4, line 26: //# create-checkpoint Checkpoint created: 1 -task 5, lines 24-28: +task 5, lines 28-32: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -43,7 +43,7 @@ Response: { } } -task 6, lines 30-34: +task 6, lines 34-38: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -58,7 +58,7 @@ Response: { } } -task 7, lines 36-40: +task 7, lines 40-44: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -73,7 +73,7 @@ Response: { } } -task 8, lines 42-46: +task 8, lines 46-50: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -87,3 +87,48 @@ Response: { } } } + +task 9, lines 52-62: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 4, + "result": { + "status": "VersionNotFound", + "details": [ + "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + 1 + ] + } +} + +task 10, lines 64-74: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 5, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "2", + "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745", + "type": "0x2::coin::Coin<0x2::sui::SUI>" + } + } +} + +task 11, lines 76-86: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 6, + "result": { + "status": "ObjectDeleted", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": 4, + "digest": "7gyGAp71YXQRoxmFBaHxofQXAipvgHyBKPyxmdSJxyvz" + } + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move index d77195ac7faac..4fa30b9279704 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move @@ -7,6 +7,10 @@ // 2. ...at its first version // 3. ...after it has been modified // 4. ...after it has been deleted +// 5. Show the type of a non-existent object verson +// 6. Show the type of an object version +// 7. Show the type of a deleted object version + //# programmable --sender A --inputs 42 @A //> 0: SplitCoins(Gas, [Input(0)]); @@ -44,3 +48,39 @@ "method": "sui_tryGetPastObject", "params": ["@{obj_1_0}", 4] } + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": [ + "@{obj_1_0}", + 1, + { + "showType": true + } + ] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": [ + "@{obj_1_0}", + 2, + { + "showType": true + } + ] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": [ + "@{obj_1_0}", + 4, + { + "showType": true + } + ] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs index e0de0a230deee..9b5c936d95e5e 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -6,7 +6,7 @@ use sui_json_rpc_types::{ SuiObjectData, SuiObjectDataOptions, SuiObjectRef, SuiPastObjectResponse, }; use sui_types::{ - base_types::{ObjectID, SequenceNumber}, + base_types::{ObjectID, ObjectType, SequenceNumber}, digests::ObjectDigest, object::Object, }; @@ -48,15 +48,17 @@ fn object( object_id: ObjectID, version: SequenceNumber, bytes: &[u8], - _options: &SuiObjectDataOptions, + options: &SuiObjectDataOptions, ) -> Result { let object: Object = bcs::from_bytes(bytes).context("Failed to deserialize object")?; + let type_ = options.show_type.then(|| ObjectType::from(&object)); + Ok(SuiObjectData { object_id, version, digest: object.digest(), - type_: None, + type_, owner: None, previous_transaction: None, storage_rebate: None, From a17e52d5b8e7eec07d4853d9b7568b15d5657d8b Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 16:49:49 +0000 Subject: [PATCH 25/30] rpc-alt: tryGetPastObject showOwner ## Description Added support for displaying ownership information. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- objects ``` --- .../tests/jsonrpc/objects/show_owner.exp | 145 ++++++++++++++++++ .../tests/jsonrpc/objects/show_owner.move | 65 ++++++++ .../jsonrpc/objects/try_get_past_object.exp | 27 ++-- .../jsonrpc/objects/try_get_past_object.move | 16 +- .../src/api/objects/response.rs | 3 +- 5 files changed, 236 insertions(+), 20 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.move diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.exp new file mode 100644 index 0000000000000..13539846e7eff --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.exp @@ -0,0 +1,145 @@ +processed 13 tasks + +init: +A: object(0,0), B: object(0,1) + +task 1, lines 12-14: +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(1,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 16-18: +//# programmable --sender A --inputs 43 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 3, lines 20-22: +//# programmable --sender A --inputs 44 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_share_object>(Result(0)) +created: object(3,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 4, lines 24-26: +//# programmable --sender A --inputs 45 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_freeze_object>(Result(0)) +created: object(4,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 5, lines 28-30: +//# programmable --sender A --inputs @A +//> 0: sui::table::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(5,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2333200, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 6, lines 32-33: +//# programmable --sender A --inputs object(5,0) 46 47 +//> 0: sui::table::add(Input(0), Input(1), Input(2)) +created: object(6,0) +mutated: object(0,0), object(5,0) +gas summary: computation_cost: 1000000, storage_cost: 3800000, storage_rebate: 2309868, non_refundable_storage_fee: 23332 + +task 7, line 35: +//# create-checkpoint +Checkpoint created: 1 + +task 8, lines 37-41: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "2", + "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + } + } + } +} + +task 9, lines 43-47: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xf2945213c986e35d59aae258ef1b094e81ead5cc982b0d1519625bf3f81bee68", + "version": "3", + "digest": "9wMLcUMLvkbqNFyoPtUJ1GPuVuXVqpLA3GP2DHL2SyL6", + "owner": { + "AddressOwner": "0xa7b032703878aa74c3126935789fd1d4d7e111d5911b09247d6963061c312b5a" + } + } + } +} + +task 10, lines 49-53: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 2, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0x0918a1917bda4d05f1cb4db824532a63476c57d65c548930406fe02051f22819", + "version": "4", + "digest": "8pirmp97ym3pzSCabaPsoPYVPbKYQHfnPqEbCyk32Au2", + "owner": { + "Shared": { + "initial_shared_version": 4 + } + } + } + } +} + +task 11, lines 55-59: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 3, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xf079987ab8a34791ae4dd8b2d2a9af3485496a49b382b22239c4ca43902fe18a", + "version": "5", + "digest": "EEF6GF5DCZRZ4zxsubwzaQhYWysCdGJ63sLRLE15d9F", + "owner": "Immutable" + } + } +} + +task 12, lines 61-65: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 4, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + "version": "7", + "digest": "9rcnz8P9Eie7KZPUjTWnmTzyUxsBMaJcb9G3ncuG4BiC", + "owner": { + "ObjectOwner": "0x2867eee076a7f41efb3dc507f27531c26319dfbf0c9a6d92f6d1fd0f8d9bdeeb" + } + } + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.move new file mode 100644 index 0000000000000..c8ab26410590d --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/show_owner.move @@ -0,0 +1,65 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A B --addresses test=0x0 --simulator + +// 1. Show the owner of an object owned by one address +// 2. ...owned by another address +// 3. ...shared +// 4. ...frozen +// 5. ...owned by an object + +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 43 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 44 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_share_object>(Result(0)) + +//# programmable --sender A --inputs 45 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_freeze_object>(Result(0)) + +//# programmable --sender A --inputs @A +//> 0: sui::table::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(5,0) 46 47 +//> 0: sui::table::add(Input(0), Input(1), Input(2)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 2, { "showOwner": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_2_0}", 3, { "showOwner": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_3_0}", 4, { "showOwner": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_4_0}", 5, { "showOwner": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_6_0}", 7, { "showOwner": true }] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp index 11e3aee3d9706..b80e79fdac2b5 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp @@ -3,7 +3,7 @@ processed 12 tasks init: A: object(0,0), B: object(0,1) -task 1, lines 15-17: +task 1, lines 14-16: //# programmable --sender A --inputs 42 @A //> 0: SplitCoins(Gas, [Input(0)]); //> 1: TransferObjects([Result(0)], Input(1)) @@ -11,25 +11,25 @@ created: object(1,0) mutated: object(0,0) gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 -task 2, lines 19-21: +task 2, lines 18-20: //# programmable --sender A --inputs 43 object(1,0) //> 0: SplitCoins(Gas, [Input(0)]); //> 1: MergeCoins(Input(1), [Result(0)]) mutated: object(0,0), object(1,0) gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 -task 3, lines 23-24: +task 3, lines 22-23: //# programmable --sender A --inputs object(1,0) //> 0: MergeCoins(Gas, [Input(0)]) mutated: object(0,0) deleted: object(1,0) gas summary: computation_cost: 1000000, storage_cost: 988000, storage_rebate: 1956240, non_refundable_storage_fee: 19760 -task 4, line 26: +task 4, line 25: //# create-checkpoint Checkpoint created: 1 -task 5, lines 28-32: +task 5, lines 27-31: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -43,7 +43,7 @@ Response: { } } -task 6, lines 34-38: +task 6, lines 33-37: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -58,7 +58,7 @@ Response: { } } -task 7, lines 40-44: +task 7, lines 39-43: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -73,7 +73,7 @@ Response: { } } -task 8, lines 46-50: +task 8, lines 45-49: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -88,7 +88,7 @@ Response: { } } -task 9, lines 52-62: +task 9, lines 51-62: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -102,7 +102,7 @@ Response: { } } -task 10, lines 64-74: +task 10, lines 64-75: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -113,12 +113,15 @@ Response: { "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", "version": "2", "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745", - "type": "0x2::coin::Coin<0x2::sui::SUI>" + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + } } } } -task 11, lines 76-86: +task 11, lines 77-88: //# run-jsonrpc Response: { "jsonrpc": "2.0", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move index 4fa30b9279704..8d4247d22d071 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move @@ -7,10 +7,9 @@ // 2. ...at its first version // 3. ...after it has been modified // 4. ...after it has been deleted -// 5. Show the type of a non-existent object verson -// 6. Show the type of an object version -// 7. Show the type of a deleted object version - +// 5. Show the details of a non-existent object verson +// 6. Show the details of an object version +// 7. Show the details of a deleted object version //# programmable --sender A --inputs 42 @A //> 0: SplitCoins(Gas, [Input(0)]); @@ -56,7 +55,8 @@ "@{obj_1_0}", 1, { - "showType": true + "showType": true, + "showOwner": true } ] } @@ -68,7 +68,8 @@ "@{obj_1_0}", 2, { - "showType": true + "showType": true, + "showOwner": true } ] } @@ -80,7 +81,8 @@ "@{obj_1_0}", 4, { - "showType": true + "showType": true, + "showOwner": true } ] } diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs index 9b5c936d95e5e..08361fd8eb144 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -53,13 +53,14 @@ fn object( let object: Object = bcs::from_bytes(bytes).context("Failed to deserialize object")?; let type_ = options.show_type.then(|| ObjectType::from(&object)); + let owner = options.show_owner.then(|| object.owner().clone()); Ok(SuiObjectData { object_id, version, digest: object.digest(), type_, - owner: None, + owner, previous_transaction: None, storage_rebate: None, display: None, From 7fb1011858b6f6cc9c90882c774ca2dbd855b23d Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 16:53:28 +0000 Subject: [PATCH 26/30] rpc-alt: tryGetPastObject showPreviousTransaction --- .../tests/jsonrpc/objects/try_get_past_object.exp | 9 +++++---- .../tests/jsonrpc/objects/try_get_past_object.move | 9 ++++++--- .../sui-indexer-alt-jsonrpc/src/api/objects/response.rs | 5 ++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp index b80e79fdac2b5..ed951fb6edf8f 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp @@ -88,7 +88,7 @@ Response: { } } -task 9, lines 51-62: +task 9, lines 51-63: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -102,7 +102,7 @@ Response: { } } -task 10, lines 64-75: +task 10, lines 65-77: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -116,12 +116,13 @@ Response: { "type": "0x2::coin::Coin<0x2::sui::SUI>", "owner": { "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" - } + }, + "previousTransaction": "ABo3jemYqdBWMRYqsaCYUKnGDp64tR7jK2fH1mKCoJLk" } } } -task 11, lines 77-88: +task 11, lines 79-91: //# run-jsonrpc Response: { "jsonrpc": "2.0", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move index 8d4247d22d071..7856d3ac850ee 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move @@ -56,7 +56,8 @@ 1, { "showType": true, - "showOwner": true + "showOwner": true, + "showPreviousTransaction": true } ] } @@ -69,7 +70,8 @@ 2, { "showType": true, - "showOwner": true + "showOwner": true, + "showPreviousTransaction": true } ] } @@ -82,7 +84,8 @@ 4, { "showType": true, - "showOwner": true + "showOwner": true, + "showPreviousTransaction": true } ] } diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs index 08361fd8eb144..7d4543fbac85d 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -54,6 +54,9 @@ fn object( let type_ = options.show_type.then(|| ObjectType::from(&object)); let owner = options.show_owner.then(|| object.owner().clone()); + let previous_transaction = options + .show_previous_transaction + .then(|| object.previous_transaction); Ok(SuiObjectData { object_id, @@ -61,7 +64,7 @@ fn object( digest: object.digest(), type_, owner, - previous_transaction: None, + previous_transaction, storage_rebate: None, display: None, content: None, From a6c82445da907f782be92437c8da87a93d256d05 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 16:55:57 +0000 Subject: [PATCH 27/30] rpc-alt: tryGetPastObject showStorageRebate ## Description Implement support for the `showStorageRebate` sui object response option. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- objects ``` --- .../tests/jsonrpc/objects/try_get_past_object.exp | 9 +++++---- .../tests/jsonrpc/objects/try_get_past_object.move | 9 ++++++--- .../sui-indexer-alt-jsonrpc/src/api/objects/response.rs | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp index ed951fb6edf8f..59dc498c12eaa 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.exp @@ -88,7 +88,7 @@ Response: { } } -task 9, lines 51-63: +task 9, lines 51-64: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -102,7 +102,7 @@ Response: { } } -task 10, lines 65-77: +task 10, lines 66-79: //# run-jsonrpc Response: { "jsonrpc": "2.0", @@ -117,12 +117,13 @@ Response: { "owner": { "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" }, - "previousTransaction": "ABo3jemYqdBWMRYqsaCYUKnGDp64tR7jK2fH1mKCoJLk" + "previousTransaction": "ABo3jemYqdBWMRYqsaCYUKnGDp64tR7jK2fH1mKCoJLk", + "storageRebate": "988000" } } } -task 11, lines 79-91: +task 11, lines 81-94: //# run-jsonrpc Response: { "jsonrpc": "2.0", diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move index 7856d3ac850ee..7f4a2533afd7b 100644 --- a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_get_past_object.move @@ -57,7 +57,8 @@ { "showType": true, "showOwner": true, - "showPreviousTransaction": true + "showPreviousTransaction": true, + "showStorageRebate": true } ] } @@ -71,7 +72,8 @@ { "showType": true, "showOwner": true, - "showPreviousTransaction": true + "showPreviousTransaction": true, + "showStorageRebate": true } ] } @@ -85,7 +87,8 @@ { "showType": true, "showOwner": true, - "showPreviousTransaction": true + "showPreviousTransaction": true, + "showStorageRebate": true } ] } diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs index 7d4543fbac85d..68c1831274442 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -57,6 +57,7 @@ fn object( let previous_transaction = options .show_previous_transaction .then(|| object.previous_transaction); + let storage_rebate = options.show_storage_rebate.then(|| object.storage_rebate); Ok(SuiObjectData { object_id, @@ -65,7 +66,7 @@ fn object( type_, owner, previous_transaction, - storage_rebate: None, + storage_rebate, display: None, content: None, bcs: None, From 6042aa47eb535e9065460791de18b8b348b96a34 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 18:22:49 +0000 Subject: [PATCH 28/30] rpc-alt: tryGetPastObject showContent, showBcs --- .../tests/jsonrpc/objects/contents.exp | 141 ++++++++++++++++++ .../tests/jsonrpc/objects/contents.move | 47 ++++++ .../src/api/objects/response.rs | 78 ++++++++-- 3 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.move diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.exp new file mode 100644 index 0000000000000..4c2d4383c9818 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.exp @@ -0,0 +1,141 @@ +processed 8 tasks + +init: +A: object(0,0) + +task 1, lines 10-19: +//# publish +created: object(1,0) +mutated: object(0,1) +gas summary: computation_cost: 1000000, storage_cost: 4620800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 21-23: +//# programmable --sender A --inputs @A +//> 0: test::mod::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2226800, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 3, lines 25-27: +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(3,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 4, line 29: +//# create-checkpoint +Checkpoint created: 1 + +task 5, lines 31-35: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xb1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73", + "version": "1", + "digest": "Sgp59rDKZRKoSZ2ZJKKcc5YQ7MrhEHDxbZLqqFjywcq", + "content": { + "dataType": "package", + "disassembled": { + "mod": "// Move bytecode v6\nmodule b1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73.mod {\nuse 0000000000000000000000000000000000000000000000000000000000000002::object;\nuse 0000000000000000000000000000000000000000000000000000000000000002::tx_context;\n\nstruct Foo has store, key {\n\tid: UID\n}\n\npublic new(Arg0: &mut TxContext): Foo {\nB0:\n\t0: MoveLoc[0](Arg0: &mut TxContext)\n\t1: Call object::new(&mut TxContext): UID\n\t2: Pack[0](Foo)\n\t3: Ret\n}\n\n}\n" + } + }, + "bcs": { + "dataType": "package", + "id": "0xb1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73", + "version": 1, + "moduleMap": { + "mod": "oRzrCwYAAAAIAQAGAgYMAxIKBRwLBycvCFZACpYBBgycAQ0ABAEGAQcAAAwAAQIEAAIBAgAABQABAAEFAAMAAQcIAgEIAAABCAEDRm9vCVR4Q29udGV4dANVSUQCaWQDbW9kA25ldwZvYmplY3QKdHhfY29udGV4dLHRFHcL/Jlooq09qcbVvL8y5Lzz0L8+umdN9dkHqD5zAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAgEDCAEAAQAAAgQLABEBEgACAA==" + }, + "typeOriginTable": [ + { + "module_name": "mod", + "datatype_name": "Foo", + "package": "0xb1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73" + } + ], + "linkageTable": { + "0x0000000000000000000000000000000000000000000000000000000000000001": { + "upgraded_id": "0x0000000000000000000000000000000000000000000000000000000000000001", + "upgraded_version": 1 + }, + "0x0000000000000000000000000000000000000000000000000000000000000002": { + "upgraded_id": "0x0000000000000000000000000000000000000000000000000000000000000002", + "upgraded_version": 1 + } + } + } + } + } +} + +task 6, lines 37-41: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0x5736caa914301b5f6bc2734fdd6ef4c0097ebf6f0b346ec0ce1119f3a86c8c37", + "version": "2", + "digest": "6pd5G7cgCu3ShFTxpyo7xJTsq5EwzbAnrmfNeR55roUH", + "content": { + "dataType": "moveObject", + "type": "0xb1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73::mod::Foo", + "hasPublicTransfer": true, + "fields": { + "id": { + "id": "0x5736caa914301b5f6bc2734fdd6ef4c0097ebf6f0b346ec0ce1119f3a86c8c37" + } + } + }, + "bcs": { + "dataType": "moveObject", + "type": "0xb1d114770bfc9968a2ad3da9c6d5bcbf32e4bcf3d0bf3eba674df5d907a83e73::mod::Foo", + "hasPublicTransfer": true, + "version": 2, + "bcsBytes": "VzbKqRQwG19rwnNP3W70wAl+v28LNG7AzhEZ86hsjDc=" + } + } + } +} + +task 7, lines 43-47: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 2, + "result": { + "status": "VersionFound", + "details": { + "objectId": "0xf2a6833ec5d2dd77e656a3fe62bdd4e4609b23fa0739312e384fdbb06080155e", + "version": "3", + "digest": "AeAb1PukmXSZNcUMrQmqqWeEwSwyoKXhSGogxN37Wdym", + "content": { + "dataType": "moveObject", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "hasPublicTransfer": true, + "fields": { + "balance": "42", + "id": { + "id": "0xf2a6833ec5d2dd77e656a3fe62bdd4e4609b23fa0739312e384fdbb06080155e" + } + } + }, + "bcs": { + "dataType": "moveObject", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "hasPublicTransfer": true, + "version": 3, + "bcsBytes": "8qaDPsXS3XfmVqP+Yr3U5GCbI/oHOTEuOE/bsGCAFV4qAAAAAAAAAA==" + } + } + } +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.move new file mode 100644 index 0000000000000..5f1fb379cb3a5 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/contents.move @@ -0,0 +1,47 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A --addresses test=0x0 --simulator + +// 1. View the contents of a package +// 2. View the contents of an arbitrary object +// 3. View the contents of a coin + +//# publish +module test::mod { + public struct Foo has key, store { + id: UID, + } + + public fun new(ctx: &mut TxContext): Foo { + Foo { id: object::new(ctx) } + } +} + +//# programmable --sender A --inputs @A +//> 0: test::mod::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_1_0}", 1, { "showContent": true, "showBcs": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_2_0}", 2, { "showContent": true, "showBcs": true }] +} + +//# run-jsonrpc +{ + "method": "sui_tryGetPastObject", + "params": ["@{obj_3_0}", 3, { "showContent": true, "showBcs": true }] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs index 68c1831274442..4b2b3f0cd1323 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/response.rs @@ -2,16 +2,25 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Context as _; +use futures::future::OptionFuture; +use move_core_types::annotated_value::MoveTypeLayout; use sui_json_rpc_types::{ - SuiObjectData, SuiObjectDataOptions, SuiObjectRef, SuiPastObjectResponse, + SuiData, SuiObjectData, SuiObjectDataOptions, SuiObjectRef, SuiParsedData, + SuiPastObjectResponse, SuiRawData, }; use sui_types::{ base_types::{ObjectID, ObjectType, SequenceNumber}, digests::ObjectDigest, - object::Object, + object::{Data, Object}, + TypeTag, }; +use tokio::join; -use crate::{context::Context, data::objects::VersionedObjectKey, error::RpcError}; +use crate::{ + context::Context, + data::objects::VersionedObjectKey, + error::{rpc_bail, RpcError}, +}; /// Fetch the necessary data from the stores in `ctx` and transform it to build a response for a /// past object identified by its ID and version, according to the response `options`. @@ -38,13 +47,14 @@ pub(super) async fn past_object( })); }; - Ok(SuiPastObjectResponse::VersionFound(object( - object_id, version, bytes, options, - )?)) + Ok(SuiPastObjectResponse::VersionFound( + object(ctx, object_id, version, bytes, options).await?, + )) } /// Extract a representation of the object from its stored form, according to its response options. -fn object( +async fn object( + ctx: &Context, object_id: ObjectID, version: SequenceNumber, bytes: &[u8], @@ -59,6 +69,26 @@ fn object( .then(|| object.previous_transaction); let storage_rebate = options.show_storage_rebate.then(|| object.storage_rebate); + let content: OptionFuture<_> = options + .show_content + .then(|| object_data::(ctx, &object)) + .into(); + + let bcs: OptionFuture<_> = options + .show_bcs + .then(|| object_data::(ctx, &object)) + .into(); + + let (content, bcs) = join!(content, bcs); + + let content = content + .transpose() + .context("Failed to deserialize object content")?; + + let bcs = bcs + .transpose() + .context("Failed to deserialize object to BCS")?; + Ok(SuiObjectData { object_id, version, @@ -68,7 +98,37 @@ fn object( previous_transaction, storage_rebate, display: None, - content: None, - bcs: None, + content, + bcs, + }) +} + +/// Extract the contents of an object, in a format chosen by the `D` type parameter. +/// This operaton can fail if it's not possible to get the type layout for the object's type. +async fn object_data(ctx: &Context, object: &Object) -> Result { + Ok(match object.data.clone() { + Data::Package(move_package) => D::try_from_package(move_package)?, + + Data::Move(move_object) => { + let type_: TypeTag = move_object.type_().clone().into(); + let MoveTypeLayout::Struct(layout) = ctx + .package_resolver() + .type_layout(type_.clone()) + .await + .with_context(|| { + format!( + "Failed to resolve type layout for {}", + type_.to_canonical_display(/*with_prefix */ true) + ) + })? + else { + rpc_bail!( + "Type {} is not a struct", + type_.to_canonical_display(/*with_prefix */ true) + ); + }; + + D::try_from_object(move_object, *layout)? + } }) } From 3e3fd495e194484c652559704aa3c0b1d6e6f3f2 Mon Sep 17 00:00:00 2001 From: Ashok Menon Date: Tue, 28 Jan 2025 22:48:09 +0000 Subject: [PATCH 29/30] rpc-alt: tryMultiGetPastObjects ## Description This was another part of #20962, now added back accounting for the changes to errors and data loaders. This change also: - Imposes a max limit on the number of objects that can be fetched in one go. - Uses the same architecture as `queryTransactionBlock`, where the multi-get works by calling the function that loads data from a data loader and generates a response concurrently. ## Test plan New E2E tests: ``` sui$ cargo nextest run -p sui-indexer-alt-e2e-tests -- objects ``` --- .../objects/try_multi_get_past_objects.exp | 242 ++++++++++++++++++ .../objects/try_multi_get_past_objects.move | 75 ++++++ .../src/api/objects/error.rs | 8 + .../src/api/objects/mod.rs | 76 +++++- crates/sui-indexer-alt-jsonrpc/src/config.rs | 35 ++- crates/sui-indexer-alt-jsonrpc/src/lib.rs | 6 +- 6 files changed, 435 insertions(+), 7 deletions(-) create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.exp create mode 100644 crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.move create mode 100644 crates/sui-indexer-alt-jsonrpc/src/api/objects/error.rs diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.exp b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.exp new file mode 100644 index 0000000000000..cca2be66acb19 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.exp @@ -0,0 +1,242 @@ +processed 11 tasks + +init: +A: object(0,0), B: object(0,1) + +task 1, lines 9-11: +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(1,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 0, non_refundable_storage_fee: 0 + +task 2, lines 13-15: +//# programmable --sender A --inputs 43 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) +created: object(2,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 3, lines 17-19: +//# programmable --sender A --inputs 44 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_share_object>(Result(0)) +created: object(3,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 4, lines 21-23: +//# programmable --sender A --inputs 45 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_freeze_object>(Result(0)) +created: object(4,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 1976000, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 5, lines 25-27: +//# programmable --sender A --inputs @A +//> 0: sui::table::new(); +//> 1: TransferObjects([Result(0)], Input(0)) +created: object(5,0) +mutated: object(0,0) +gas summary: computation_cost: 1000000, storage_cost: 2333200, storage_rebate: 978120, non_refundable_storage_fee: 9880 + +task 6, lines 29-30: +//# programmable --sender A --inputs object(5,0) 46 47 +//> 0: sui::table::add(Input(0), Input(1), Input(2)) +created: object(6,0) +mutated: object(0,0), object(5,0) +gas summary: computation_cost: 1000000, storage_cost: 3800000, storage_rebate: 2309868, non_refundable_storage_fee: 23332 + +task 7, lines 32-33: +//# programmable --sender A --inputs object(5,0) 46 +//> 0: sui::table::remove(Input(0), Input(1)) +mutated: object(0,0), object(5,0) +deleted: object(6,0) +gas summary: computation_cost: 1000000, storage_cost: 2333200, storage_rebate: 3762000, non_refundable_storage_fee: 38000 + +task 8, line 35: +//# create-checkpoint +Checkpoint created: 1 + +task 9, lines 37-52: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 0, + "result": [ + { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "2", + "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0xf2945213c986e35d59aae258ef1b094e81ead5cc982b0d1519625bf3f81bee68", + "version": "3", + "digest": "9wMLcUMLvkbqNFyoPtUJ1GPuVuXVqpLA3GP2DHL2SyL6" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x0918a1917bda4d05f1cb4db824532a63476c57d65c548930406fe02051f22819", + "version": "4", + "digest": "8pirmp97ym3pzSCabaPsoPYVPbKYQHfnPqEbCyk32Au2" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0xf079987ab8a34791ae4dd8b2d2a9af3485496a49b382b22239c4ca43902fe18a", + "version": "5", + "digest": "EEF6GF5DCZRZ4zxsubwzaQhYWysCdGJ63sLRLE15d9F" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x2867eee076a7f41efb3dc507f27531c26319dfbf0c9a6d92f6d1fd0f8d9bdeeb", + "version": "6", + "digest": "EtpqCkaW8jrLkvbeL9Mgy3REUMX73FrGxEjGiFCz8TSm" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + "version": "7", + "digest": "9rcnz8P9Eie7KZPUjTWnmTzyUxsBMaJcb9G3ncuG4BiC" + } + }, + { + "status": "ObjectDeleted", + "details": { + "objectId": "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + "version": 8, + "digest": "7gyGAp71YXQRoxmFBaHxofQXAipvgHyBKPyxmdSJxyvz" + } + }, + { + "status": "VersionNotFound", + "details": [ + "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + 9 + ] + } + ] +} + +task 10, lines 54-75: +//# run-jsonrpc +Response: { + "jsonrpc": "2.0", + "id": 1, + "result": [ + { + "status": "VersionFound", + "details": { + "objectId": "0xc442f9431f789a6c80078a63e707de84c7815c3150e08f1e44be256bd05a2b81", + "version": "2", + "digest": "4wV3VBJGNaMCv9zfjSna8c4as6EkuYoqTHHjueS6S745", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "previousTransaction": "ABo3jemYqdBWMRYqsaCYUKnGDp64tR7jK2fH1mKCoJLk", + "storageRebate": "988000" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0xf2945213c986e35d59aae258ef1b094e81ead5cc982b0d1519625bf3f81bee68", + "version": "3", + "digest": "9wMLcUMLvkbqNFyoPtUJ1GPuVuXVqpLA3GP2DHL2SyL6", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "owner": { + "AddressOwner": "0xa7b032703878aa74c3126935789fd1d4d7e111d5911b09247d6963061c312b5a" + }, + "previousTransaction": "DHGGkfgVSyMV4vZqbHe5jPcJLX5Wrp3m7d5mdLUn62dX", + "storageRebate": "988000" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x0918a1917bda4d05f1cb4db824532a63476c57d65c548930406fe02051f22819", + "version": "4", + "digest": "8pirmp97ym3pzSCabaPsoPYVPbKYQHfnPqEbCyk32Au2", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "owner": { + "Shared": { + "initial_shared_version": 4 + } + }, + "previousTransaction": "FDCEMiaPqGUUFjB4sV1EMCFisw7W7WMJJ6nWTBwnoHcc", + "storageRebate": "988000" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0xf079987ab8a34791ae4dd8b2d2a9af3485496a49b382b22239c4ca43902fe18a", + "version": "5", + "digest": "EEF6GF5DCZRZ4zxsubwzaQhYWysCdGJ63sLRLE15d9F", + "type": "0x2::coin::Coin<0x2::sui::SUI>", + "owner": "Immutable", + "previousTransaction": "4ZmQdHopY1d8eg5zsTVciezKwhTbuQGaybXXBMWjs7ax", + "storageRebate": "988000" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x2867eee076a7f41efb3dc507f27531c26319dfbf0c9a6d92f6d1fd0f8d9bdeeb", + "version": "6", + "digest": "EtpqCkaW8jrLkvbeL9Mgy3REUMX73FrGxEjGiFCz8TSm", + "type": "0x2::table::Table", + "owner": { + "AddressOwner": "0xfccc9a421bbb13c1a66a1aa98f0ad75029ede94857779c6915b44f94068b921e" + }, + "previousTransaction": "7DxXXvjVgFuqLXPRaZKAVm12cQWopxbVySBdNqmUCRVJ", + "storageRebate": "1345200" + } + }, + { + "status": "VersionFound", + "details": { + "objectId": "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + "version": "7", + "digest": "9rcnz8P9Eie7KZPUjTWnmTzyUxsBMaJcb9G3ncuG4BiC", + "type": "0x2::dynamic_field::Field", + "owner": { + "ObjectOwner": "0x2867eee076a7f41efb3dc507f27531c26319dfbf0c9a6d92f6d1fd0f8d9bdeeb" + }, + "previousTransaction": "6mXaEAhjYrN5QF4mBX1bLK2T8h4XtmYqqi3FNvtsd3TY", + "storageRebate": "1466800" + } + }, + { + "status": "ObjectDeleted", + "details": { + "objectId": "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + "version": 8, + "digest": "7gyGAp71YXQRoxmFBaHxofQXAipvgHyBKPyxmdSJxyvz" + } + }, + { + "status": "VersionNotFound", + "details": [ + "0x0dfe3bf8505600c374ecc59e9580d3504a37a14f65b53084ef86c5861f39cc3f", + 9 + ] + } + ] +} diff --git a/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.move b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.move new file mode 100644 index 0000000000000..5dda488ea4313 --- /dev/null +++ b/crates/sui-indexer-alt-e2e-tests/tests/jsonrpc/objects/try_multi_get_past_objects.move @@ -0,0 +1,75 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//# init --protocol-version 70 --accounts A B --addresses test=0x0 --simulator + +// 1. Multi-get objects (with found object, with non-existent object, with deleted object) +// 2. Multi-get objects with options + +//# programmable --sender A --inputs 42 @A +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 43 @B +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: TransferObjects([Result(0)], Input(1)) + +//# programmable --sender A --inputs 44 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_share_object>(Result(0)) + +//# programmable --sender A --inputs 45 +//> 0: SplitCoins(Gas, [Input(0)]); +//> 1: sui::transfer::public_freeze_object>(Result(0)) + +//# programmable --sender A --inputs @A +//> 0: sui::table::new(); +//> 1: TransferObjects([Result(0)], Input(0)) + +//# programmable --sender A --inputs object(5,0) 46 47 +//> 0: sui::table::add(Input(0), Input(1), Input(2)) + +//# programmable --sender A --inputs object(5,0) 46 +//> 0: sui::table::remove(Input(0), Input(1)) + +//# create-checkpoint + +//# run-jsonrpc +{ + "method": "sui_tryMultiGetPastObjects", + "params": [ + [ + { "objectId": "@{obj_1_0}", "version": "2" }, + { "objectId": "@{obj_2_0}", "version": "3" }, + { "objectId": "@{obj_3_0}", "version": "4" }, + { "objectId": "@{obj_4_0}", "version": "5" }, + { "objectId": "@{obj_5_0}", "version": "6" }, + { "objectId": "@{obj_6_0}", "version": "7" }, + { "objectId": "@{obj_6_0}", "version": "8" }, + { "objectId": "@{obj_6_0}", "version": "9" } + ] + ] +} + +//# run-jsonrpc +{ + "method": "sui_tryMultiGetPastObjects", + "params": [ + [ + { "objectId": "@{obj_1_0}", "version": "2" }, + { "objectId": "@{obj_2_0}", "version": "3" }, + { "objectId": "@{obj_3_0}", "version": "4" }, + { "objectId": "@{obj_4_0}", "version": "5" }, + { "objectId": "@{obj_5_0}", "version": "6" }, + { "objectId": "@{obj_6_0}", "version": "7" }, + { "objectId": "@{obj_6_0}", "version": "8" }, + { "objectId": "@{obj_6_0}", "version": "9" } + ], + { + "showType": true, + "showOwner": true, + "showPreviousTransaction": true, + "showStorageRebate": true + } + ] +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/error.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/error.rs new file mode 100644 index 0000000000000..d898341eaedb6 --- /dev/null +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/error.rs @@ -0,0 +1,8 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +#[derive(thiserror::Error, Debug)] +pub(super) enum Error { + #[error("Requested {requested} keys, exceeding maximum {max}")] + TooManyKeys { requested: usize, max: usize }, +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs b/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs index 887243438e07d..b8f9b67e7bddd 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/api/objects/mod.rs @@ -1,16 +1,24 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +use futures::future; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; -use sui_json_rpc_types::{SuiObjectDataOptions, SuiPastObjectResponse}; +use serde::{Deserialize, Serialize}; +use sui_json_rpc_types::{SuiGetPastObjectRequest, SuiObjectDataOptions, SuiPastObjectResponse}; use sui_open_rpc::Module; use sui_open_rpc_macros::open_rpc; use sui_types::base_types::{ObjectID, SequenceNumber}; -use crate::context::Context; +use crate::{ + context::Context, + error::{invalid_params, InternalContext}, +}; use super::rpc_module::RpcModule; +use self::error::Error; + +mod error; mod response; #[open_rpc(namespace = "sui", tag = "Objects API")] @@ -31,9 +39,29 @@ trait ObjectsApi { /// Options for specifying the content to be returned options: Option, ) -> RpcResult; + + /// Return the object information for multiple specified objects and versions. + /// + /// Note that past versions of an object may be pruned from the system, even if they once + /// existed. Different RPC services may return different responses for the same request as a + /// result, based on their pruning policies. + #[method(name = "tryMultiGetPastObjects")] + async fn try_multi_get_past_objects( + &self, + /// A vector of object and versions to be queried + past_objects: Vec, + /// Options for specifying the content to be returned + options: Option, + ) -> RpcResult>; } -pub(crate) struct Objects(pub Context); +pub(crate) struct Objects(pub Context, pub ObjectsConfig); + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ObjectsConfig { + /// The maximum number of keys that can be queried in a single multi-get request. + pub max_multi_get_objects: usize, +} #[async_trait::async_trait] impl ObjectsApiServer for Objects { @@ -43,10 +71,42 @@ impl ObjectsApiServer for Objects { version: SequenceNumber, options: Option, ) -> RpcResult { - let Self(ctx) = self; + let Self(ctx, _) = self; let options = options.unwrap_or_default(); Ok(response::past_object(ctx, object_id, version, &options).await?) } + + async fn try_multi_get_past_objects( + &self, + past_objects: Vec, + options: Option, + ) -> RpcResult> { + let Self(ctx, config) = self; + if past_objects.len() > config.max_multi_get_objects { + return Err(invalid_params(Error::TooManyKeys { + requested: past_objects.len(), + max: config.max_multi_get_objects, + }) + .into()); + } + + let options = options.unwrap_or_default(); + + let obj_futures = past_objects + .iter() + .map(|obj| response::past_object(ctx, obj.object_id, obj.version, &options)); + + Ok(future::join_all(obj_futures) + .await + .into_iter() + .zip(past_objects) + .map(|(r, o)| { + let id = o.object_id.to_canonical_display(/* with_prefix */ true); + let v = o.version; + r.with_internal_context(|| format!("Failed to get object {id} at version {v}")) + }) + .collect::, _>>()?) + } } impl RpcModule for Objects { @@ -58,3 +118,11 @@ impl RpcModule for Objects { self.into_rpc() } } + +impl Default for ObjectsConfig { + fn default() -> Self { + Self { + max_multi_get_objects: 50, + } + } +} diff --git a/crates/sui-indexer-alt-jsonrpc/src/config.rs b/crates/sui-indexer-alt-jsonrpc/src/config.rs index d689f4a5a3445..ede7dedf4691e 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/config.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/config.rs @@ -6,11 +6,14 @@ use std::mem; use sui_default_config::DefaultConfig; use tracing::warn; -use crate::api::transactions::TransactionsConfig; +use crate::api::{objects::ObjectsConfig, transactions::TransactionsConfig}; #[DefaultConfig] #[derive(Clone, Default, Debug)] pub struct RpcConfig { + /// Configuration for object-related RPC methods. + pub objects: ObjectsLayer, + /// Configuration for transaction-related RPC methods. pub transactions: TransactionsLayer, @@ -18,6 +21,15 @@ pub struct RpcConfig { pub extra: toml::Table, } +#[DefaultConfig] +#[derive(Clone, Default, Debug)] +pub struct ObjectsLayer { + pub max_multi_get_objects: Option, + + #[serde(flatten)] + pub extra: toml::Table, +} + #[DefaultConfig] #[derive(Clone, Default, Debug)] pub struct TransactionsLayer { @@ -33,6 +45,7 @@ impl RpcConfig { /// configure. pub fn example() -> Self { Self { + objects: ObjectsConfig::default().into(), transactions: TransactionsConfig::default().into(), extra: Default::default(), } @@ -44,6 +57,17 @@ impl RpcConfig { } } +impl ObjectsLayer { + pub fn finish(self, base: ObjectsConfig) -> ObjectsConfig { + check_extra("objects", self.extra); + ObjectsConfig { + max_multi_get_objects: self + .max_multi_get_objects + .unwrap_or(base.max_multi_get_objects), + } + } +} + impl TransactionsLayer { pub fn finish(self, base: TransactionsConfig) -> TransactionsConfig { check_extra("transactions", self.extra); @@ -54,6 +78,15 @@ impl TransactionsLayer { } } +impl From for ObjectsLayer { + fn from(config: ObjectsConfig) -> Self { + Self { + max_multi_get_objects: Some(config.max_multi_get_objects), + extra: Default::default(), + } + } +} + impl From for TransactionsLayer { fn from(config: TransactionsConfig) -> Self { Self { diff --git a/crates/sui-indexer-alt-jsonrpc/src/lib.rs b/crates/sui-indexer-alt-jsonrpc/src/lib.rs index 6f2fb8ab8e775..5a4a24fdf316e 100644 --- a/crates/sui-indexer-alt-jsonrpc/src/lib.rs +++ b/crates/sui-indexer-alt-jsonrpc/src/lib.rs @@ -5,7 +5,7 @@ use std::net::SocketAddr; use std::sync::Arc; use anyhow::Context as _; -use api::objects::Objects; +use api::objects::{Objects, ObjectsConfig}; use api::rpc_module::RpcModule; use api::transactions::{QueryTransactions, Transactions, TransactionsConfig}; use config::RpcConfig; @@ -204,10 +204,12 @@ pub async fn start_rpc( cancel: CancellationToken, ) -> anyhow::Result> { let RpcConfig { + objects, transactions, extra: _, } = rpc_config.finish(); + let objects_config = objects.finish(ObjectsConfig::default()); let transactions_config = transactions.finish(TransactionsConfig::default()); let mut rpc = RpcService::new(rpc_args, registry, cancel.child_token()) @@ -222,7 +224,7 @@ pub async fn start_rpc( ); rpc.add_module(Governance(context.clone()))?; - rpc.add_module(Objects(context.clone()))?; + rpc.add_module(Objects(context.clone(), objects_config))?; rpc.add_module(QueryTransactions(context.clone(), transactions_config))?; rpc.add_module(Transactions(context.clone()))?; From b0e76cc60fd80f72854908183f8274db3ff94f8d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Thu, 30 Jan 2025 09:15:51 -0600 Subject: [PATCH 30/30] jsonrpc: unlimit the number of concurrent jsonrpc requests The jsonrpsee library is artificially limiting the number of concurrent requests that the server can process to ~100 since its treating each request as a separate connection. In order to work around this we can just set the maximum number of connections for the jsonrpsee service to a high number since its not actually controlling the number of connections. --- crates/sui-json-rpc/src/lib.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/sui-json-rpc/src/lib.rs b/crates/sui-json-rpc/src/lib.rs index 977151fd548ba..0555e50492cfb 100644 --- a/crates/sui-json-rpc/src/lib.rs +++ b/crates/sui-json-rpc/src/lib.rs @@ -206,8 +206,13 @@ impl JsonRpcServerBuilder { let rpc_middleware = jsonrpsee::server::middleware::rpc::RpcServiceBuilder::new() .layer_fn(move |s| MetricsLayer::new(s, metrics.clone())) .layer_fn(move |s| TrafficControllerService::new(s, traffic_controller.clone())); - let service_builder = - jsonrpsee::server::ServerBuilder::new().set_rpc_middleware(rpc_middleware); + let service_builder = jsonrpsee::server::ServerBuilder::new() + // Since we're not using jsonrpsee's server to actually handle connections this value + // is instead limiting the number of concurrent requests and has no impact on the + // number of connections. As such, for now we can just set this to a very high value to + // disable it artificially limiting us to ~100 conncurrent requests. + .max_connections(u32::MAX) + .set_rpc_middleware(rpc_middleware); let mut router = axum::Router::new(); match server_type {