From efb125159f442716411d935decead4c8c4e372fe Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Wed, 12 Jun 2024 12:12:26 +0200 Subject: [PATCH 1/3] Add documentation generation for datafusion tables. To test it: `cargo xtask generate-table-docs` Unrelated fix to some dependencies that broke my build locally. Use Rust doc for documenting DF table schemas --- Cargo.lock | 5 +- crates/invoker-api/Cargo.toml | 3 + crates/storage-query-datafusion/Cargo.toml | 1 + .../src/deployment/mod.rs | 2 +- .../src/deployment/schema.rs | 7 ++ .../src/idempotency/mod.rs | 2 +- .../src/idempotency/schema.rs | 9 +++ .../storage-query-datafusion/src/inbox/mod.rs | 2 +- .../src/inbox/schema.rs | 7 ++ .../src/invocation_state/mod.rs | 2 +- .../src/invocation_state/schema.rs | 27 ++++++++ .../src/invocation_status/mod.rs | 2 +- .../src/invocation_status/schema.rs | 37 ++++++++++ .../src/journal/mod.rs | 2 +- .../src/journal/schema.rs | 18 +++++ .../src/keyed_service_status/mod.rs | 2 +- .../src/keyed_service_status/schema.rs | 5 ++ crates/storage-query-datafusion/src/lib.rs | 68 +++++++++++++++++++ .../src/promise/mod.rs | 2 +- .../src/promise/schema.rs | 14 +++- .../src/service/mod.rs | 2 +- .../src/service/schema.rs | 7 ++ .../storage-query-datafusion/src/state/mod.rs | 2 +- .../src/state/schema.rs | 11 +++ .../src/table_macro.rs | 53 ++++++++++++++- tools/xtask/Cargo.toml | 1 + tools/xtask/src/main.rs | 45 +++++++++++- 27 files changed, 322 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42c02d2ac5..4d76988dd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4906,9 +4906,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b33eb56c327dec362a9e55b3ad14f9d2f0904fb5a5b03b513ab5465399e9f43" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -8506,6 +8506,7 @@ dependencies = [ "restate-schema-api", "restate-service-client", "restate-service-protocol", + "restate-storage-query-datafusion", "restate-types", "restate-worker", "schemars", diff --git a/crates/invoker-api/Cargo.toml b/crates/invoker-api/Cargo.toml index cf9484d9bb..ccc80d2fac 100644 --- a/crates/invoker-api/Cargo.toml +++ b/crates/invoker-api/Cargo.toml @@ -24,3 +24,6 @@ futures = { workspace = true } serde = { workspace = true, optional = true } thiserror = { workspace = true } tokio = { workspace = true } + +[dev-dependencies] +restate-types = { workspace = true, features = ["test-util"] } \ No newline at end of file diff --git a/crates/storage-query-datafusion/Cargo.toml b/crates/storage-query-datafusion/Cargo.toml index 6258967753..a4f363757f 100644 --- a/crates/storage-query-datafusion/Cargo.toml +++ b/crates/storage-query-datafusion/Cargo.toml @@ -10,6 +10,7 @@ publish = false [features] default = [] options_schema = ["dep:schemars"] +table_docs = [] [dependencies] restate-core = { workspace = true } diff --git a/crates/storage-query-datafusion/src/deployment/mod.rs b/crates/storage-query-datafusion/src/deployment/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/deployment/mod.rs +++ b/crates/storage-query-datafusion/src/deployment/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/deployment/schema.rs b/crates/storage-query-datafusion/src/deployment/schema.rs index 5acd3b9421..c77ce7b109 100644 --- a/crates/storage-query-datafusion/src/deployment/schema.rs +++ b/crates/storage-query-datafusion/src/deployment/schema.rs @@ -15,8 +15,15 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(deployment( + /// The ID of the service deployment. id: DataType::LargeUtf8, + + /// The type of the endpoint. Either `http` or `lambda`. ty: DataType::LargeUtf8, + + /// The address of the endpoint. Either HTTP URL or Lambda ARN. endpoint: DataType::LargeUtf8, + + /// Timestamp indicating the deployment registration time. created_at: DataType::Date64, )); diff --git a/crates/storage-query-datafusion/src/idempotency/mod.rs b/crates/storage-query-datafusion/src/idempotency/mod.rs index f614fc33fe..0e30a3af93 100644 --- a/crates/storage-query-datafusion/src/idempotency/mod.rs +++ b/crates/storage-query-datafusion/src/idempotency/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/idempotency/schema.rs b/crates/storage-query-datafusion/src/idempotency/schema.rs index cebc959ef3..608d4d8f8c 100644 --- a/crates/storage-query-datafusion/src/idempotency/schema.rs +++ b/crates/storage-query-datafusion/src/idempotency/schema.rs @@ -15,12 +15,21 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(idempotency( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + /// The name for the invoked service. service_name: DataType::LargeUtf8, + + /// The key of the Virtual Object or of the Workflow. Null for regular services. service_key: DataType::LargeUtf8, + + /// The invoked handler. service_handler: DataType::LargeUtf8, + + /// The user provided idempotency key. idempotency_key: DataType::LargeUtf8, + /// [Invocation ID](/operate/invocation#invocation-identifier). invocation_id: DataType::LargeUtf8 )); diff --git a/crates/storage-query-datafusion/src/inbox/mod.rs b/crates/storage-query-datafusion/src/inbox/mod.rs index f614fc33fe..0e30a3af93 100644 --- a/crates/storage-query-datafusion/src/inbox/mod.rs +++ b/crates/storage-query-datafusion/src/inbox/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/inbox/schema.rs b/crates/storage-query-datafusion/src/inbox/schema.rs index 1008139be2..753731b56c 100644 --- a/crates/storage-query-datafusion/src/inbox/schema.rs +++ b/crates/storage-query-datafusion/src/inbox/schema.rs @@ -15,14 +15,21 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(inbox( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + /// The name for the invoked virtual object/workflow. service_name: DataType::LargeUtf8, + + /// The key of the virtual object/workflow. service_key: DataType::LargeUtf8, + /// [Invocation ID](/operate/invocation#invocation-identifier). id: DataType::LargeUtf8, + /// Sequence number in the inbox. sequence_number: DataType::UInt64, + /// Timestamp indicating the start of this invocation. created_at: DataType::Date64, )); diff --git a/crates/storage-query-datafusion/src/invocation_state/mod.rs b/crates/storage-query-datafusion/src/invocation_state/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/invocation_state/mod.rs +++ b/crates/storage-query-datafusion/src/invocation_state/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/invocation_state/schema.rs b/crates/storage-query-datafusion/src/invocation_state/schema.rs index 17785d9e46..362ab49438 100644 --- a/crates/storage-query-datafusion/src/invocation_state/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_state/schema.rs @@ -15,23 +15,50 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(state( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + + /// [Invocation ID](/operate/invocation#invocation-identifier). id: DataType::LargeUtf8, + /// If true, the invocation is currently in-flight in_flight: DataType::Boolean, + + /// The number of attempts since the last successful attempt of this invocation. + /// Increments on start, so 2 or more means a failure occurred. retry_count: DataType::UInt64, + + /// Timestamp indicating the start of the most recent attempt of this invocation. last_start_at: DataType::Date64, // The deployment that was selected in the last invocation attempt. This is // guaranteed to be set unlike in `sys_status` table which require that the // deployment to be committed before it is set. + + /// The opaque service deployment ID that was used in the most recent attempt of this + /// invocation; this will be set before a journal entry is stored, but can change later. last_attempt_deployment_id: DataType::LargeUtf8, + + /// Server/SDK version, e.g. `restate-sdk-java/1.0.1` last_attempt_server: DataType::LargeUtf8, + + /// Timestamp indicating the start of the next attempt of this invocation. next_retry_at: DataType::Date64, + /// An error message describing the most recent failed attempt of this invocation, if any. last_failure: DataType::LargeUtf8, + + /// The error code of the most recent failed attempt of this invocation, if any. last_failure_error_code: DataType::LargeUtf8, + + /// The index of the journal entry that caused the failure, if any. It may be out-of-bound + /// of the currently stored entries in `sys_journal`. last_failure_related_entry_index: DataType::UInt64, + + /// The name of the journal entry that caused the failure, if any. last_failure_related_entry_name: DataType::LargeUtf8, + + /// The type of the journal entry that caused the failure, if any. You can check all the + /// available entry types in [`entries.rs`](https://github.com/restatedev/restate/blob/main/crates/types/src/journal/entries.rs). last_failure_related_entry_type: DataType::LargeUtf8, )); diff --git a/crates/storage-query-datafusion/src/invocation_status/mod.rs b/crates/storage-query-datafusion/src/invocation_status/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/invocation_status/mod.rs +++ b/crates/storage-query-datafusion/src/invocation_status/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/invocation_status/schema.rs b/crates/storage-query-datafusion/src/invocation_status/schema.rs index 768368b280..01ea6cfafd 100644 --- a/crates/storage-query-datafusion/src/invocation_status/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_status/schema.rs @@ -15,25 +15,62 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(invocation_status( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + + /// [Invocation ID](/operate/invocation#invocation-identifier). id: DataType::LargeUtf8, + /// Either `inboxed` or `invoked` or `suspended` or `completed` status: DataType::LargeUtf8, + /// Invocation Target. Format for plain services: `ServiceName/HandlerName`, e.g. + /// `Greeter/greet`. Format for Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, + /// e.g. `Greeter/Francesco/greet`. target: DataType::LargeUtf8, + + /// The name for the invoked service. target_service_name: DataType::LargeUtf8, + + /// The key of the Virtual Object or of the Workflow. Null for regular services. target_service_key: DataType::LargeUtf8, + + /// The invoked handler. target_handler_name: DataType::LargeUtf8, + + /// The service type. Either `service` or `virtual_object` or `workflow`. target_service_ty: DataType::LargeUtf8, + /// Either `ingress` if the service was invoked externally or `service` if the service was + /// invoked by another Restate service. invoked_by: DataType::LargeUtf8, + + /// The name of the invoking service. Or `null` if invoked externally. invoked_by_service_name: DataType::LargeUtf8, + + /// The caller [Invocation ID](/operate/invocation#invocation-identifier) if the service was + /// invoked by another Restate service. Or `null` if invoked externally. invoked_by_id: DataType::LargeUtf8, + + /// The caller invocation target if the service was invoked by another Restate service. Or + /// `null` if invoked externally. invoked_by_target: DataType::LargeUtf8, + /// The opaque service deployment ID that has been committed for this invocation; this is set + /// after the first journal entry is stored for this invocation. pinned_deployment_id: DataType::LargeUtf8, + + /// The ID of the trace that is assigned to this invocation. Only relevant when tracing is + /// enabled. trace_id: DataType::LargeUtf8, + + /// The number of journal entries durably logged for this invocation. journal_size: DataType::UInt32, + + /// Timestamp indicating the start of this invocation. created_at: DataType::Date64, + + /// Timestamp indicating the last state transition. For example, last time the status changed + /// from `invoked` to `suspended`. modified_at: DataType::Date64, )); diff --git a/crates/storage-query-datafusion/src/journal/mod.rs b/crates/storage-query-datafusion/src/journal/mod.rs index f614fc33fe..0e30a3af93 100644 --- a/crates/storage-query-datafusion/src/journal/mod.rs +++ b/crates/storage-query-datafusion/src/journal/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/journal/schema.rs b/crates/storage-query-datafusion/src/journal/schema.rs index 11eed2d88e..05e05bc5b4 100644 --- a/crates/storage-query-datafusion/src/journal/schema.rs +++ b/crates/storage-query-datafusion/src/journal/schema.rs @@ -15,19 +15,37 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(journal( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + + /// [Invocation ID](/operate/invocation#invocation-identifier). id: DataType::LargeUtf8, + /// The index of this journal entry. index: DataType::UInt32, + + /// The entry type. You can check all the available entry types in [`entries.rs`](https://github.com/restatedev/restate/blob/main/crates/types/src/journal/entries.rs). entry_type: DataType::LargeUtf8, + + /// The name of the entry supplied by the user, if any. name: DataType::LargeUtf8, + /// Indicates whether this journal entry has been completed; this is only valid for some entry + /// types. completed: DataType::Boolean, + /// If this entry represents an outbound invocation, indicates the ID of that invocation. invoked_id: DataType::LargeUtf8, + + /// If this entry represents an outbound invocation, indicates the invocation Target. Format + /// for plain services: `ServiceName/HandlerName`, e.g. `Greeter/greet`. Format for + /// Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, e.g. `Greeter/Francesco/greet`. invoked_target: DataType::LargeUtf8, + /// If this entry represents a sleep, indicates wakeup time. sleep_wakeup_at: DataType::Date64, + /// Raw binary representation of the entry. Check the [service protocol](https://github.com/restatedev/service-protocol) + /// for more details to decode it. raw: DataType::LargeBinary, )); diff --git a/crates/storage-query-datafusion/src/keyed_service_status/mod.rs b/crates/storage-query-datafusion/src/keyed_service_status/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/mod.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs index 42c344752e..a8ce8ff06b 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs @@ -15,10 +15,15 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(keyed_service_status( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + /// The name for the invoked virtual object/workflow. service_name: DataType::LargeUtf8, + + /// The key of the virtual object/workflow. service_key: DataType::LargeUtf8, + /// [Invocation ID](/operate/invocation#invocation-identifier). invocation_id: DataType::LargeUtf8, )); diff --git a/crates/storage-query-datafusion/src/lib.rs b/crates/storage-query-datafusion/src/lib.rs index ccd710ad1f..51ef1782ce 100644 --- a/crates/storage-query-datafusion/src/lib.rs +++ b/crates/storage-query-datafusion/src/lib.rs @@ -28,6 +28,74 @@ mod table_util; pub use context::BuildError; +#[cfg(feature = "table_docs")] +pub fn table_docs(table: &str) -> Vec<(&str, &str, &str)> { + match table { + "sys_journal" => Vec::from(journal::schema::TABLE_DOCS), + "state" => Vec::from(state::schema::TABLE_DOCS), + "sys_keyed_service_status" => Vec::from(keyed_service_status::schema::TABLE_DOCS), + "sys_inbox" => Vec::from(inbox::schema::TABLE_DOCS), + "sys_deployment" => Vec::from(deployment::schema::TABLE_DOCS), + "sys_service" => Vec::from(service::schema::TABLE_DOCS), + "sys_idempotency" => Vec::from(idempotency::schema::TABLE_DOCS), + "sys_promise" => Vec::from(promise::schema::TABLE_DOCS), + "sys_invocation" => { + // We need to compile this manually, due to the fact that it's a view. + use std::collections::HashMap; + let mut sys_invocation_state: HashMap<&'static str, (&'static str, &'static str)> = + invocation_state::schema::TABLE_DOCS + .iter() + .map(|(name, ty, desc)| (*name, (*ty, *desc))) + .collect(); + let mut sys_invocation_status: HashMap<&'static str, (&'static str, &'static str)> = + invocation_status::schema::TABLE_DOCS + .iter() + .map(|(name, ty, desc)| (*name, (*ty, *desc))) + .collect(); + + fn copy_desc<'a>( + map: &mut HashMap<&'a str, (&'a str, &'a str)>, + name: &str, + ) -> (&'a str, &'a str, &'a str) { + let (name, (ty, desc)) = map.remove_entry(name).unwrap(); + (name, ty, desc) + } + + vec![ + copy_desc(&mut sys_invocation_status, "id"), + copy_desc(&mut sys_invocation_status, "target"), + copy_desc(&mut sys_invocation_status, "target_service_name"), + copy_desc(&mut sys_invocation_status, "target_service_key"), + copy_desc(&mut sys_invocation_status, "target_handler_name"), + copy_desc(&mut sys_invocation_status, "target_service_ty"), + copy_desc(&mut sys_invocation_status, "invoked_by"), + copy_desc(&mut sys_invocation_status, "invoked_by_service_name"), + copy_desc(&mut sys_invocation_status, "invoked_by_id"), + copy_desc(&mut sys_invocation_status, "invoked_by_target"), + copy_desc(&mut sys_invocation_status, "pinned_deployment_id"), + copy_desc(&mut sys_invocation_status, "trace_id"), + copy_desc(&mut sys_invocation_status, "journal_size"), + copy_desc(&mut sys_invocation_status, "created_at"), + copy_desc(&mut sys_invocation_status, "modified_at"), + + copy_desc(&mut sys_invocation_state, "retry_count"), + copy_desc(&mut sys_invocation_state, "last_start_at"), + copy_desc(&mut sys_invocation_state, "next_retry_at"), + copy_desc(&mut sys_invocation_state, "last_attempt_deployment_id"), + copy_desc(&mut sys_invocation_state, "last_attempt_server"), + copy_desc(&mut sys_invocation_state, "last_failure"), + copy_desc(&mut sys_invocation_state, "last_failure_error_code"), + copy_desc(&mut sys_invocation_state, "last_failure_related_entry_index"), + copy_desc(&mut sys_invocation_state, "last_failure_related_entry_name"), + copy_desc(&mut sys_invocation_state, "last_failure_related_entry_type"), + + ("status", "Utf8", "Either `pending` or `ready` or `running` or `backing-off` or `suspended` or `completed`.") + ] + } + _ => panic!("Unknown table '{table}'"), + } +} + #[cfg(test)] pub(crate) mod mocks; diff --git a/crates/storage-query-datafusion/src/promise/mod.rs b/crates/storage-query-datafusion/src/promise/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/promise/mod.rs +++ b/crates/storage-query-datafusion/src/promise/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/promise/schema.rs b/crates/storage-query-datafusion/src/promise/schema.rs index 17cf0f377d..26179ce073 100644 --- a/crates/storage-query-datafusion/src/promise/schema.rs +++ b/crates/storage-query-datafusion/src/promise/schema.rs @@ -15,15 +15,27 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(promise( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + /// The name for the workflow. service_name: DataType::LargeUtf8, + + /// The key of the virtual workflow. service_key: DataType::LargeUtf8, + /// The promise key. key: DataType::LargeUtf8, + + /// True if the promise was completed. completed: DataType::Boolean, + /// The completion success, if any. completion_success_value: DataType::LargeBinary, + + /// The completion success as UTF-8 string, if any. completion_success_value_utf8: DataType::LargeUtf8, - completion_failure: DataType::LargeUtf8 + + /// The completion failure, if any. + completion_failure: DataType::LargeUtf8, )); diff --git a/crates/storage-query-datafusion/src/service/mod.rs b/crates/storage-query-datafusion/src/service/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/service/mod.rs +++ b/crates/storage-query-datafusion/src/service/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/service/schema.rs b/crates/storage-query-datafusion/src/service/schema.rs index 51815c5bab..0d48bd74ef 100644 --- a/crates/storage-query-datafusion/src/service/schema.rs +++ b/crates/storage-query-datafusion/src/service/schema.rs @@ -15,11 +15,18 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(service( + /// The name of the registered user service. name: DataType::LargeUtf8, + + /// The latest deployed revision. revision: DataType::UInt64, + /// Whether the service is accessible through the ingress endpoint or not. public: DataType::Boolean, + /// The service type. Either `service` or `virtual_object` or `workflow`. ty: DataType::LargeUtf8, + + /// The ID of the latest deployment deployment_id: DataType::LargeUtf8, )); diff --git a/crates/storage-query-datafusion/src/state/mod.rs b/crates/storage-query-datafusion/src/state/mod.rs index dca1c728e5..a80e200a5b 100644 --- a/crates/storage-query-datafusion/src/state/mod.rs +++ b/crates/storage-query-datafusion/src/state/mod.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. mod row; -mod schema; +pub(crate) mod schema; mod table; pub(crate) use table::register_self; diff --git a/crates/storage-query-datafusion/src/state/schema.rs b/crates/storage-query-datafusion/src/state/schema.rs index d8a9cde33a..c7e7e11c37 100644 --- a/crates/storage-query-datafusion/src/state/schema.rs +++ b/crates/storage-query-datafusion/src/state/schema.rs @@ -15,12 +15,23 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; define_table!(state( + /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, + /// The name of the invoked service. service_name: DataType::LargeUtf8, + + /// The key of the Virtual Object. service_key: DataType::LargeUtf8, + /// The `utf8` state key. key: DataType::LargeUtf8, + + /// Only contains meaningful values when a service stores state as `utf8`. This is the case for + /// TypeScript services since the TypeScript SDK serializes values as JSON. value_utf8: DataType::LargeUtf8, + + /// A binary, uninterpreted representation of the value. You can use the more specific column + /// `value_utf8` if the value is a string. value: DataType::LargeBinary, )); diff --git a/crates/storage-query-datafusion/src/table_macro.rs b/crates/storage-query-datafusion/src/table_macro.rs index a9e494b232..fb0831e951 100644 --- a/crates/storage-query-datafusion/src/table_macro.rs +++ b/crates/storage-query-datafusion/src/table_macro.rs @@ -68,6 +68,36 @@ macro_rules! define_primitive_trait { }; } +macro_rules! document_type { + (DataType::Utf8) => { + "Utf8" + }; + (DataType::LargeUtf8) => { + "Utf8" + }; + (DataType::Binary) => { + "Binary" + }; + (DataType::LargeBinary) => { + "Binary" + }; + (DataType::UInt32) => { + "UInt32" + }; + (DataType::UInt64) => { + "UInt64" + }; + (DataType::Int32) => { + "Int32" + }; + (DataType::Date64) => { + "Date64" + }; + (DataType::Boolean) => { + "Boolean" + }; +} + /// /// Given the following table definition: /// @@ -284,7 +314,12 @@ macro_rules! define_primitive_trait { /// And it can be used to create RecordBatches from rows. macro_rules! define_table { - ($table_name: ident ($($element: ident : $ty: expr),+ $(,)? ) ) => (paste::paste! { + ($table_name: ident ( + $( + $(#[doc = $doc:expr])* + $element:ident: $ty:expr + ),+ $(,)?) + ) => (paste::paste! { pub struct [< $table_name:camel Builder >] { rows_inserted_so_far: usize, @@ -293,7 +328,10 @@ macro_rules! define_table { } struct [< $table_name:camel ArrayBuilder >] { - $($element : Option< define_builder!($ty) > ,)+ + $( + $(#[doc = $doc])* + $element : Option< define_builder!($ty) > , + )+ } pub struct [< $table_name:camel RowBuilder >]<'a> { @@ -438,9 +476,20 @@ macro_rules! define_table { } } + // -------------------------------------------------------------------------- + // Docs function + // -------------------------------------------------------------------------- + + #[cfg(feature = "table_docs")] + pub const TABLE_DOCS: &'static [(&'static str, &'static str, &'static str)] = &[ + $( + (stringify!($element), document_type!($ty), concat!($($doc),*)), + )+ + ]; }) } pub(crate) use define_builder; pub(crate) use define_primitive_trait; pub(crate) use define_table; +pub(crate) use document_type; diff --git a/tools/xtask/Cargo.toml b/tools/xtask/Cargo.toml index 89205e5b2c..203f7cf2a1 100644 --- a/tools/xtask/Cargo.toml +++ b/tools/xtask/Cargo.toml @@ -16,6 +16,7 @@ restate-node-services = { workspace = true, features = ["clients"] } restate-schema-api = { workspace = true, features = ["subscription"] } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = ["discovery"]} +restate-storage-query-datafusion = { workspace = true, features = ["table_docs"] } restate-types = { workspace = true, features = ["schemars"] } restate-worker = { workspace = true, features = ["options_schema"] } diff --git a/tools/xtask/src/main.rs b/tools/xtask/src/main.rs index 16889a19d6..4604c9b751 100644 --- a/tools/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -28,8 +28,9 @@ use restate_worker::SubscriptionController; use restate_worker::WorkerHandle; use restate_worker::WorkerHandleError; use schemars::gen::SchemaSettings; -use std::env; +use std::io::Write; use std::time::Duration; +use std::{env, io}; use tonic::transport::{Channel, Uri}; fn generate_config_schema() -> anyhow::Result<()> { @@ -148,6 +149,46 @@ async fn generate_rest_api_doc() -> anyhow::Result<()> { Ok(()) } +fn render_table_docs(table: &str, mut w: impl io::Write) -> io::Result<()> { + let columns = restate_storage_query_datafusion::table_docs(table); + + writeln!(&mut w, "## Table: `{table}`\n")?; + writeln!(&mut w, "| Column name | Type | Description |")?; + writeln!(&mut w, "|-------------|------|-------------|")?; + for (name, ty, desc) in columns { + writeln!(&mut w, "| `{name}` | `{ty}` | {} |", desc.trim())?; + } + writeln!(&mut w)?; + + Ok(()) +} + +fn generate_table_docs() -> anyhow::Result<()> { + let mut dest = io::stdout(); + + // File header + writeln!( + &mut dest, + r"# SQL Introspection API + +This page contains the reference of the introspection tables. +To learn how to access the instrospection interface, check out the [instrospection documentation](/operate/introspection). +" + )?; + + render_table_docs("state", &mut dest)?; + render_table_docs("sys_invocation", &mut dest)?; + render_table_docs("sys_journal", &mut dest)?; + render_table_docs("sys_keyed_service_status", &mut dest)?; + render_table_docs("sys_inbox", &mut dest)?; + render_table_docs("sys_idempotency", &mut dest)?; + render_table_docs("sys_promise", &mut dest)?; + render_table_docs("sys_service", &mut dest)?; + render_table_docs("sys_deployment", &mut dest)?; + + Ok(()) +} + fn print_help() { println!( " @@ -156,6 +197,7 @@ Tasks: generate-config-schema: Generate config schema for restate configuration. generate-default-config: Generate default configuration. generate-rest-api-doc: Generate Rest API documentation. Make sure to have the port 8081 open. + generate-table-docs: Generate default configuration. " ); } @@ -169,6 +211,7 @@ async fn main() -> anyhow::Result<()> { "generate-config-schema" => generate_config_schema()?, "generate-default-config" => generate_default_config(), "generate-rest-api-doc" => generate_rest_api_doc().await?, + "generate-table-docs" => generate_table_docs()?, invalid => { print_help(); bail!("Invalid task name: {}", invalid) From eb490265d056076be5533923ba8369e5b2fefab0 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Wed, 19 Jun 2024 17:30:19 +0200 Subject: [PATCH 2/3] Only require single place where to register table docs This commit changes the place where the table docs need to be registered to be included in the automatic table docs generation to a single place. --- .../src/deployment/row.rs | 4 +- .../src/deployment/schema.rs | 2 +- .../src/deployment/table.rs | 8 +- .../src/idempotency/row.rs | 4 +- .../src/idempotency/schema.rs | 2 +- .../src/idempotency/table.rs | 6 +- .../storage-query-datafusion/src/inbox/row.rs | 4 +- .../src/inbox/schema.rs | 2 +- .../src/inbox/table.rs | 6 +- .../src/invocation_state/row.rs | 6 +- .../src/invocation_state/schema.rs | 2 +- .../src/invocation_state/table.rs | 16 ++- .../src/invocation_status/row.rs | 12 +- .../src/invocation_status/schema.rs | 2 +- .../src/invocation_status/table.rs | 6 +- .../src/journal/row.rs | 4 +- .../src/journal/schema.rs | 2 +- .../src/journal/table.rs | 6 +- .../src/keyed_service_status/row.rs | 4 +- .../src/keyed_service_status/schema.rs | 2 +- .../src/keyed_service_status/table.rs | 6 +- crates/storage-query-datafusion/src/lib.rs | 70 +--------- .../src/promise/row.rs | 4 +- .../src/promise/schema.rs | 2 +- .../src/promise/table.rs | 6 +- .../src/service/row.rs | 4 +- .../src/service/schema.rs | 2 +- .../src/service/table.rs | 8 +- .../src/table_docs.rs | 131 ++++++++++++++++++ .../src/table_macro.rs | 17 ++- tools/xtask/src/main.rs | 42 +++--- 31 files changed, 237 insertions(+), 155 deletions(-) create mode 100644 crates/storage-query-datafusion/src/table_docs.rs diff --git a/crates/storage-query-datafusion/src/deployment/row.rs b/crates/storage-query-datafusion/src/deployment/row.rs index 21376ab2fe..623ec5226f 100644 --- a/crates/storage-query-datafusion/src/deployment/row.rs +++ b/crates/storage-query-datafusion/src/deployment/row.rs @@ -8,13 +8,13 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::schema::DeploymentBuilder; +use super::schema::SysDeploymentBuilder; use crate::table_util::format_using; use restate_schema_api::deployment::{Deployment, DeploymentType}; #[inline] pub(crate) fn append_deployment_row( - builder: &mut DeploymentBuilder, + builder: &mut SysDeploymentBuilder, output: &mut String, deployment: Deployment, ) { diff --git a/crates/storage-query-datafusion/src/deployment/schema.rs b/crates/storage-query-datafusion/src/deployment/schema.rs index c77ce7b109..1976bac4e9 100644 --- a/crates/storage-query-datafusion/src/deployment/schema.rs +++ b/crates/storage-query-datafusion/src/deployment/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(deployment( +define_table!(sys_deployment( /// The ID of the service deployment. id: DataType::LargeUtf8, diff --git a/crates/storage-query-datafusion/src/deployment/table.rs b/crates/storage-query-datafusion/src/deployment/table.rs index 147a6bb995..f66011d7fc 100644 --- a/crates/storage-query-datafusion/src/deployment/table.rs +++ b/crates/storage-query-datafusion/src/deployment/table.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc::Sender; use restate_schema_api::deployment::{Deployment, DeploymentResolver}; use restate_types::identifiers::ServiceRevision; -use super::schema::DeploymentBuilder; +use super::schema::SysDeploymentBuilder; use crate::context::QueryContext; use crate::deployment::row::append_deployment_row; use crate::table_providers::{GenericTableProvider, Scan}; @@ -32,7 +32,7 @@ pub(crate) fn register_self( resolver: impl DeploymentResolver + Send + Sync + Debug + 'static, ) -> datafusion::common::Result<()> { let deployment_table = GenericTableProvider::new( - DeploymentBuilder::schema(), + SysDeploymentBuilder::schema(), Arc::new(DeploymentMetadataScanner(resolver)), ); @@ -71,7 +71,7 @@ async fn for_each_state( tx: Sender>, rows: Vec<(Deployment, Vec<(String, ServiceRevision)>)>, ) { - let mut builder = DeploymentBuilder::new(schema.clone()); + let mut builder = SysDeploymentBuilder::new(schema.clone()); let mut temp = String::new(); for (deployment, _) in rows { append_deployment_row(&mut builder, &mut temp, deployment); @@ -83,7 +83,7 @@ async fn for_each_state( // we probably don't want to panic, is it will cause the entire process to exit return; } - builder = DeploymentBuilder::new(schema.clone()); + builder = SysDeploymentBuilder::new(schema.clone()); } } if !builder.empty() { diff --git a/crates/storage-query-datafusion/src/idempotency/row.rs b/crates/storage-query-datafusion/src/idempotency/row.rs index daed4d2e4c..c66f374be6 100644 --- a/crates/storage-query-datafusion/src/idempotency/row.rs +++ b/crates/storage-query-datafusion/src/idempotency/row.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::schema::IdempotencyBuilder; +use super::schema::SysIdempotencyBuilder; use crate::table_util::format_using; use restate_storage_api::idempotency_table::IdempotencyMetadata; @@ -16,7 +16,7 @@ use restate_types::identifiers::{IdempotencyId, WithPartitionKey}; #[inline] pub(crate) fn append_idempotency_row( - builder: &mut IdempotencyBuilder, + builder: &mut SysIdempotencyBuilder, output: &mut String, idempotency_id: IdempotencyId, idempotency_metadata: IdempotencyMetadata, diff --git a/crates/storage-query-datafusion/src/idempotency/schema.rs b/crates/storage-query-datafusion/src/idempotency/schema.rs index 608d4d8f8c..02b928707a 100644 --- a/crates/storage-query-datafusion/src/idempotency/schema.rs +++ b/crates/storage-query-datafusion/src/idempotency/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(idempotency( +define_table!(sys_idempotency( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/idempotency/table.rs b/crates/storage-query-datafusion/src/idempotency/table.rs index 2623606c45..530e8b2fbb 100644 --- a/crates/storage-query-datafusion/src/idempotency/table.rs +++ b/crates/storage-query-datafusion/src/idempotency/table.rs @@ -19,7 +19,7 @@ use restate_storage_api::idempotency_table::{IdempotencyMetadata, ReadOnlyIdempo use restate_types::identifiers::{IdempotencyId, PartitionKey}; use super::row::append_idempotency_row; -use super::schema::IdempotencyBuilder; +use super::schema::SysIdempotencyBuilder; use crate::context::{QueryContext, SelectPartitions}; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -31,7 +31,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let table = PartitionedTableProvider::new( partition_selector, - IdempotencyBuilder::schema(), + SysIdempotencyBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, IdempotencyScanner), ); @@ -44,7 +44,7 @@ pub(crate) fn register_self( struct IdempotencyScanner; impl ScanLocalPartition for IdempotencyScanner { - type Builder = IdempotencyBuilder; + type Builder = SysIdempotencyBuilder; type Item = (IdempotencyId, IdempotencyMetadata); fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/inbox/row.rs b/crates/storage-query-datafusion/src/inbox/row.rs index b60c1f5c5f..f7ed6d0e17 100644 --- a/crates/storage-query-datafusion/src/inbox/row.rs +++ b/crates/storage-query-datafusion/src/inbox/row.rs @@ -8,14 +8,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::schema::InboxBuilder; +use super::schema::SysInboxBuilder; use crate::table_util::format_using; use restate_storage_api::inbox_table::{InboxEntry, SequenceNumberInboxEntry}; use restate_types::identifiers::{TimestampAwareId, WithPartitionKey}; #[inline] pub(crate) fn append_inbox_row( - builder: &mut InboxBuilder, + builder: &mut SysInboxBuilder, output: &mut String, inbox_entry: SequenceNumberInboxEntry, ) { diff --git a/crates/storage-query-datafusion/src/inbox/schema.rs b/crates/storage-query-datafusion/src/inbox/schema.rs index 753731b56c..26df9fa9a7 100644 --- a/crates/storage-query-datafusion/src/inbox/schema.rs +++ b/crates/storage-query-datafusion/src/inbox/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(inbox( +define_table!(sys_inbox( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/inbox/table.rs b/crates/storage-query-datafusion/src/inbox/table.rs index f05731ead3..c518d1d1d2 100644 --- a/crates/storage-query-datafusion/src/inbox/table.rs +++ b/crates/storage-query-datafusion/src/inbox/table.rs @@ -20,7 +20,7 @@ use restate_types::identifiers::PartitionKey; use crate::context::{QueryContext, SelectPartitions}; use crate::inbox::row::append_inbox_row; -use crate::inbox::schema::InboxBuilder; +use crate::inbox::schema::SysInboxBuilder; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -31,7 +31,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let table = PartitionedTableProvider::new( partition_selector, - InboxBuilder::schema(), + SysInboxBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, InboxScanner), ); @@ -44,7 +44,7 @@ pub(crate) fn register_self( struct InboxScanner; impl ScanLocalPartition for InboxScanner { - type Builder = InboxBuilder; + type Builder = SysInboxBuilder; type Item = SequenceNumberInboxEntry; fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/invocation_state/row.rs b/crates/storage-query-datafusion/src/invocation_state/row.rs index 227d5b349b..fa0e2aeb57 100644 --- a/crates/storage-query-datafusion/src/invocation_state/row.rs +++ b/crates/storage-query-datafusion/src/invocation_state/row.rs @@ -8,15 +8,15 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::invocation_state::schema::StateBuilder; +use crate::invocation_state::schema::SysInvocationStateBuilder; use crate::table_util::format_using; use restate_invoker_api::InvocationStatusReport; use restate_types::identifiers::WithPartitionKey; use restate_types::time::MillisSinceEpoch; #[inline] -pub(crate) fn append_state_row( - builder: &mut StateBuilder, +pub(crate) fn append_invocation_state_row( + builder: &mut SysInvocationStateBuilder, output: &mut String, status_row: InvocationStatusReport, ) { diff --git a/crates/storage-query-datafusion/src/invocation_state/schema.rs b/crates/storage-query-datafusion/src/invocation_state/schema.rs index 362ab49438..f085221fa4 100644 --- a/crates/storage-query-datafusion/src/invocation_state/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_state/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(state( +define_table!(sys_invocation_state( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/invocation_state/table.rs b/crates/storage-query-datafusion/src/invocation_state/table.rs index 3011d3a45c..6a69a5f8e8 100644 --- a/crates/storage-query-datafusion/src/invocation_state/table.rs +++ b/crates/storage-query-datafusion/src/invocation_state/table.rs @@ -22,8 +22,8 @@ use restate_invoker_api::{InvocationStatusReport, StatusHandle}; use restate_types::identifiers::{PartitionKey, WithPartitionKey}; use crate::context::QueryContext; -use crate::invocation_state::row::append_state_row; -use crate::invocation_state::schema::StateBuilder; +use crate::invocation_state::row::append_invocation_state_row; +use crate::invocation_state::schema::SysInvocationStateBuilder; use crate::table_providers::{GenericTableProvider, Scan}; use crate::table_util::Builder; @@ -31,8 +31,10 @@ pub(crate) fn register_self( ctx: &QueryContext, status: impl StatusHandle + Send + Sync + Debug + Clone + 'static, ) -> datafusion::common::Result<()> { - let status_table = - GenericTableProvider::new(StateBuilder::schema(), Arc::new(StatusScanner(status))); + let status_table = GenericTableProvider::new( + SysInvocationStateBuilder::schema(), + Arc::new(StatusScanner(status)), + ); ctx.as_ref() .register_table("sys_invocation_state", Arc::new(status_table)) @@ -71,13 +73,13 @@ async fn for_each_state<'a, I>( ) where I: Iterator + 'a, { - let mut builder = StateBuilder::new(schema.clone()); + let mut builder = SysInvocationStateBuilder::new(schema.clone()); let mut temp = String::new(); let mut rows = rows.collect::>(); // need to be ordered by partition key for symmetric joins rows.sort_unstable_by_key(|row| row.invocation_id().partition_key()); for row in rows { - append_state_row(&mut builder, &mut temp, row); + append_invocation_state_row(&mut builder, &mut temp, row); if builder.full() { let batch = builder.finish(); if tx.send(batch).await.is_err() { @@ -86,7 +88,7 @@ async fn for_each_state<'a, I>( // we probably don't want to panic, is it will cause the entire process to exit return; } - builder = StateBuilder::new(schema.clone()); + builder = SysInvocationStateBuilder::new(schema.clone()); } } if !builder.empty() { diff --git a/crates/storage-query-datafusion/src/invocation_status/row.rs b/crates/storage-query-datafusion/src/invocation_status/row.rs index 7dece5dc60..d4f2c3a6c7 100644 --- a/crates/storage-query-datafusion/src/invocation_status/row.rs +++ b/crates/storage-query-datafusion/src/invocation_status/row.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::invocation_status::schema::{InvocationStatusBuilder, InvocationStatusRowBuilder}; +use crate::invocation_status::schema::{SysInvocationStatusBuilder, SysInvocationStatusRowBuilder}; use crate::table_util::format_using; use restate_storage_api::invocation_status_table::{ InFlightInvocationMetadata, InvocationStatus, JournalMetadata, StatusTimestamps, @@ -18,7 +18,7 @@ use restate_types::invocation::{ServiceType, Source, TraceId}; #[inline] pub(crate) fn append_invocation_status_row( - builder: &mut InvocationStatusBuilder, + builder: &mut SysInvocationStatusBuilder, output: &mut String, invocation_id: InvocationId, invocation_status: InvocationStatus, @@ -82,7 +82,7 @@ pub(crate) fn append_invocation_status_row( } fn fill_in_flight_invocation_metadata( - row: &mut InvocationStatusRowBuilder, + row: &mut SysInvocationStatusRowBuilder, output: &mut String, meta: InFlightInvocationMetadata, ) { @@ -94,7 +94,7 @@ fn fill_in_flight_invocation_metadata( } #[inline] -fn fill_invoked_by(row: &mut InvocationStatusRowBuilder, output: &mut String, source: Source) { +fn fill_invoked_by(row: &mut SysInvocationStatusRowBuilder, output: &mut String, source: Source) { match source { Source::Service(invocation_id, invocation_target) => { row.invoked_by("service"); @@ -116,14 +116,14 @@ fn fill_invoked_by(row: &mut InvocationStatusRowBuilder, output: &mut String, so } #[inline] -fn fill_timestamps(row: &mut InvocationStatusRowBuilder, stat: &StatusTimestamps) { +fn fill_timestamps(row: &mut SysInvocationStatusRowBuilder, stat: &StatusTimestamps) { row.created_at(stat.creation_time().as_u64() as i64); row.modified_at(stat.modification_time().as_u64() as i64); } #[inline] fn fill_journal_metadata( - row: &mut InvocationStatusRowBuilder, + row: &mut SysInvocationStatusRowBuilder, output: &mut String, journal_metadata: &JournalMetadata, ) { diff --git a/crates/storage-query-datafusion/src/invocation_status/schema.rs b/crates/storage-query-datafusion/src/invocation_status/schema.rs index 01ea6cfafd..1fdb2fdd1c 100644 --- a/crates/storage-query-datafusion/src/invocation_status/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_status/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(invocation_status( +define_table!(sys_invocation_status( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/invocation_status/table.rs b/crates/storage-query-datafusion/src/invocation_status/table.rs index f0bd6c9eb2..b1373d6f46 100644 --- a/crates/storage-query-datafusion/src/invocation_status/table.rs +++ b/crates/storage-query-datafusion/src/invocation_status/table.rs @@ -22,7 +22,7 @@ use restate_types::identifiers::{InvocationId, PartitionKey}; use crate::context::{QueryContext, SelectPartitions}; use crate::invocation_status::row::append_invocation_status_row; -use crate::invocation_status::schema::InvocationStatusBuilder; +use crate::invocation_status::schema::SysInvocationStatusBuilder; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -33,7 +33,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let status_table = PartitionedTableProvider::new( partition_selector, - InvocationStatusBuilder::schema(), + SysInvocationStatusBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, StatusScanner), ); @@ -46,7 +46,7 @@ pub(crate) fn register_self( struct StatusScanner; impl ScanLocalPartition for StatusScanner { - type Builder = InvocationStatusBuilder; + type Builder = SysInvocationStatusBuilder; type Item = (InvocationId, InvocationStatus); fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/journal/row.rs b/crates/storage-query-datafusion/src/journal/row.rs index 460a6cd9ca..91abe723a3 100644 --- a/crates/storage-query-datafusion/src/journal/row.rs +++ b/crates/storage-query-datafusion/src/journal/row.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::journal::schema::JournalBuilder; +use crate::journal::schema::SysJournalBuilder; use restate_service_protocol::codec::ProtobufRawEntryCodec; @@ -21,7 +21,7 @@ use restate_types::journal::{Entry, SleepEntry}; #[inline] pub(crate) fn append_journal_row( - builder: &mut JournalBuilder, + builder: &mut SysJournalBuilder, output: &mut String, journal_entry_id: JournalEntryId, journal_entry: JournalEntry, diff --git a/crates/storage-query-datafusion/src/journal/schema.rs b/crates/storage-query-datafusion/src/journal/schema.rs index 05e05bc5b4..ea3da170d6 100644 --- a/crates/storage-query-datafusion/src/journal/schema.rs +++ b/crates/storage-query-datafusion/src/journal/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(journal( +define_table!(sys_journal( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/journal/table.rs b/crates/storage-query-datafusion/src/journal/table.rs index 3f489628aa..7344b37c8e 100644 --- a/crates/storage-query-datafusion/src/journal/table.rs +++ b/crates/storage-query-datafusion/src/journal/table.rs @@ -19,7 +19,7 @@ use restate_types::identifiers::{JournalEntryId, PartitionKey}; use crate::context::{QueryContext, SelectPartitions}; use crate::journal::row::append_journal_row; -use crate::journal::schema::JournalBuilder; +use crate::journal::schema::SysJournalBuilder; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -30,7 +30,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let journal_table = PartitionedTableProvider::new( partition_selector, - JournalBuilder::schema(), + SysJournalBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, JournalScanner), ); @@ -43,7 +43,7 @@ pub(crate) fn register_self( struct JournalScanner; impl ScanLocalPartition for JournalScanner { - type Builder = JournalBuilder; + type Builder = SysJournalBuilder; type Item = (JournalEntryId, JournalEntry); fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/keyed_service_status/row.rs b/crates/storage-query-datafusion/src/keyed_service_status/row.rs index 55f28f0b9e..f8dfc5d1a8 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/row.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/row.rs @@ -8,14 +8,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::keyed_service_status::schema::KeyedServiceStatusBuilder; +use crate::keyed_service_status::schema::SysKeyedServiceStatusBuilder; use crate::table_util::format_using; use restate_storage_api::service_status_table::VirtualObjectStatus; use restate_types::identifiers::{ServiceId, WithPartitionKey}; #[inline] pub(crate) fn append_virtual_object_status_row( - builder: &mut KeyedServiceStatusBuilder, + builder: &mut SysKeyedServiceStatusBuilder, output: &mut String, service_id: ServiceId, status: VirtualObjectStatus, diff --git a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs index a8ce8ff06b..b4608faaf5 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(keyed_service_status( +define_table!(sys_keyed_service_status( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/keyed_service_status/table.rs b/crates/storage-query-datafusion/src/keyed_service_status/table.rs index eabf28068a..511e9e0d2a 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/table.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/table.rs @@ -22,7 +22,7 @@ use restate_types::identifiers::{PartitionKey, ServiceId}; use crate::context::{QueryContext, SelectPartitions}; use crate::keyed_service_status::row::append_virtual_object_status_row; -use crate::keyed_service_status::schema::KeyedServiceStatusBuilder; +use crate::keyed_service_status::schema::SysKeyedServiceStatusBuilder; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -33,7 +33,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let status_table = PartitionedTableProvider::new( partition_selector, - KeyedServiceStatusBuilder::schema(), + SysKeyedServiceStatusBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, VirtualObjectStatusScanner), ); @@ -46,7 +46,7 @@ pub(crate) fn register_self( struct VirtualObjectStatusScanner; impl ScanLocalPartition for VirtualObjectStatusScanner { - type Builder = KeyedServiceStatusBuilder; + type Builder = SysKeyedServiceStatusBuilder; type Item = (ServiceId, VirtualObjectStatus); fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/lib.rs b/crates/storage-query-datafusion/src/lib.rs index 51ef1782ce..23edfbdc19 100644 --- a/crates/storage-query-datafusion/src/lib.rs +++ b/crates/storage-query-datafusion/src/lib.rs @@ -22,80 +22,14 @@ mod physical_optimizer; mod promise; mod service; mod state; +#[cfg(feature = "table_docs")] +pub mod table_docs; mod table_macro; mod table_providers; mod table_util; pub use context::BuildError; -#[cfg(feature = "table_docs")] -pub fn table_docs(table: &str) -> Vec<(&str, &str, &str)> { - match table { - "sys_journal" => Vec::from(journal::schema::TABLE_DOCS), - "state" => Vec::from(state::schema::TABLE_DOCS), - "sys_keyed_service_status" => Vec::from(keyed_service_status::schema::TABLE_DOCS), - "sys_inbox" => Vec::from(inbox::schema::TABLE_DOCS), - "sys_deployment" => Vec::from(deployment::schema::TABLE_DOCS), - "sys_service" => Vec::from(service::schema::TABLE_DOCS), - "sys_idempotency" => Vec::from(idempotency::schema::TABLE_DOCS), - "sys_promise" => Vec::from(promise::schema::TABLE_DOCS), - "sys_invocation" => { - // We need to compile this manually, due to the fact that it's a view. - use std::collections::HashMap; - let mut sys_invocation_state: HashMap<&'static str, (&'static str, &'static str)> = - invocation_state::schema::TABLE_DOCS - .iter() - .map(|(name, ty, desc)| (*name, (*ty, *desc))) - .collect(); - let mut sys_invocation_status: HashMap<&'static str, (&'static str, &'static str)> = - invocation_status::schema::TABLE_DOCS - .iter() - .map(|(name, ty, desc)| (*name, (*ty, *desc))) - .collect(); - - fn copy_desc<'a>( - map: &mut HashMap<&'a str, (&'a str, &'a str)>, - name: &str, - ) -> (&'a str, &'a str, &'a str) { - let (name, (ty, desc)) = map.remove_entry(name).unwrap(); - (name, ty, desc) - } - - vec![ - copy_desc(&mut sys_invocation_status, "id"), - copy_desc(&mut sys_invocation_status, "target"), - copy_desc(&mut sys_invocation_status, "target_service_name"), - copy_desc(&mut sys_invocation_status, "target_service_key"), - copy_desc(&mut sys_invocation_status, "target_handler_name"), - copy_desc(&mut sys_invocation_status, "target_service_ty"), - copy_desc(&mut sys_invocation_status, "invoked_by"), - copy_desc(&mut sys_invocation_status, "invoked_by_service_name"), - copy_desc(&mut sys_invocation_status, "invoked_by_id"), - copy_desc(&mut sys_invocation_status, "invoked_by_target"), - copy_desc(&mut sys_invocation_status, "pinned_deployment_id"), - copy_desc(&mut sys_invocation_status, "trace_id"), - copy_desc(&mut sys_invocation_status, "journal_size"), - copy_desc(&mut sys_invocation_status, "created_at"), - copy_desc(&mut sys_invocation_status, "modified_at"), - - copy_desc(&mut sys_invocation_state, "retry_count"), - copy_desc(&mut sys_invocation_state, "last_start_at"), - copy_desc(&mut sys_invocation_state, "next_retry_at"), - copy_desc(&mut sys_invocation_state, "last_attempt_deployment_id"), - copy_desc(&mut sys_invocation_state, "last_attempt_server"), - copy_desc(&mut sys_invocation_state, "last_failure"), - copy_desc(&mut sys_invocation_state, "last_failure_error_code"), - copy_desc(&mut sys_invocation_state, "last_failure_related_entry_index"), - copy_desc(&mut sys_invocation_state, "last_failure_related_entry_name"), - copy_desc(&mut sys_invocation_state, "last_failure_related_entry_type"), - - ("status", "Utf8", "Either `pending` or `ready` or `running` or `backing-off` or `suspended` or `completed`.") - ] - } - _ => panic!("Unknown table '{table}'"), - } -} - #[cfg(test)] pub(crate) mod mocks; diff --git a/crates/storage-query-datafusion/src/promise/row.rs b/crates/storage-query-datafusion/src/promise/row.rs index 1a2d33bd9d..97fc7d4fcd 100644 --- a/crates/storage-query-datafusion/src/promise/row.rs +++ b/crates/storage-query-datafusion/src/promise/row.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::schema::PromiseBuilder; +use super::schema::SysPromiseBuilder; use crate::table_util::format_using; use restate_storage_api::promise_table::{OwnedPromiseRow, PromiseState}; @@ -18,7 +18,7 @@ use restate_types::journal::EntryResult; #[inline] pub(crate) fn append_promise_row( - builder: &mut PromiseBuilder, + builder: &mut SysPromiseBuilder, output: &mut String, owned_promise_row: OwnedPromiseRow, ) { diff --git a/crates/storage-query-datafusion/src/promise/schema.rs b/crates/storage-query-datafusion/src/promise/schema.rs index 26179ce073..b9af63f088 100644 --- a/crates/storage-query-datafusion/src/promise/schema.rs +++ b/crates/storage-query-datafusion/src/promise/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(promise( +define_table!(sys_promise( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, diff --git a/crates/storage-query-datafusion/src/promise/table.rs b/crates/storage-query-datafusion/src/promise/table.rs index cb7ef542ec..342c7ad5f0 100644 --- a/crates/storage-query-datafusion/src/promise/table.rs +++ b/crates/storage-query-datafusion/src/promise/table.rs @@ -19,7 +19,7 @@ use restate_storage_api::promise_table::{OwnedPromiseRow, ReadOnlyPromiseTable}; use restate_types::identifiers::PartitionKey; use super::row::append_promise_row; -use super::schema::PromiseBuilder; +use super::schema::SysPromiseBuilder; use crate::context::{QueryContext, SelectPartitions}; use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition}; use crate::table_providers::PartitionedTableProvider; @@ -31,7 +31,7 @@ pub(crate) fn register_self( ) -> datafusion::common::Result<()> { let table = PartitionedTableProvider::new( partition_selector, - PromiseBuilder::schema(), + SysPromiseBuilder::schema(), LocalPartitionsScanner::new(partition_store_manager, PromiseScanner), ); @@ -44,7 +44,7 @@ pub(crate) fn register_self( struct PromiseScanner; impl ScanLocalPartition for PromiseScanner { - type Builder = PromiseBuilder; + type Builder = SysPromiseBuilder; type Item = OwnedPromiseRow; fn scan_partition_store( diff --git a/crates/storage-query-datafusion/src/service/row.rs b/crates/storage-query-datafusion/src/service/row.rs index 9ef4a62c86..08d4928ba3 100644 --- a/crates/storage-query-datafusion/src/service/row.rs +++ b/crates/storage-query-datafusion/src/service/row.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::schema::ServiceBuilder; +use super::schema::SysServiceBuilder; use crate::table_util::format_using; use restate_schema_api::service::ServiceMetadata; @@ -16,7 +16,7 @@ use restate_types::invocation::ServiceType; #[inline] pub(crate) fn append_service_row( - builder: &mut ServiceBuilder, + builder: &mut SysServiceBuilder, output: &mut String, service_metadata: ServiceMetadata, ) { diff --git a/crates/storage-query-datafusion/src/service/schema.rs b/crates/storage-query-datafusion/src/service/schema.rs index 0d48bd74ef..5e2bafc98a 100644 --- a/crates/storage-query-datafusion/src/service/schema.rs +++ b/crates/storage-query-datafusion/src/service/schema.rs @@ -14,7 +14,7 @@ use crate::table_macro::*; use datafusion::arrow::datatypes::DataType; -define_table!(service( +define_table!(sys_service( /// The name of the registered user service. name: DataType::LargeUtf8, diff --git a/crates/storage-query-datafusion/src/service/table.rs b/crates/storage-query-datafusion/src/service/table.rs index 0e62f48c1f..3364ac436b 100644 --- a/crates/storage-query-datafusion/src/service/table.rs +++ b/crates/storage-query-datafusion/src/service/table.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::Sender; use restate_schema_api::service::{ServiceMetadata, ServiceMetadataResolver}; -use super::schema::ServiceBuilder; +use super::schema::SysServiceBuilder; use crate::context::QueryContext; use crate::service::row::append_service_row; use crate::table_providers::{GenericTableProvider, Scan}; @@ -31,7 +31,7 @@ pub(crate) fn register_self( resolver: impl ServiceMetadataResolver + Send + Sync + Debug + 'static, ) -> datafusion::common::Result<()> { let service_table = GenericTableProvider::new( - ServiceBuilder::schema(), + SysServiceBuilder::schema(), Arc::new(ServiceMetadataScanner(resolver)), ); @@ -70,7 +70,7 @@ async fn for_each_state( tx: Sender>, rows: Vec, ) { - let mut builder = ServiceBuilder::new(schema.clone()); + let mut builder = SysServiceBuilder::new(schema.clone()); let mut temp = String::new(); for service in rows { append_service_row(&mut builder, &mut temp, service); @@ -82,7 +82,7 @@ async fn for_each_state( // we probably don't want to panic, is it will cause the entire process to exit return; } - builder = ServiceBuilder::new(schema.clone()); + builder = SysServiceBuilder::new(schema.clone()); } } if !builder.empty() { diff --git a/crates/storage-query-datafusion/src/table_docs.rs b/crates/storage-query-datafusion/src/table_docs.rs new file mode 100644 index 0000000000..f6a01b821f --- /dev/null +++ b/crates/storage-query-datafusion/src/table_docs.rs @@ -0,0 +1,131 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use crate::{ + deployment, idempotency, inbox, invocation_state, invocation_status, journal, + keyed_service_status, promise, service, state, +}; +use std::borrow::Cow; + +/// List of available table docs. Whenever you add aa new table, add its table docs to +/// this array. This will ensure that the table docs will be included in the automatic +/// table docs generation process. +pub const ALL_TABLE_DOCS: &[StaticTableDocs] = &[ + state::schema::TABLE_DOCS, + journal::schema::TABLE_DOCS, + keyed_service_status::schema::TABLE_DOCS, + inbox::schema::TABLE_DOCS, + idempotency::schema::TABLE_DOCS, + promise::schema::TABLE_DOCS, + service::schema::TABLE_DOCS, + deployment::schema::TABLE_DOCS, +]; + +pub trait TableDocs { + fn name(&self) -> &str; + + fn columns(&self) -> &[TableColumn]; +} + +#[derive(Debug, Copy, Clone)] +pub struct TableColumn { + pub name: &'static str, + pub column_type: &'static str, + pub description: &'static str, +} + +#[derive(Debug, Copy, Clone)] +pub struct StaticTableDocs { + pub name: &'static str, + pub columns: &'static [TableColumn], +} + +impl TableDocs for StaticTableDocs { + fn name(&self) -> &str { + self.name + } + + fn columns(&self) -> &[TableColumn] { + self.columns + } +} + +#[derive(Debug)] +pub struct OwnedTableDocs { + pub name: Cow<'static, str>, + pub columns: Vec, +} + +impl TableDocs for OwnedTableDocs { + fn name(&self) -> &str { + self.name.as_ref() + } + + fn columns(&self) -> &[TableColumn] { + &self.columns + } +} + +pub fn sys_invocation_table_docs() -> OwnedTableDocs { + // We need to compile this manually, due to the fact that it's a view. + use std::collections::HashMap; + let mut sys_invocation_state: HashMap<&'static str, TableColumn> = + invocation_state::schema::TABLE_DOCS + .columns + .iter() + .map(|column| (column.name, *column)) + .collect(); + let mut sys_invocation_status: HashMap<&'static str, TableColumn> = + invocation_status::schema::TABLE_DOCS + .columns + .iter() + .map(|column| (column.name, *column)) + .collect(); + + let columns = vec![ + sys_invocation_status.remove("id").expect("id should exist"), + sys_invocation_status.remove("target").expect("target should exist"), + sys_invocation_status.remove("target_service_name").expect("target_service_name should exist"), + sys_invocation_status.remove("target_service_key").expect("target_service_key should exist"), + sys_invocation_status.remove("target_handler_name").expect("target_handler_name should exist"), + sys_invocation_status.remove("target_service_ty").expect("target_service_ty should exist"), + sys_invocation_status.remove("invoked_by").expect("invoked_by should exist"), + sys_invocation_status.remove("invoked_by_service_name").expect("invoked_by_service_name should exist"), + sys_invocation_status.remove("invoked_by_id").expect("invoked_by_id should exist"), + sys_invocation_status.remove("invoked_by_target").expect("invoked_by_target should exist"), + sys_invocation_status.remove("pinned_deployment_id").expect("pinned_deployment_id should exist"), + sys_invocation_status.remove("trace_id").expect("trace_id should exist"), + sys_invocation_status.remove("journal_size").expect("journal_size should exist"), + sys_invocation_status.remove("created_at").expect("created_at should exist"), + sys_invocation_status.remove("modified_at").expect("modified_at should exist"), + + sys_invocation_state.remove("retry_count").expect("retry_count should exist"), + sys_invocation_state.remove("last_start_at").expect("last_start_at should exist"), + sys_invocation_state.remove("next_retry_at").expect("next_retry_at should exist"), + sys_invocation_state.remove("last_attempt_deployment_id").expect("last_attempt_deployment_id should exist"), + sys_invocation_state.remove("last_attempt_server").expect("last_attempt_server should exist"), + sys_invocation_state.remove("last_failure").expect("last_failure should exist"), + sys_invocation_state.remove("last_failure_error_code").expect("last_failure_error_code should exist"), + sys_invocation_state.remove("last_failure_related_entry_index").expect("last_failure_related_entry_index should exist"), + sys_invocation_state.remove("last_failure_related_entry_name").expect("last_failure_related_entry_name should exist"), + sys_invocation_state.remove("last_failure_related_entry_type").expect("last_failure_related_entry_type should exist"), + + TableColumn { + name: "status", + column_type: "Utf8", + description: "Either `pending` or `ready` or `running` or `backing-off` or `suspended` or `completed`.", + }, + ]; + + OwnedTableDocs { + name: Cow::Borrowed("sys_invocation"), + columns, + } +} diff --git a/crates/storage-query-datafusion/src/table_macro.rs b/crates/storage-query-datafusion/src/table_macro.rs index fb0831e951..125d533673 100644 --- a/crates/storage-query-datafusion/src/table_macro.rs +++ b/crates/storage-query-datafusion/src/table_macro.rs @@ -481,11 +481,18 @@ macro_rules! define_table { // -------------------------------------------------------------------------- #[cfg(feature = "table_docs")] - pub const TABLE_DOCS: &'static [(&'static str, &'static str, &'static str)] = &[ - $( - (stringify!($element), document_type!($ty), concat!($($doc),*)), - )+ - ]; + pub const TABLE_DOCS: $crate::table_docs::StaticTableDocs = $crate::table_docs::StaticTableDocs { + name: stringify!($table_name), + columns: &[ + $( + $crate::table_docs::TableColumn { + name: stringify!($element), + column_type: document_type!($ty), + description: concat!($($doc),*) + }, + )+ + ], + }; }) } diff --git a/tools/xtask/src/main.rs b/tools/xtask/src/main.rs index 4604c9b751..c083f6bdb9 100644 --- a/tools/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -18,6 +18,7 @@ use restate_node_services::node_svc::node_svc_client::NodeSvcClient; use restate_schema_api::subscription::Subscription; use restate_service_client::{AssumeRoleCacheMode, ServiceClient}; use restate_service_protocol::discovery::ServiceDiscovery; +use restate_storage_query_datafusion::table_docs; use restate_types::arc_util::Constant; use restate_types::config::Configuration; use restate_types::identifiers::SubscriptionId; @@ -149,16 +150,31 @@ async fn generate_rest_api_doc() -> anyhow::Result<()> { Ok(()) } -fn render_table_docs(table: &str, mut w: impl io::Write) -> io::Result<()> { - let columns = restate_storage_query_datafusion::table_docs(table); +fn render_table_docs(mut write: impl Write) -> io::Result<()> { + for table_doc in restate_storage_query_datafusion::table_docs::ALL_TABLE_DOCS { + render_table_doc(table_doc, &mut write)?; + } + + // sys_invocation is a view which was not registered at table_docs::TABLE_DOCS + render_table_doc(&table_docs::sys_invocation_table_docs(), &mut write)?; + + Ok(()) +} - writeln!(&mut w, "## Table: `{table}`\n")?; - writeln!(&mut w, "| Column name | Type | Description |")?; - writeln!(&mut w, "|-------------|------|-------------|")?; - for (name, ty, desc) in columns { - writeln!(&mut w, "| `{name}` | `{ty}` | {} |", desc.trim())?; +fn render_table_doc(table_doc: &impl table_docs::TableDocs, w: &mut impl Write) -> io::Result<()> { + writeln!(w, "## Table: `{}`\n", table_doc.name())?; + writeln!(w, "| Column name | Type | Description |")?; + writeln!(w, "|-------------|------|-------------|")?; + for column in table_doc.columns() { + writeln!( + w, + "| `{}` | `{}` | {} |", + column.name, + column.column_type, + column.description.trim() + )?; } - writeln!(&mut w)?; + writeln!(w)?; Ok(()) } @@ -176,15 +192,7 @@ To learn how to access the instrospection interface, check out the [instrospecti " )?; - render_table_docs("state", &mut dest)?; - render_table_docs("sys_invocation", &mut dest)?; - render_table_docs("sys_journal", &mut dest)?; - render_table_docs("sys_keyed_service_status", &mut dest)?; - render_table_docs("sys_inbox", &mut dest)?; - render_table_docs("sys_idempotency", &mut dest)?; - render_table_docs("sys_promise", &mut dest)?; - render_table_docs("sys_service", &mut dest)?; - render_table_docs("sys_deployment", &mut dest)?; + render_table_docs(&mut dest)?; Ok(()) } From b1b6806f48f8a7b90adb627487aec56f4e10c948 Mon Sep 17 00:00:00 2001 From: Till Rohrmann Date: Thu, 20 Jun 2024 09:23:49 +0200 Subject: [PATCH 3/3] Update sql table column descriptions This commit updates the sql table column descriptions wrt feedback from https://github.com/restatedev/documentation/pull/422. --- .../src/idempotency/schema.rs | 4 ++-- .../storage-query-datafusion/src/inbox/schema.rs | 2 +- .../src/invocation_state/schema.rs | 9 +++++---- .../src/invocation_status/schema.rs | 15 ++++++++------- .../src/journal/schema.rs | 3 ++- .../src/keyed_service_status/schema.rs | 2 +- .../src/promise/schema.rs | 4 ++-- .../storage-query-datafusion/src/state/schema.rs | 3 ++- 8 files changed, 23 insertions(+), 19 deletions(-) diff --git a/crates/storage-query-datafusion/src/idempotency/schema.rs b/crates/storage-query-datafusion/src/idempotency/schema.rs index 02b928707a..6cc76ac5fa 100644 --- a/crates/storage-query-datafusion/src/idempotency/schema.rs +++ b/crates/storage-query-datafusion/src/idempotency/schema.rs @@ -18,10 +18,10 @@ define_table!(sys_idempotency( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, - /// The name for the invoked service. + /// The name of the invoked service. service_name: DataType::LargeUtf8, - /// The key of the Virtual Object or of the Workflow. Null for regular services. + /// The key of the virtual object or the workflow ID. Null for regular services. service_key: DataType::LargeUtf8, /// The invoked handler. diff --git a/crates/storage-query-datafusion/src/inbox/schema.rs b/crates/storage-query-datafusion/src/inbox/schema.rs index 26df9fa9a7..ad35f504e1 100644 --- a/crates/storage-query-datafusion/src/inbox/schema.rs +++ b/crates/storage-query-datafusion/src/inbox/schema.rs @@ -18,7 +18,7 @@ define_table!(sys_inbox( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, - /// The name for the invoked virtual object/workflow. + /// The name of the invoked virtual object/workflow. service_name: DataType::LargeUtf8, /// The key of the virtual object/workflow. diff --git a/crates/storage-query-datafusion/src/invocation_state/schema.rs b/crates/storage-query-datafusion/src/invocation_state/schema.rs index f085221fa4..4d5bc850db 100644 --- a/crates/storage-query-datafusion/src/invocation_state/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_state/schema.rs @@ -24,8 +24,9 @@ define_table!(sys_invocation_state( /// If true, the invocation is currently in-flight in_flight: DataType::Boolean, - /// The number of attempts since the last successful attempt of this invocation. - /// Increments on start, so 2 or more means a failure occurred. + /// The number of invocation attempts since the current leader started executing it. Increments + /// on start, so a value greater than 1 means a failure occurred. Note: the value is not a + /// global attempt counter across invocation suspensions and leadership changes. retry_count: DataType::UInt64, /// Timestamp indicating the start of the most recent attempt of this invocation. @@ -35,8 +36,8 @@ define_table!(sys_invocation_state( // guaranteed to be set unlike in `sys_status` table which require that the // deployment to be committed before it is set. - /// The opaque service deployment ID that was used in the most recent attempt of this - /// invocation; this will be set before a journal entry is stored, but can change later. + /// The ID of the service deployment that executed the most recent attempt of this invocation; + /// this is set before a journal entry is stored, but can change later. last_attempt_deployment_id: DataType::LargeUtf8, /// Server/SDK version, e.g. `restate-sdk-java/1.0.1` diff --git a/crates/storage-query-datafusion/src/invocation_status/schema.rs b/crates/storage-query-datafusion/src/invocation_status/schema.rs index 1fdb2fdd1c..226ca1fd73 100644 --- a/crates/storage-query-datafusion/src/invocation_status/schema.rs +++ b/crates/storage-query-datafusion/src/invocation_status/schema.rs @@ -25,14 +25,14 @@ define_table!(sys_invocation_status( status: DataType::LargeUtf8, /// Invocation Target. Format for plain services: `ServiceName/HandlerName`, e.g. - /// `Greeter/greet`. Format for Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, + /// `Greeter/greet`. Format for virtual objects/workflows: `VirtualObjectName/Key/HandlerName`, /// e.g. `Greeter/Francesco/greet`. target: DataType::LargeUtf8, - /// The name for the invoked service. + /// The name of the invoked service. target_service_name: DataType::LargeUtf8, - /// The key of the Virtual Object or of the Workflow. Null for regular services. + /// The key of the virtual object or the workflow ID. Null for regular services. target_service_key: DataType::LargeUtf8, /// The invoked handler. @@ -56,8 +56,9 @@ define_table!(sys_invocation_status( /// `null` if invoked externally. invoked_by_target: DataType::LargeUtf8, - /// The opaque service deployment ID that has been committed for this invocation; this is set - /// after the first journal entry is stored for this invocation. + /// The ID of the service deployment that started processing this invocation, and will continue + /// to do so (e.g. for retries). This gets set after the first journal entry has been stored for + /// this invocation. pinned_deployment_id: DataType::LargeUtf8, /// The ID of the trace that is assigned to this invocation. Only relevant when tracing is @@ -70,7 +71,7 @@ define_table!(sys_invocation_status( /// Timestamp indicating the start of this invocation. created_at: DataType::Date64, - /// Timestamp indicating the last state transition. For example, last time the status changed - /// from `invoked` to `suspended`. + /// Timestamp indicating the last invocation status transition. For example, last time the + /// status changed from `invoked` to `suspended`. modified_at: DataType::Date64, )); diff --git a/crates/storage-query-datafusion/src/journal/schema.rs b/crates/storage-query-datafusion/src/journal/schema.rs index ea3da170d6..16cdbae162 100644 --- a/crates/storage-query-datafusion/src/journal/schema.rs +++ b/crates/storage-query-datafusion/src/journal/schema.rs @@ -39,7 +39,8 @@ define_table!(sys_journal( /// If this entry represents an outbound invocation, indicates the invocation Target. Format /// for plain services: `ServiceName/HandlerName`, e.g. `Greeter/greet`. Format for - /// Virtual Objects/Workflows: `VirtualObjectName/Key/HandlerName`, e.g. `Greeter/Francesco/greet`. + /// virtual objects/workflows: `VirtualObjectName/Key/HandlerName`, e.g. + /// `Greeter/Francesco/greet`. invoked_target: DataType::LargeUtf8, /// If this entry represents a sleep, indicates wakeup time. diff --git a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs index b4608faaf5..74473d6f13 100644 --- a/crates/storage-query-datafusion/src/keyed_service_status/schema.rs +++ b/crates/storage-query-datafusion/src/keyed_service_status/schema.rs @@ -18,7 +18,7 @@ define_table!(sys_keyed_service_status( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, - /// The name for the invoked virtual object/workflow. + /// The name of the invoked virtual object/workflow. service_name: DataType::LargeUtf8, /// The key of the virtual object/workflow. diff --git a/crates/storage-query-datafusion/src/promise/schema.rs b/crates/storage-query-datafusion/src/promise/schema.rs index b9af63f088..ebbd31fca4 100644 --- a/crates/storage-query-datafusion/src/promise/schema.rs +++ b/crates/storage-query-datafusion/src/promise/schema.rs @@ -18,10 +18,10 @@ define_table!(sys_promise( /// Internal column that is used for partitioning the services invocations. Can be ignored. partition_key: DataType::UInt64, - /// The name for the workflow. + /// The name of the workflow service. service_name: DataType::LargeUtf8, - /// The key of the virtual workflow. + /// The workflow ID. service_key: DataType::LargeUtf8, /// The promise key. diff --git a/crates/storage-query-datafusion/src/state/schema.rs b/crates/storage-query-datafusion/src/state/schema.rs index c7e7e11c37..7102aac609 100644 --- a/crates/storage-query-datafusion/src/state/schema.rs +++ b/crates/storage-query-datafusion/src/state/schema.rs @@ -28,7 +28,8 @@ define_table!(state( key: DataType::LargeUtf8, /// Only contains meaningful values when a service stores state as `utf8`. This is the case for - /// TypeScript services since the TypeScript SDK serializes values as JSON. + /// services that serialize state using JSON (default for Typescript SDK, Java/Kotlin SDK if + /// using JsonSerdes). value_utf8: DataType::LargeUtf8, /// A binary, uninterpreted representation of the value. You can use the more specific column