Generate documentation of SQL tables

crates/invoker-api/Cargo.toml

@@ -24,3 +24,6 @@ futures = { workspace = true }
 serde = { workspace = true, optional = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
+
+[dev-dependencies]
+restate-types = { workspace = true, features = ["test-util"] }

crates/storage-query-datafusion/Cargo.toml

@@ -10,6 +10,7 @@ publish = false
 [features]
 default = []
 options_schema = ["dep:schemars"]
+table_docs = []
 
 [dependencies]
 restate-core = { workspace = true }

crates/storage-query-datafusion/src/deployment/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/deployment/row.rs

@@ -8,13 +8,13 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-use super::schema::DeploymentBuilder;
+use super::schema::SysDeploymentBuilder;
 use crate::table_util::format_using;
 use restate_schema_api::deployment::{Deployment, DeploymentType};
 
 #[inline]
 pub(crate) fn append_deployment_row(
-    builder: &mut DeploymentBuilder,
+    builder: &mut SysDeploymentBuilder,
     output: &mut String,
     deployment: Deployment,
 ) {

crates/storage-query-datafusion/src/deployment/schema.rs

@@ -14,9 +14,16 @@ use crate::table_macro::*;
 
 use datafusion::arrow::datatypes::DataType;
 
-define_table!(deployment(
+define_table!(sys_deployment(
+    /// The ID of the service deployment.
     id: DataType::LargeUtf8,
+
+    /// The type of the endpoint. Either `http` or `lambda`.
     ty: DataType::LargeUtf8,
+
+    /// The address of the endpoint. Either HTTP URL or Lambda ARN.
     endpoint: DataType::LargeUtf8,
+
+    /// Timestamp indicating the deployment registration time.
     created_at: DataType::Date64,
 ));

crates/storage-query-datafusion/src/deployment/table.rs

@@ -21,7 +21,7 @@ use tokio::sync::mpsc::Sender;
 use restate_schema_api::deployment::{Deployment, DeploymentResolver};
 use restate_types::identifiers::ServiceRevision;
 
-use super::schema::DeploymentBuilder;
+use super::schema::SysDeploymentBuilder;
 use crate::context::QueryContext;
 use crate::deployment::row::append_deployment_row;
 use crate::table_providers::{GenericTableProvider, Scan};
@@ -32,7 +32,7 @@ pub(crate) fn register_self(
     resolver: impl DeploymentResolver + Send + Sync + Debug + 'static,
 ) -> datafusion::common::Result<()> {
     let deployment_table = GenericTableProvider::new(
-        DeploymentBuilder::schema(),
+        SysDeploymentBuilder::schema(),
         Arc::new(DeploymentMetadataScanner(resolver)),
     );
 
@@ -71,7 +71,7 @@ async fn for_each_state(
     tx: Sender<datafusion::common::Result<RecordBatch>>,
     rows: Vec<(Deployment, Vec<(String, ServiceRevision)>)>,
 ) {
-    let mut builder = DeploymentBuilder::new(schema.clone());
+    let mut builder = SysDeploymentBuilder::new(schema.clone());
     let mut temp = String::new();
     for (deployment, _) in rows {
         append_deployment_row(&mut builder, &mut temp, deployment);
@@ -83,7 +83,7 @@ async fn for_each_state(
                 // we probably don't want to panic, is it will cause the entire process to exit
                 return;
             }
-            builder = DeploymentBuilder::new(schema.clone());
+            builder = SysDeploymentBuilder::new(schema.clone());
         }
     }
     if !builder.empty() {

crates/storage-query-datafusion/src/idempotency/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/idempotency/row.rs

@@ -8,15 +8,15 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-use super::schema::IdempotencyBuilder;
+use super::schema::SysIdempotencyBuilder;
 
 use crate::table_util::format_using;
 use restate_storage_api::idempotency_table::IdempotencyMetadata;
 use restate_types::identifiers::{IdempotencyId, WithPartitionKey};
 
 #[inline]
 pub(crate) fn append_idempotency_row(
-    builder: &mut IdempotencyBuilder,
+    builder: &mut SysIdempotencyBuilder,
     output: &mut String,
     idempotency_id: IdempotencyId,
     idempotency_metadata: IdempotencyMetadata,

crates/storage-query-datafusion/src/idempotency/schema.rs

@@ -14,13 +14,22 @@ use crate::table_macro::*;
 
 use datafusion::arrow::datatypes::DataType;
 
-define_table!(idempotency(
+define_table!(sys_idempotency(
+    /// Internal column that is used for partitioning the services invocations. Can be ignored.
     partition_key: DataType::UInt64,
 
+    /// The name of the invoked service.
     service_name: DataType::LargeUtf8,
+
+    /// The key of the virtual object or the workflow ID. Null for regular services.
     service_key: DataType::LargeUtf8,
+
+    /// The invoked handler.
     service_handler: DataType::LargeUtf8,
+
+    /// The user provided idempotency key.
     idempotency_key: DataType::LargeUtf8,
 
+    /// [Invocation ID](/operate/invocation#invocation-identifier).
     invocation_id: DataType::LargeUtf8
 ));

crates/storage-query-datafusion/src/idempotency/table.rs

@@ -19,7 +19,7 @@ use restate_storage_api::idempotency_table::{IdempotencyMetadata, ReadOnlyIdempo
 use restate_types::identifiers::{IdempotencyId, PartitionKey};
 
 use super::row::append_idempotency_row;
-use super::schema::IdempotencyBuilder;
+use super::schema::SysIdempotencyBuilder;
 use crate::context::{QueryContext, SelectPartitions};
 use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition};
 use crate::table_providers::PartitionedTableProvider;
@@ -31,7 +31,7 @@ pub(crate) fn register_self(
 ) -> datafusion::common::Result<()> {
     let table = PartitionedTableProvider::new(
         partition_selector,
-        IdempotencyBuilder::schema(),
+        SysIdempotencyBuilder::schema(),
         LocalPartitionsScanner::new(partition_store_manager, IdempotencyScanner),
     );
 
@@ -44,7 +44,7 @@ pub(crate) fn register_self(
 struct IdempotencyScanner;
 
 impl ScanLocalPartition for IdempotencyScanner {
-    type Builder = IdempotencyBuilder;
+    type Builder = SysIdempotencyBuilder;
     type Item = (IdempotencyId, IdempotencyMetadata);
 
     fn scan_partition_store(

crates/storage-query-datafusion/src/inbox/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/inbox/row.rs

@@ -8,14 +8,14 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-use super::schema::InboxBuilder;
+use super::schema::SysInboxBuilder;
 use crate::table_util::format_using;
 use restate_storage_api::inbox_table::{InboxEntry, SequenceNumberInboxEntry};
 use restate_types::identifiers::{TimestampAwareId, WithPartitionKey};
 
 #[inline]
 pub(crate) fn append_inbox_row(
-    builder: &mut InboxBuilder,
+    builder: &mut SysInboxBuilder,
     output: &mut String,
     inbox_entry: SequenceNumberInboxEntry,
 ) {

crates/storage-query-datafusion/src/inbox/schema.rs

@@ -14,15 +14,22 @@ use crate::table_macro::*;
 
 use datafusion::arrow::datatypes::DataType;
 
-define_table!(inbox(
+define_table!(sys_inbox(
+    /// Internal column that is used for partitioning the services invocations. Can be ignored.
     partition_key: DataType::UInt64,
 
+    /// The name of the invoked virtual object/workflow.
     service_name: DataType::LargeUtf8,
+
+    /// The key of the virtual object/workflow.
     service_key: DataType::LargeUtf8,
 
+    /// [Invocation ID](/operate/invocation#invocation-identifier).
     id: DataType::LargeUtf8,
 
+    /// Sequence number in the inbox.
     sequence_number: DataType::UInt64,
 
+    /// Timestamp indicating the start of this invocation.
     created_at: DataType::Date64,
 ));

crates/storage-query-datafusion/src/inbox/table.rs

@@ -20,7 +20,7 @@ use restate_types::identifiers::PartitionKey;
 
 use crate::context::{QueryContext, SelectPartitions};
 use crate::inbox::row::append_inbox_row;
-use crate::inbox::schema::InboxBuilder;
+use crate::inbox::schema::SysInboxBuilder;
 use crate::partition_store_scanner::{LocalPartitionsScanner, ScanLocalPartition};
 use crate::table_providers::PartitionedTableProvider;
 
@@ -31,7 +31,7 @@ pub(crate) fn register_self(
 ) -> datafusion::common::Result<()> {
     let table = PartitionedTableProvider::new(
         partition_selector,
-        InboxBuilder::schema(),
+        SysInboxBuilder::schema(),
         LocalPartitionsScanner::new(partition_store_manager, InboxScanner),
     );
 
@@ -44,7 +44,7 @@ pub(crate) fn register_self(
 struct InboxScanner;
 
 impl ScanLocalPartition for InboxScanner {
-    type Builder = InboxBuilder;
+    type Builder = SysInboxBuilder;
     type Item = SequenceNumberInboxEntry;
 
     fn scan_partition_store(

crates/storage-query-datafusion/src/invocation_state/mod.rs

@@ -9,7 +9,7 @@
 // by the Apache License, Version 2.0.
 
 mod row;
-mod schema;
+pub(crate) mod schema;
 mod table;
 
 pub(crate) use table::register_self;

crates/storage-query-datafusion/src/invocation_state/row.rs

@@ -8,15 +8,15 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-use crate::invocation_state::schema::StateBuilder;
+use crate::invocation_state::schema::SysInvocationStateBuilder;
 use crate::table_util::format_using;
 use restate_invoker_api::InvocationStatusReport;
 use restate_types::identifiers::WithPartitionKey;
 use restate_types::time::MillisSinceEpoch;
 
 #[inline]
-pub(crate) fn append_state_row(
-    builder: &mut StateBuilder,
+pub(crate) fn append_invocation_state_row(
+    builder: &mut SysInvocationStateBuilder,
     output: &mut String,
     status_row: InvocationStatusReport,
 ) {

crates/storage-query-datafusion/src/invocation_state/schema.rs

@@ -14,24 +14,52 @@ use crate::table_macro::*;
 
 use datafusion::arrow::datatypes::DataType;
 
-define_table!(state(
+define_table!(sys_invocation_state(
+    /// Internal column that is used for partitioning the services invocations. Can be ignored.
     partition_key: DataType::UInt64,
+
+    /// [Invocation ID](/operate/invocation#invocation-identifier).
     id: DataType::LargeUtf8,
 
+    /// If true, the invocation is currently in-flight
     in_flight: DataType::Boolean,
+
+    /// The number of invocation attempts since the current leader started executing it. Increments
+    /// on start, so a value greater than 1 means a failure occurred. Note: the value is not a
+    /// global attempt counter across invocation suspensions and leadership changes.
     retry_count: DataType::UInt64,
+
+    /// Timestamp indicating the start of the most recent attempt of this invocation.
     last_start_at: DataType::Date64,
 
     // The deployment that was selected in the last invocation attempt. This is
     // guaranteed to be set unlike in `sys_status` table which require that the
     // deployment to be committed before it is set.
+
+    /// The ID of the service deployment that executed the most recent attempt of this invocation;
+    /// this is set before a journal entry is stored, but can change later.
     last_attempt_deployment_id: DataType::LargeUtf8,
+
+    /// Server/SDK version, e.g. `restate-sdk-java/1.0.1`
     last_attempt_server: DataType::LargeUtf8,
+
+    /// Timestamp indicating the start of the next attempt of this invocation.
     next_retry_at: DataType::Date64,
 
+    /// An error message describing the most recent failed attempt of this invocation, if any.
     last_failure: DataType::LargeUtf8,
+
+    /// The error code of the most recent failed attempt of this invocation, if any.
     last_failure_error_code: DataType::LargeUtf8,
+
+    /// The index of the journal entry that caused the failure, if any. It may be out-of-bound
+    /// of the currently stored entries in `sys_journal`.
     last_failure_related_entry_index: DataType::UInt64,
+
+    /// The name of the journal entry that caused the failure, if any.
     last_failure_related_entry_name: DataType::LargeUtf8,
+
+    /// The type of the journal entry that caused the failure, if any. You can check all the
+    /// available entry types in [`entries.rs`](https://github.com/restatedev/restate/blob/main/crates/types/src/journal/entries.rs).
     last_failure_related_entry_type: DataType::LargeUtf8,
 ));