Generate component/datatype docs (#3535)

### What

* Closes #3503

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/3535) (if
applicable)

- [PR Build Summary](https://build.rerun.io/pr/3535)
- [Docs
preview](https://rerun.io/preview/9a5f7290e23ae2c14ec3d80de5d2df6015056787/docs)
<!--DOCS-PREVIEW-->
- [Examples
preview](https://rerun.io/preview/9a5f7290e23ae2c14ec3d80de5d2df6015056787/examples)
<!--EXAMPLES-PREVIEW-->
- [Recent benchmark results](https://ref.rerun.io/dev/bench/)
- [Wasm size tracking](https://ref.rerun.io/dev/sizes/)

crates/re_types/build.rs

@@ -118,7 +118,7 @@ fn main() {
         ),
     );
 
-    report.panic_on_errors();
+    report.finalize();
 
     write_versioning_hash(SOURCE_HASH_PATH, new_hash);
 }

crates/re_types/definitions/rerun/components/clear_is_recursive.fbs

@@ -9,7 +9,7 @@ namespace rerun.components;
 
 // ---
 
-/// Configures how a clear operation should behave - recurive or no?
+/// Configures how a clear operation should behave - recursive or not?
 struct ClearIsRecursive (
   "attr.arrow.transparent",
   "attr.python.aliases": "bool",

crates/re_types/definitions/rerun/components/resolution.fbs

@@ -8,7 +8,7 @@ namespace rerun.components;
 
 /// Pixel resolution width & height, e.g. of a camera sensor.
 ///
-/// Typically in integer units, but for some usecases floating point may be used.
+/// Typically in integer units, but for some use cases floating point may be used.
 struct Resolution (
   "attr.rust.derive": "Copy, PartialEq"
 ) {

crates/re_types_builder/src/bin/build_re_types.rs

@@ -118,7 +118,7 @@ fn main() {
         ),
     );
 
-    report.panic_on_errors();
+    report.finalize();
 
     write_versioning_hash(re_types_source_hash_path, new_hash);
     write_versioning_hash(re_types_builder_source_hash_path, builder_hash);

crates/re_types_builder/src/codegen/docs/mod.rs

@@ -2,14 +2,15 @@ use crate::codegen::common::ExampleInfo;
 use crate::objects::FieldKind;
 use crate::CodeGenerator;
 use crate::Object;
-use crate::ObjectField;
 use crate::ObjectKind;
 use crate::Objects;
 use crate::Reporter;
 use camino::Utf8PathBuf;
 use std::collections::BTreeSet;
 use std::fmt::Write;
 
+type ObjectMap = std::collections::BTreeMap<String, Object>;
+
 use super::common::get_documentation;
 
 macro_rules! putln {
@@ -32,81 +33,235 @@ impl DocsCodeGenerator {
 impl CodeGenerator for DocsCodeGenerator {
     fn generate(
         &mut self,
-        _reporter: &Reporter,
+        reporter: &Reporter,
         objects: &Objects,
         _arrow_registry: &crate::ArrowRegistry,
     ) -> BTreeSet<camino::Utf8PathBuf> {
         re_tracing::profile_function!();
 
         let mut filepaths = BTreeSet::new();
 
-        let components = objects.ordered_objects(Some(ObjectKind::Component));
-        for object in objects.ordered_objects(Some(ObjectKind::Archetype)) {
+        let (mut archetypes, mut components, mut datatypes) = (Vec::new(), Vec::new(), Vec::new());
+        let object_map = &objects.objects;
+        for object in object_map.values() {
             // skip test-only archetypes
             if object.is_testing() {
                 continue;
             }
 
-            let top_level_docs = get_documentation(&object.docs, &[]);
-            let examples = object
-                .docs
-                .tagged_docs
-                .get("example")
-                .iter()
-                .flat_map(|v| v.iter())
-                .map(ExampleInfo::parse)
-                .collect::<Vec<_>>();
-
-            let mut o = String::new();
-
-            frontmatter(&mut o, &object.name);
-            putln!(o);
-            for mut line in top_level_docs {
-                if line.starts_with(char::is_whitespace) {
-                    line.remove(0);
-                }
-                putln!(o, "{line}");
+            match object.kind {
+                ObjectKind::Datatype => datatypes.push(object),
+                ObjectKind::Component => components.push(object),
+                ObjectKind::Archetype => archetypes.push(object),
             }
-            putln!(o);
-            write_archetype_fields(&mut o, &components, &object.fields);
-            putln!(o);
-            write_example_list(&mut o, &examples);
 
+            let page = object_page(reporter, object, object_map);
+            let path = self.docs_dir.join(format!(
+                "{}/{}.md",
+                object.kind.plural_snake_case(),
+                object.snake_case_name()
+            ));
+            super::common::write_file(&path, &page);
+            filepaths.insert(path);
+        }
+
+        for (kind, order, prelude, objects) in [
+            (
+                ObjectKind::Archetype,
+                1,
+                "Archetypes are bundles of components",
+                &archetypes,
+            ),
+            (
+                ObjectKind::Component,
+                2,
+                "Archetypes are bundles of components",
+                &components,
+            ),
+            (
+                ObjectKind::Datatype,
+                3,
+                "Data types are the lowest layer of the data model hierarchy",
+                &datatypes,
+            ),
+        ] {
+            let page = index_page(kind, order, prelude, objects);
             let path = self
                 .docs_dir
-                .join(format!("{}.md", object.snake_case_name()));
-            super::common::write_file(&path, &o);
+                .join(format!("{}.md", kind.plural_snake_case()));
+            super::common::write_file(&path, &page);
             filepaths.insert(path);
         }
 
         filepaths
     }
 }
 
-fn frontmatter(o: &mut String, title: &str) {
+fn index_page(kind: ObjectKind, order: u64, prelude: &str, objects: &[&Object]) -> String {
+    let mut page = String::new();
+
+    write_frontmatter(&mut page, kind.plural_name(), Some(order));
+    putln!(page);
+    putln!(page, "{prelude}");
+    putln!(page);
+    if !objects.is_empty() {
+        putln!(page, "## Available {}", kind.plural_name().to_lowercase());
+        putln!(page);
+        for object in objects {
+            putln!(
+                page,
+                "* [`{}`]({}/{}.md)",
+                object.name,
+                object.kind.plural_snake_case(),
+                object.snake_case_name()
+            );
+        }
+    }
+
+    page
+}
+
+fn object_page(reporter: &Reporter, object: &Object, object_map: &ObjectMap) -> String {
+    let top_level_docs = get_documentation(&object.docs, &[]);
+    let examples = object
+        .docs
+        .tagged_docs
+        .get("example")
+        .iter()
+        .flat_map(|v| v.iter())
+        .map(ExampleInfo::parse)
+        .collect::<Vec<_>>();
+
+    let mut page = String::new();
+
+    write_frontmatter(&mut page, &object.name, None);
+    putln!(page);
+    for mut line in top_level_docs {
+        if line.starts_with(char::is_whitespace) {
+            line.remove(0);
+        }
+        putln!(page, "{line}");
+    }
+    putln!(page);
+
+    match object.kind {
+        ObjectKind::Datatype | ObjectKind::Component => {
+            write_fields(&mut page, object, object_map);
+        }
+        ObjectKind::Archetype => write_archetype_fields(&mut page, object, object_map),
+    }
+
+    putln!(page);
+    write_example_list(&mut page, &examples);
+
+    match object.kind {
+        ObjectKind::Datatype | ObjectKind::Component => {
+            putln!(page);
+            write_used_by(&mut page, reporter, object, object_map);
+        }
+        ObjectKind::Archetype => {}
+    }
+
+    page
+}
+
+fn write_frontmatter(o: &mut String, title: &str, order: Option<u64>) {
     putln!(o, "---");
     putln!(o, "title: {title:?}");
+    if let Some(order) = order {
+        // The order is used to sort `rerun.io/docs` side navigation
+        putln!(o, "order: {order}");
+    }
     putln!(o, "---");
 }
 
-fn write_archetype_fields(o: &mut String, all_components: &[&Object], fields: &[ObjectField]) {
-    if fields.is_empty() {
+fn write_fields(o: &mut String, object: &Object, object_map: &ObjectMap) {
+    if object.fields.is_empty() {
+        return;
+    }
+
+    let mut fields = Vec::new();
+    for field in &object.fields {
+        let Some(fqname) = field.typ.fqname() else {
+            continue;
+        };
+        let Some(ty) = object_map.get(fqname) else {
+            continue;
+        };
+        fields.push(format!(
+            "* {}: [`{}`](../{}/{}.md)",
+            field.name,
+            ty.name,
+            ty.kind.plural_snake_case(),
+            ty.snake_case_name()
+        ));
+    }
+
+    if !fields.is_empty() {
+        putln!(o, "## Fields");
+        putln!(o);
+        for field in fields {
+            putln!(o, "{field}");
+        }
+    }
+}
+
+fn write_used_by(o: &mut String, reporter: &Reporter, object: &Object, object_map: &ObjectMap) {
+    let mut used_by = Vec::new();
+    for ty in object_map.values() {
+        for field in &ty.fields {
+            if field.typ.fqname() == Some(object.fqname.as_str()) {
+                used_by.push(format!(
+                    "* [`{}`](../{}/{}.md)",
+                    ty.name,
+                    ty.kind.plural_snake_case(),
+                    ty.snake_case_name()
+                ));
+            }
+        }
+    }
+
+    if used_by.is_empty() {
+        reporter.warn(format!("{:?} is unused", object.fqname.as_str()));
+    } else {
+        putln!(o, "## Used by");
+        putln!(o);
+        for ty in used_by {
+            putln!(o, "{ty}");
+        }
+    }
+}
+
+fn write_archetype_fields(o: &mut String, object: &Object, object_map: &ObjectMap) {
+    if object.fields.is_empty() {
         return;
     }
 
     // collect names of field _components_ by their `FieldKind`
     let (mut required, mut recommended, mut optional) = (Vec::new(), Vec::new(), Vec::new());
-    for field in fields {
-        let (target, component) = match field
-            .kind()
-            .and_then(|kind| Some((kind, find_component(field, all_components)?)))
-        {
-            Some((FieldKind::Required, component)) => (&mut required, component),
-            Some((FieldKind::Recommended, component)) => (&mut recommended, component),
-            Some((FieldKind::Optional, component)) => (&mut optional, component),
+    for field in &object.fields {
+        let Some(fqname) = field.typ.fqname() else {
+            continue;
+        };
+        let Some(ty) = object_map.get(fqname) else {
+            continue;
+        };
+        let target = match field.kind() {
+            Some(FieldKind::Required) => &mut required,
+            Some(FieldKind::Recommended) => &mut recommended,
+            Some(FieldKind::Optional) => &mut optional,
             _ => continue,
         };
-        target.push(format!("`{}`", component.name));
+        target.push(format!(
+            "[`{}`](../{}/{}.md)",
+            ty.name,
+            ty.kind.plural_snake_case(),
+            ty.snake_case_name()
+        ));
+    }
+
+    if required.is_empty() && recommended.is_empty() && optional.is_empty() {
+        return;
     }
 
     putln!(o, "## Components");
@@ -124,14 +279,6 @@ fn write_archetype_fields(o: &mut String, all_components: &[&Object], fields: &[
     }
 }
 
-fn find_component<'a>(field: &ObjectField, components: &[&'a Object]) -> Option<&'a Object> {
-    field
-        .typ
-        .fqname()
-        .and_then(|fqname| components.iter().find(|c| c.fqname == fqname))
-        .copied()
-}
-
 fn write_example_list(o: &mut String, examples: &[ExampleInfo<'_>]) {
     if examples.is_empty() {
         return;

crates/re_types_builder/src/objects.rs

@@ -218,6 +218,14 @@ impl ObjectKind {
             ObjectKind::Archetype => "archetypes",
         }
     }
+
+    pub fn plural_name(&self) -> &'static str {
+        match self {
+            ObjectKind::Datatype => "Datatypes",
+            ObjectKind::Component => "Components",
+            ObjectKind::Archetype => "Archetypes",
+        }
+    }
 }
 
 /// A high-level representation of a flatbuffers object's documentation.