support inlined nodes in grammar

- Created `ProductionDefinition` enum to let parent `Production` hold common properties, similar to how `ParserDefinition`, `ScannerDefinition`, etc... work today. - Added `Production::inlined` boolean property, which defaults to `false` - Inlined productions no longer produce `ProductionKind`, `TokenKind`, or `RuleKind`. To test this end-to-end, I inlined a `Scanner`, a `Parser`, and a `PrecedenceParser` in the current grammar. Will follow up in the next few PRs with: - inlining additional nodes. - adding validation to make sure references between inlined nodes are valid, and that they are not leaked to public APIs.
NomicFoundation · Jun 12, 2023 · b79f40c · b79f40c
1 parent 0f613cc
commit b79f40c
Show file tree

Hide file tree

Showing 40 changed files with 1,857 additions and 1,422 deletions.
diff --git a/.changeset/strange-frogs-camp.md b/.changeset/strange-frogs-camp.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+inlining CST nodes that offer no additional syntactic information
diff --git a/crates/codegen/ebnf/src/serialization.rs b/crates/codegen/ebnf/src/serialization.rs
@@ -1,6 +1,6 @@
 use std::{collections::VecDeque, mem::discriminant};
 
-use codegen_schema::types::{LanguageDefinition, Production, ProductionRef};
+use codegen_schema::types::{LanguageDefinition, ProductionDefinition, ProductionRef};
 use semver::Version;
 
 use crate::nodes::EbnfNode;
@@ -23,25 +23,25 @@ impl<'language> EbnfSerializer<'language> {
         production: &'language ProductionRef,
         version: &Version,
     ) -> Option<String> {
-        let body = match production.as_ref() {
-            Production::Scanner { version_map, .. } => {
+        let body = match &production.definition {
+            ProductionDefinition::Scanner { version_map, .. } => {
                 version_map.get_for_version(version)?.generate_ebnf()
             }
-            Production::TriviaParser { version_map, .. } => {
+            ProductionDefinition::TriviaParser { version_map, .. } => {
                 version_map.get_for_version(version)?.generate_ebnf()
             }
-            Production::Parser { version_map, .. } => {
+            ProductionDefinition::Parser { version_map, .. } => {
                 version_map.get_for_version(version)?.generate_ebnf()
             }
-            Production::PrecedenceParser { version_map, .. } => {
+            ProductionDefinition::PrecedenceParser { version_map, .. } => {
                 version_map.get_for_version(version)?.generate_ebnf()
             }
         };
 
         let mut instance = Self {
             language,
             buffer: String::new(),
-            base_production: production.name(),
+            base_production: &production.name,
             queue: VecDeque::new(),
         };
 
@@ -184,7 +184,7 @@ impl<'language> EbnfSerializer<'language> {
 
     fn display_name(&self, name: &String) -> String {
         if let Some(production) = self.language.productions.get(name) {
-            if matches!(production.as_ref(), Production::Scanner { .. }) {
+            if matches!(production.definition, ProductionDefinition::Scanner { .. }) {
                 return format!("«{name}»");
             }
         }

diff --git a/crates/codegen/schema/src/compiler.rs b/crates/codegen/schema/src/compiler.rs
@@ -28,7 +28,7 @@ impl LanguageDefinition {
             .iter()
             .flat_map(|section| &section.topics)
             .flat_map(|topic| &topic.productions)
-            .map(|production| (production.name().to_owned(), production.clone()))
+            .map(|production| (production.name.to_owned(), production.clone()))
             .collect();
 
         let language = LanguageDefinitionRef::new(LanguageDefinition {

diff --git a/crates/codegen/schema/src/types/production.rs b/crates/codegen/schema/src/types/production.rs
@@ -10,63 +10,61 @@ use super::{parser::Parser, precedence_parser::PrecedenceParser, scanner::Scanne
 pub type ProductionRef = Rc<Production>;
 
 #[derive(Deserialize, Serialize, JsonSchema, Clone, Debug)]
-#[serde(deny_unknown_fields)]
-#[serde(tag = "kind")]
-pub enum Production {
-    Scanner {
-        name: String,
-        #[serde(flatten)]
-        version_map: VersionMap<Scanner>,
-    },
-    TriviaParser {
-        name: String,
-        #[serde(flatten)]
-        version_map: VersionMap<Parser>,
-    },
-    Parser {
-        name: String,
-        #[serde(flatten)]
-        version_map: VersionMap<Parser>,
-    },
-    PrecedenceParser {
-        name: String,
-        #[serde(flatten)]
-        version_map: VersionMap<PrecedenceParser>,
-    },
+pub struct Production {
+    pub name: String,
+
+    #[serde(default)]
+    pub inlined: bool,
+
+    #[serde(flatten)]
+    pub definition: ProductionDefinition,
 }
 
 impl Production {
-    pub fn name(&self) -> &String {
-        match self {
-            Self::Scanner { name, .. }
-            | Self::TriviaParser { name, .. }
-            | Self::Parser { name, .. }
-            | Self::PrecedenceParser { name, .. } => name,
-        }
-    }
-
     pub fn versions(&self) -> Option<Vec<&Version>> {
-        match self {
-            Production::Scanner { version_map, .. } => match version_map {
+        match &self.definition {
+            ProductionDefinition::Scanner { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => None,
                 VersionMap::Versioned(ref map) => Some(map.keys().collect()),
             },
-            Production::TriviaParser { version_map, .. } => match version_map {
+            ProductionDefinition::TriviaParser { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => None,
                 VersionMap::Versioned(ref map) => Some(map.keys().collect()),
             },
-            Production::Parser { version_map, .. } => match version_map {
+            ProductionDefinition::Parser { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => None,
                 VersionMap::Versioned(ref map) => Some(map.keys().collect()),
             },
-            Production::PrecedenceParser { version_map, .. } => match version_map {
+            ProductionDefinition::PrecedenceParser { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => None,
                 VersionMap::Versioned(ref map) => Some(map.keys().collect()),
             },
         }
     }
 }
 
+#[derive(Deserialize, Serialize, JsonSchema, Clone, Debug)]
+#[serde(deny_unknown_fields)]
+#[serde(tag = "kind")]
+pub enum ProductionDefinition {
+    Scanner {
+        #[serde(flatten)]
+        version_map: VersionMap<Scanner>,
+    },
+    TriviaParser {
+        #[serde(flatten)]
+        version_map: VersionMap<Parser>,
+    },
+    Parser {
+        #[serde(flatten)]
+        version_map: VersionMap<Parser>,
+    },
+    PrecedenceParser {
+        #[serde(flatten)]
+        version_map: VersionMap<PrecedenceParser>,
+    },
+}
+
 #[derive(Deserialize, Serialize, JsonSchema, Clone, Debug)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 pub enum VersionMap<T> {

diff --git a/crates/codegen/schema/src/validation/rules/definitions/mod.rs b/crates/codegen/schema/src/validation/rules/definitions/mod.rs
@@ -48,7 +48,7 @@ impl Visitor for Definitions {
         location: &LocationRef,
         reporter: &mut Reporter,
     ) -> bool {
-        let name = production.name();
+        let name = &production.name;
         if !self.productions_so_far.insert(name.to_owned()) {
             reporter.report(location, Errors::DuplicateProduction(name.to_owned()));
         }

diff --git a/crates/codegen/schema/src/validation/rules/empty_roots/mod.rs b/crates/codegen/schema/src/validation/rules/empty_roots/mod.rs
@@ -36,7 +36,7 @@ impl Visitor for EmptyRoots {
         _location: &LocationRef,
         _reporter: &mut Reporter,
     ) -> bool {
-        if production.name() == &self.language.root_production {
+        if production.name == self.language.root_production {
             // Skip, as it is allowed to be empty.
             return false;
         }

diff --git a/crates/codegen/schema/src/validation/rules/references/collector.rs b/crates/codegen/schema/src/validation/rules/references/collector.rs
@@ -31,7 +31,7 @@ impl Visitor for Collector {
         location: &LocationRef,
         _reporter: &mut Reporter,
     ) -> bool {
-        self.metadata.add_production(&production.name(), location);
+        self.metadata.add_production(&production.name, location);
         self.current_production = Some(production.to_owned());
 
         return true;
@@ -43,8 +43,8 @@ impl Visitor for Collector {
         _location: &LocationRef,
         _reporter: &mut Reporter,
     ) -> bool {
-        let production = self.current_production.as_ref().unwrap().name();
-        self.metadata.add_version(production, version_set);
+        let production = self.current_production.as_ref().unwrap();
+        self.metadata.add_version(&production.name, version_set);
 
         return false;
     }

diff --git a/crates/codegen/schema/src/validation/rules/references/validator.rs b/crates/codegen/schema/src/validation/rules/references/validator.rs
@@ -1,6 +1,6 @@
 use crate::{
     types::{
-        LanguageDefinitionRef, ParserDefinition, ParserRef, Production, ProductionRef,
+        LanguageDefinitionRef, ParserDefinition, ParserRef, ProductionDefinition, ProductionRef,
         ScannerDefinition, ScannerRef,
     },
     validation::{
@@ -116,8 +116,8 @@ impl Validator {
         reporter: &mut Reporter,
     ) {
         match self.language.productions.get(reference) {
-            Some(production) => match production.as_ref() {
-                Production::Scanner { .. } => {
+            Some(production) => match production.definition {
+                ProductionDefinition::Scanner { .. } => {
                     self.insert_reference(reference, location, reporter);
                 }
                 _ => {
@@ -136,12 +136,12 @@ impl Validator {
         location: &LocationRef,
         reporter: &mut Reporter,
     ) {
-        let production = self.current_production.as_ref().unwrap().name();
+        let production = self.current_production.as_ref().unwrap();
         let version_set = self.current_version_set.as_ref().unwrap();
 
         let can_be_added = self
             .metadata
-            .add_reference(production, &version_set, reference);
+            .add_reference(&production.name, &version_set, reference);
 
         if !can_be_added {
             reporter.report(

diff --git a/crates/codegen/schema/src/validation/visitors/receivers.rs b/crates/codegen/schema/src/validation/visitors/receivers.rs
@@ -2,8 +2,8 @@ use std::rc::Rc;
 
 use crate::{
     types::{
-        LanguageDefinitionRef, ParserDefinition, ParserRef, PrecedenceParserRef, Production,
-        ProductionRef, ScannerDefinition, ScannerRef, VersionMap,
+        LanguageDefinitionRef, ParserDefinition, ParserRef, PrecedenceParserRef,
+        ProductionDefinition, ProductionRef, ScannerDefinition, ScannerRef, VersionMap,
     },
     validation::visitors::{
         location::LocationRef, reporter::Reporter, visitor::Visitor, VersionSet,
@@ -32,17 +32,30 @@ impl Receiver for ProductionRef {
             return;
         }
 
-        match self.as_ref() {
-            Production::Scanner { version_map, .. } => {
+        self.definition
+            .receive(visitor, language, location, reporter);
+    }
+}
+
+impl Receiver for ProductionDefinition {
+    fn receive(
+        &self,
+        visitor: &mut impl Visitor,
+        language: &LanguageDefinitionRef,
+        location: LocationRef,
+        reporter: &mut Reporter,
+    ) {
+        match self {
+            Self::Scanner { version_map, .. } => {
                 version_map.receive(visitor, language, location, reporter);
             }
-            Production::TriviaParser { version_map, .. } => {
+            Self::TriviaParser { version_map, .. } => {
                 version_map.receive(visitor, language, location, reporter);
             }
-            Production::Parser { version_map, .. } => {
+            Self::Parser { version_map, .. } => {
                 version_map.receive(visitor, language, location, reporter);
             }
-            Production::PrecedenceParser { version_map, .. } => {
+            Self::PrecedenceParser { version_map, .. } => {
                 version_map.receive(visitor, language, location, reporter);
             }
         };

diff --git a/crates/codegen/spec/src/snippets.rs b/crates/codegen/spec/src/snippets.rs
@@ -3,7 +3,8 @@ use std::path::PathBuf;
 use anyhow::Result;
 use codegen_ebnf::EbnfSerializer;
 use codegen_schema::types::{
-    LanguageDefinition, LanguageSection, LanguageTopic, Production, ProductionRef, VersionMap,
+    LanguageDefinition, LanguageSection, LanguageTopic, ProductionDefinition, ProductionRef,
+    VersionMap,
 };
 use codegen_utils::context::CodegenContext;
 use inflector::Inflector;
@@ -48,28 +49,27 @@ impl<'context> Snippets<'context> {
         production: &ProductionRef,
         version: &Version,
     ) -> Option<PathBuf> {
-        let production_name = &production.name();
-        let (section, topic) = self.locate_production(production_name);
+        let (section, topic) = self.locate_production(&production.name);
 
-        let file_name = match production.as_ref() {
-            Production::Scanner { version_map, .. } => match version_map {
+        let file_name = match &production.definition {
+            ProductionDefinition::Scanner { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => Some("unversioned".to_owned()),
                 VersionMap::Versioned(versions) => versions
                     .keys()
                     .rev()
                     .find(|v| *v <= version)
                     .and_then(|v| versions.get(v).unwrap().as_ref().map(|_| v.to_string())),
             },
-            Production::TriviaParser { version_map, .. }
-            | Production::Parser { version_map, .. } => match version_map {
+            ProductionDefinition::TriviaParser { version_map, .. }
+            | ProductionDefinition::Parser { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => Some("unversioned".to_owned()),
                 VersionMap::Versioned(versions) => versions
                     .keys()
                     .rev()
                     .find(|v| *v <= version)
                     .and_then(|v| versions.get(v).unwrap().as_ref().map(|_| v.to_string())),
             },
-            Production::PrecedenceParser { version_map, .. } => match version_map {
+            ProductionDefinition::PrecedenceParser { version_map, .. } => match version_map {
                 VersionMap::Unversioned(_) => Some("unversioned".to_owned()),
                 VersionMap::Versioned(versions) => versions
                     .keys()
@@ -84,7 +84,7 @@ impl<'context> Snippets<'context> {
                 .join("ebnf")
                 .join(&section.path)
                 .join(&topic.path)
-                .join(production_name.to_kebab_case())
+                .join(production.name.to_kebab_case())
                 .join(format!("{file_name}.md"))
         });
     }
@@ -94,7 +94,7 @@ impl<'context> Snippets<'context> {
 
         let language = "ebnf"; // https://pygments.org/languages/
         let class = "slang-ebnf"; // used to select code blocks via JS during runtime
-        let id = production.name(); // used for navigation (generarating URL hashes)
+        let id = &production.name; // used for navigation (generarating URL hashes)
 
         let contents = &EbnfSerializer::serialize_version(self.language, production, version)?;
         snippet.write_code_block(language, class, id, contents);
@@ -106,7 +106,7 @@ impl<'context> Snippets<'context> {
         for section in &self.language.sections {
             for topic in &section.topics {
                 for production in &topic.productions {
-                    if name == production.name() {
+                    if name == production.name {
                         return (section, topic);
                     }
                 }

diff --git a/crates/codegen/syntax/src/char_set.rs b/crates/codegen/syntax/src/char_set.rs
@@ -3,7 +3,7 @@ use std::{
     ops::Range,
 };
 
-use codegen_schema::types::{Production, ScannerDefinition, ScannerRef};
+use codegen_schema::types::{ProductionDefinition, ScannerDefinition, ScannerRef};
 use proc_macro2::TokenStream;
 use quote::quote;
 
@@ -168,15 +168,15 @@ impl CharSet {
 
             ScannerDefinition::Reference(name) => Self::from_scanner(
                 tree,
-                match tree.context.get_tree_by_name(name).production.as_ref() {
-                    Production::Scanner { name, version_map } => {
+                match &tree.context.get_tree_by_name(name).production.definition {
+                    ProductionDefinition::Scanner { version_map } => {
                         version_map.get_for_version(&tree.context.version).expect(
                             &format!("Validation should have ensured: no version of {name} exists for version {version}", version = tree.context.version)
                         ).clone()
                     }
-                    Production::TriviaParser { .. }
-                    | Production::Parser { .. }
-                    | Production::PrecedenceParser { .. } => {
+                    ProductionDefinition::TriviaParser { .. }
+                    | ProductionDefinition::Parser { .. }
+                    | ProductionDefinition::PrecedenceParser { .. } => {
                         unreachable!(
                             "Validation should have ensured: scanners can only reference scanners"
                         )