add generated language-definition.json (NomicFoundation#1206)

cc @AntonyBlakey
OmarTawfik-forks · Dec 31, 2024 · 893a631 · 893a631
1 parent ba075d1
commit 893a631
Show file tree

Hide file tree

Showing 19 changed files with 9,926 additions and 15 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/external_types.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/external_types.rs
@@ -1,9 +1,8 @@
 use std::fmt::Debug;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::rc::Rc;
 
 use indexmap::{IndexMap, IndexSet};
-use infra_utils::paths::PathExtensions;
 use proc_macro2::Ident;
 use semver::Version;
 use syn::parse::ParseStream;
@@ -100,7 +99,7 @@ impl ParseInputTokens for PathBuf {
     fn parse_value(input: ParseStream<'_>, errors: &mut ErrorsCollection) -> Result<Self> {
         let value = String::parse_value(input, errors)?;
 
-        Ok(Path::repo_path(value))
+        Ok(PathBuf::from(value))
     }
 }
 

diff --git a/crates/codegen/language/definition/src/model/built_ins.rs b/crates/codegen/language/definition/src/model/built_ins.rs
@@ -18,23 +18,37 @@ pub enum BuiltIn {
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct BuiltInFunction {
     pub name: String,
+
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     pub parameters: Vec<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub return_type: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct BuiltInType {
     pub name: String,
+
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     pub fields: Vec<BuiltInField>,
+
+    #[serde(skip_serializing_if = "Vec::is_empty")]
     pub functions: Vec<BuiltInFunction>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct BuiltInField {
     pub definition: String,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 }
diff --git a/crates/codegen/language/definition/src/model/manifest.rs b/crates/codegen/language/definition/src/model/manifest.rs
@@ -167,7 +167,8 @@ impl Section {
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Topic {
     pub title: String,
-    pub notes_file: Option<String>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub lexical_context: Option<Identifier>,
 
     pub items: Vec<Item>,

diff --git a/crates/codegen/language/definition/src/model/nonterminals/enum_.rs b/crates/codegen/language/definition/src/model/nonterminals/enum_.rs
@@ -8,6 +8,7 @@ use crate::model::{Identifier, VersionSpecifier};
 pub struct EnumItem {
     pub name: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
     pub variants: Vec<EnumVariant>,
@@ -18,5 +19,6 @@ pub struct EnumItem {
 pub struct EnumVariant {
     pub reference: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 }
diff --git a/crates/codegen/language/definition/src/model/nonterminals/field.rs b/crates/codegen/language/definition/src/model/nonterminals/field.rs
@@ -6,7 +6,10 @@ use crate::model::{Identifier, VersionSpecifier};
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct FieldsErrorRecovery {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub terminator: Option<Identifier>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub delimiters: Option<FieldDelimiters>,
 }
 
@@ -15,12 +18,14 @@ pub struct FieldsErrorRecovery {
 pub struct FieldDelimiters {
     pub open: Identifier,
     pub close: Identifier,
+
     /// How many tokens have to be matched to trigger the error recovery.
     /// For ambiguous syntaxes this needs to be set to at least N, where N
     /// is the token lookahead required to disambiguate the syntax.
     ///
     /// By default, we assume no lookahead (0) is required to recover from
     /// unrecognized body between delimiters, so it's always triggered.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub terminals_matched_acceptance_threshold: Option<u8>,
 }
 
@@ -33,6 +38,7 @@ pub enum Field {
     Optional {
         reference: Identifier,
 
+        #[serde(skip_serializing_if = "Option::is_none")]
         enabled: Option<VersionSpecifier>,
     },
 }
diff --git a/crates/codegen/language/definition/src/model/nonterminals/precedence.rs b/crates/codegen/language/definition/src/model/nonterminals/precedence.rs
@@ -11,6 +11,7 @@ use crate::model::{Field, FieldsErrorRecovery, Identifier, VersionSpecifier};
 pub struct PrecedenceItem {
     pub name: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
     pub precedence_expressions: Vec<Rc<PrecedenceExpression>>,
@@ -30,9 +31,12 @@ pub struct PrecedenceExpression {
 pub struct PrecedenceOperator {
     pub model: OperatorModel,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub error_recovery: Option<FieldsErrorRecovery>,
+
     pub fields: IndexMap<Identifier, Field>,
 }
 
@@ -50,5 +54,6 @@ pub enum OperatorModel {
 pub struct PrimaryExpression {
     pub reference: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 }
diff --git a/crates/codegen/language/definition/src/model/nonterminals/repeated.rs b/crates/codegen/language/definition/src/model/nonterminals/repeated.rs
@@ -9,7 +9,9 @@ pub struct RepeatedItem {
     pub name: Identifier,
     pub reference: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub allow_empty: Option<bool>,
 }
diff --git a/crates/codegen/language/definition/src/model/nonterminals/separated.rs b/crates/codegen/language/definition/src/model/nonterminals/separated.rs
@@ -10,7 +10,9 @@ pub struct SeparatedItem {
     pub reference: Identifier,
     pub separator: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub allow_empty: Option<bool>,
 }
diff --git a/crates/codegen/language/definition/src/model/nonterminals/struct_.rs b/crates/codegen/language/definition/src/model/nonterminals/struct_.rs
@@ -9,8 +9,11 @@ use crate::model::{Field, FieldsErrorRecovery, Identifier, VersionSpecifier};
 pub struct StructItem {
     pub name: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub error_recovery: Option<FieldsErrorRecovery>,
+
     pub fields: IndexMap<Identifier, Field>,
 }
diff --git a/crates/codegen/language/definition/src/model/terminals/fragment.rs b/crates/codegen/language/definition/src/model/terminals/fragment.rs
@@ -8,6 +8,7 @@ use crate::model::{Identifier, Scanner, VersionSpecifier};
 pub struct FragmentItem {
     pub name: Identifier,
 
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
     pub scanner: Scanner,

diff --git a/crates/codegen/language/definition/src/model/terminals/keyword.rs b/crates/codegen/language/definition/src/model/terminals/keyword.rs
@@ -16,10 +16,14 @@ pub struct KeywordItem {
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct KeywordDefinition {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
+
     /// When the keyword is reserved, i.e. can't be used in other position (e.g. as a name)
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub reserved: Option<VersionSpecifier>,
-    // Underlying keyword scanner (i.e. identifier scanner)
+
+    /// Underlying keyword scanner (i.e. identifier scanner)
     pub value: KeywordValue,
 }
 

diff --git a/crates/codegen/language/definition/src/model/terminals/token.rs b/crates/codegen/language/definition/src/model/terminals/token.rs
@@ -14,6 +14,7 @@ pub struct TokenItem {
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct TokenDefinition {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub enabled: Option<VersionSpecifier>,
 
     pub scanner: Scanner,

diff --git a/crates/codegen/language/tests/src/pass/tiny_language.rs b/crates/codegen/language/tests/src/pass/tiny_language.rs
@@ -1,10 +1,7 @@
-use std::path::Path;
-
 use codegen_language_definition::model::{
     Field, Item, Language, Scanner, Section, StructItem, TokenDefinition, TokenItem, Topic,
     TriviaParser,
 };
-use infra_utils::paths::PathExtensions;
 use semver::Version;
 
 codegen_language_macros::compile!(Language(
@@ -49,8 +46,8 @@ fn definition() {
         tiny::TinyDefinition::create(),
         Language {
             name: "Tiny".into(),
-            documentation_dir: Path::repo_path("tiny/docs"),
-            binding_rules_file: Path::repo_path("tiny/bindings/rules.msgb"),
+            documentation_dir: "tiny/docs".into(),
+            binding_rules_file: "tiny/bindings/rules.msgb".into(),
             file_extension: Some(".tiny".into()),
             root_item: "Foo".into(),
             leading_trivia: TriviaParser::Sequence { parsers: [].into() },
@@ -65,7 +62,6 @@ fn definition() {
                 title: "Section One".into(),
                 topics: vec![Topic {
                     title: "Topic One".into(),
-                    notes_file: None,
                     lexical_context: None,
                     items: [
                         Item::Struct {

diff --git a/crates/codegen/runtime/generator/src/bindings/mod.rs b/crates/codegen/runtime/generator/src/bindings/mod.rs
@@ -1,4 +1,5 @@
 use std::collections::BTreeSet;
+use std::path::Path;
 
 use anyhow::Result;
 use codegen_language_definition::model;
@@ -17,9 +18,10 @@ pub struct BindingsModel {
 impl BindingsModel {
     pub fn from_language(language: &model::Language) -> Result<Self> {
         // We use `CodegenFileSystem` here to ensure the rules are rebuilt if the rules file changes
-        let binding_rules_dir = language.binding_rules_file.unwrap_parent();
+        let binding_rules_file = Path::repo_path(&language.binding_rules_file);
+        let binding_rules_dir = binding_rules_file.unwrap_parent();
         let mut fs = CodegenFileSystem::new(binding_rules_dir)?;
-        let binding_rules_source = fs.read_file(&language.binding_rules_file)?;
+        let binding_rules_source = fs.read_file(&binding_rules_file)?;
         let built_ins_versions = language.collect_built_ins_versions();
         let file_extension = language.file_extension.clone().unwrap_or_default();
 

diff --git a/crates/codegen/spec/Cargo.toml b/crates/codegen/spec/Cargo.toml
@@ -14,6 +14,7 @@ Inflector = { workspace = true }
 infra_utils = { workspace = true }
 itertools = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 
 [lints]
 workspace = true
diff --git a/crates/codegen/spec/src/generators/topic_page.rs b/crates/codegen/spec/src/generators/topic_page.rs
@@ -53,7 +53,7 @@ pub fn generate_topic_page(
         }
     }
 
-    let documentation_dir = model.language.documentation_dir.strip_repo_root()?;
+    let documentation_dir = &model.language.documentation_dir;
 
     writeln!(buffer)?;
     writeln!(

diff --git a/crates/codegen/spec/src/lib.rs b/crates/codegen/spec/src/lib.rs
@@ -15,6 +15,7 @@ use std::rc::Rc;
 use anyhow::Result;
 use codegen_language_definition::model::Language;
 use infra_utils::codegen::CodegenFileSystem;
+use infra_utils::paths::PathExtensions;
 
 use crate::generators::grammar_ebnf::generate_grammar_ebnf;
 use crate::generators::navigation::{SpecDir, SpecPage};
@@ -26,7 +27,13 @@ pub struct Spec;
 
 impl Spec {
     pub fn generate(language: Rc<Language>, output_dir: &Path) -> Result<()> {
-        let mut fs = CodegenFileSystem::new(&language.documentation_dir)?;
+        let documentation_dir = Path::repo_path(&language.documentation_dir);
+        let mut fs = CodegenFileSystem::new(documentation_dir)?;
+
+        fs.write_file(
+            output_dir.join("language-definition.json"),
+            serde_json::to_string(&language)?,
+        )?;
 
         let model = SpecModel::build(language);
         let public_dir = Self::generate_public_dir(&model)?;