Implement (contextual) keywords and use their versioning from v2

crates/codegen/grammar/src/grammar.rs

@@ -4,7 +4,7 @@ use semver::Version;
 
 use crate::parser_definition::{ParserDefinitionRef, TriviaParserDefinitionRef};
 use crate::visitor::{GrammarVisitor, Visitable};
-use crate::{PrecedenceParserDefinitionRef, ScannerDefinitionRef};
+use crate::{KeywordScannerDefinitionRef, PrecedenceParserDefinitionRef, ScannerDefinitionRef};
 
 pub struct Grammar {
     pub name: String,
@@ -36,6 +36,7 @@ impl Grammar {
 #[derive(Clone)]
 pub enum GrammarElement {
     ScannerDefinition(ScannerDefinitionRef),
+    KeywordScannerDefinition(KeywordScannerDefinitionRef),
     TriviaParserDefinition(TriviaParserDefinitionRef),
     ParserDefinition(ParserDefinitionRef),
     PrecedenceParserDefinition(PrecedenceParserDefinitionRef),
@@ -45,6 +46,7 @@ impl GrammarElement {
     pub fn name(&self) -> &'static str {
         match self {
             Self::ScannerDefinition(scanner) => scanner.name(),
+            Self::KeywordScannerDefinition(scanner) => scanner.name(),
             Self::TriviaParserDefinition(trivia_parser) => trivia_parser.name(),
             Self::ParserDefinition(parser) => parser.name(),
             Self::PrecedenceParserDefinition(precedence_parser) => precedence_parser.name(),
@@ -80,6 +82,7 @@ impl Visitable for GrammarElement {
     fn accept_visitor<V: GrammarVisitor>(&self, visitor: &mut V) {
         match self {
             Self::ScannerDefinition(scanner) => scanner.accept_visitor(visitor),
+            Self::KeywordScannerDefinition(scanner) => scanner.accept_visitor(visitor),
             Self::TriviaParserDefinition(trivia_parser) => trivia_parser.accept_visitor(visitor),
             Self::ParserDefinition(parser) => parser.accept_visitor(visitor),
             Self::PrecedenceParserDefinition(precedence_parser) => {

crates/codegen/grammar/src/parser_definition.rs

@@ -2,7 +2,10 @@ use std::fmt::Debug;
 use std::rc::Rc;
 
 use crate::visitor::{GrammarVisitor, Visitable};
-use crate::{PrecedenceParserDefinitionRef, ScannerDefinitionRef, VersionQualityRange};
+use crate::{
+    KeywordScannerDefinitionRef, PrecedenceParserDefinitionRef, ScannerDefinitionRef,
+    VersionQualityRange,
+};
 
 /// A named wrapper, used to give a name to a [`ParserDefinitionNode`].
 #[derive(Clone, Debug)]
@@ -59,6 +62,7 @@ pub enum ParserDefinitionNode {
     Sequence(Vec<Named<Self>>),
     Choice(Named<Vec<Self>>),
     ScannerDefinition(ScannerDefinitionRef),
+    KeywordScannerDefinition(KeywordScannerDefinitionRef),
     TriviaParserDefinition(TriviaParserDefinitionRef),
     ParserDefinition(ParserDefinitionRef),
     PrecedenceParserDefinition(PrecedenceParserDefinitionRef),
@@ -128,6 +132,7 @@ impl Visitable for ParserDefinitionNode {
             }
 
             Self::ScannerDefinition(_)
+            | Self::KeywordScannerDefinition(_)
             | Self::TriviaParserDefinition(_)
             | Self::ParserDefinition(_)
             | Self::PrecedenceParserDefinition(_) => {}

crates/codegen/grammar/src/scanner_definition.rs

@@ -65,3 +65,98 @@ impl Visitable for ScannerDefinitionNode {
         }
     }
 }
+
+pub trait KeywordScannerDefinition: Debug {
+    fn name(&self) -> &'static str;
+    fn identifier_scanner(&self) -> &'static str;
+    fn definitions(&self) -> &[KeywordScannerDefinitionVersionedNode];
+}
+
+pub type KeywordScannerDefinitionRef = Rc<dyn KeywordScannerDefinition>;
+
+impl Visitable for KeywordScannerDefinitionRef {
+    fn accept_visitor<V: GrammarVisitor>(&self, visitor: &mut V) {
+        visitor.keyword_scanner_definition_enter(self);
+    }
+}
+
+#[derive(Debug)]
+pub struct KeywordScannerDefinitionVersionedNode {
+    // Underlying keyword scanner (i.e. identifier scanner)
+    pub value: KeywordScannerDefinitionNode,
+    /// When the keyword scanner is enabled
+    pub enabled: Vec<VersionQualityRange>,
+    /// When the keyword is reserved, i.e. can't be used in other position (e.g. as a name)
+    pub reserved: Vec<VersionQualityRange>,
+}
+
+#[derive(Clone, Debug)]
+pub enum KeywordScannerDefinitionNode {
+    Optional(Box<Self>),
+    Sequence(Vec<Self>),
+    Choice(Vec<Self>),
+    Atom(String),
+    // No repeatable combinators, because keywords are assumed to be finite
+}
+
+impl From<KeywordScannerDefinitionNode> for ScannerDefinitionNode {
+    fn from(val: KeywordScannerDefinitionNode) -> Self {
+        match val {
+            KeywordScannerDefinitionNode::Optional(node) => {
+                ScannerDefinitionNode::Optional(Box::new((*node).into()))
+            }
+            KeywordScannerDefinitionNode::Sequence(nodes) => {
+                ScannerDefinitionNode::Sequence(nodes.into_iter().map(Into::into).collect())
+            }
+            KeywordScannerDefinitionNode::Atom(string) => ScannerDefinitionNode::Literal(string),
+            KeywordScannerDefinitionNode::Choice(nodes) => {
+                ScannerDefinitionNode::Choice(nodes.into_iter().map(Into::into).collect())
+            }
+        }
+    }
+}
+
+/// A [`KeywordScannerDefinitionRef`] that only has a single atom value.
+///
+/// The main usage for this type is to construct a keyword trie in parser generator, as trie will
+/// only work with single atom values and keyword promotion needs to additionally account for
+/// keyword reservation, rather than just literal presence.
+#[derive(Clone)]
+pub struct KeywordScannerAtomic(KeywordScannerDefinitionRef);
+
+impl KeywordScannerAtomic {
+    /// Wraps the keyword scanner definition if it is a single atom value.
+    pub fn try_from_def(def: &KeywordScannerDefinitionRef) -> Option<Self> {
+        match def.definitions() {
+            [KeywordScannerDefinitionVersionedNode {
+                value: KeywordScannerDefinitionNode::Atom(_),
+                ..
+            }] => Some(Self(def.clone())),
+            _ => None,
+        }
+    }
+}
+
+impl std::ops::Deref for KeywordScannerAtomic {
+    type Target = KeywordScannerDefinitionRef;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl KeywordScannerAtomic {
+    pub fn definition(&self) -> &KeywordScannerDefinitionVersionedNode {
+        let def = &self.0.definitions().get(0);
+        def.expect("KeywordScannerAtomic should have exactly one definition")
+    }
+    pub fn value(&self) -> &str {
+        match self.definition() {
+            KeywordScannerDefinitionVersionedNode {
+                value: KeywordScannerDefinitionNode::Atom(atom),
+                ..
+            } => atom,
+            _ => unreachable!("KeywordScannerAtomic should have a single atom value"),
+        }
+    }
+}

crates/codegen/grammar/src/visitor.rs

@@ -1,14 +1,15 @@
 use crate::{
-    Grammar, ParserDefinitionNode, ParserDefinitionRef, PrecedenceParserDefinitionNode,
-    PrecedenceParserDefinitionRef, ScannerDefinitionNode, ScannerDefinitionRef,
-    TriviaParserDefinitionRef,
+    Grammar, KeywordScannerDefinitionRef, ParserDefinitionNode, ParserDefinitionRef,
+    PrecedenceParserDefinitionNode, PrecedenceParserDefinitionRef, ScannerDefinitionNode,
+    ScannerDefinitionRef, TriviaParserDefinitionRef,
 };
 
 pub trait GrammarVisitor {
     fn grammar_enter(&mut self, _grammar: &Grammar) {}
     fn grammar_leave(&mut self, _grammar: &Grammar) {}
 
     fn scanner_definition_enter(&mut self, _scanner: &ScannerDefinitionRef) {}
+    fn keyword_scanner_definition_enter(&mut self, _scanner: &KeywordScannerDefinitionRef) {}
     fn trivia_parser_definition_enter(&mut self, _trivia_parser: &TriviaParserDefinitionRef) {}
     fn parser_definition_enter(&mut self, _parser: &ParserDefinitionRef) {}
     fn precedence_parser_definition_enter(&mut self, _parser: &PrecedenceParserDefinitionRef) {}