Rust & Typescript Cursors for the CST

NomicFoundation · Jul 20, 2023 · 22128d5 · 22128d5
1 parent ef0a2a5
commit 22128d5
Show file tree

Hide file tree

Showing 54 changed files with 2,207 additions and 2,084 deletions.
diff --git a/.changeset/cuddly-avocados-sing.md b/.changeset/cuddly-avocados-sing.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+Add a Rust Cursor API and refactor the Rust Visitor API to run on top of it.
diff --git a/.changeset/metal-fans-raise.md b/.changeset/metal-fans-raise.md
@@ -0,0 +1,5 @@
+---
+"changelog": patch
+---
+
+Add TokenNode.text to the TS API.
diff --git a/.changeset/modern-tools-shake.md b/.changeset/modern-tools-shake.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+Move Visitor et al to node:: namespace, which is where Cursor is.
diff --git a/.changeset/neat-paws-unite.md b/.changeset/neat-paws-unite.md
@@ -0,0 +1,5 @@
+---
+"changelog": minor
+---
+
+Rename `range` functions that return a TextRange to `text_range`
diff --git a/.changeset/plenty-terms-rest.md b/.changeset/plenty-terms-rest.md
@@ -0,0 +1,5 @@
+---
+"changelog": patch
+---
+
+Add first pass of Typescript binding to the Cursor API, but no TS Visitor yet.
diff --git a/crates/codegen/syntax/src/code_generator.rs b/crates/codegen/syntax/src/code_generator.rs
@@ -441,6 +441,15 @@ impl CodeGenerator {
             )
             .unwrap();
 
+        codegen
+            .copy_file(
+                &codegen
+                    .repo_root
+                    .join("crates/codegen/syntax_templates/src/shared/cursor.rs"),
+                &output_dir.join("cursor.rs"),
+            )
+            .unwrap();
+
         codegen
             .copy_file(
                 &codegen
@@ -513,6 +522,15 @@ impl CodeGenerator {
             )
             .unwrap();
 
+        codegen
+            .copy_file(
+                &codegen
+                    .repo_root
+                    .join("crates/codegen/syntax_templates/src/shared/visitor.rs"),
+                &output_dir.join("visitor.rs"),
+            )
+            .unwrap();
+
         {
             // Use `format!` here because the content contains comments, that `quote!` throws away.
             let content = format!(

diff --git a/crates/codegen/syntax/src/rust_lib_code_generator.rs b/crates/codegen/syntax/src/rust_lib_code_generator.rs
@@ -29,28 +29,20 @@ impl CodeGenerator {
                mod stream;
 
                pub mod cst;
-               pub mod cst_visitor;
+               pub mod cursor;
                pub mod kinds;
                pub mod language;
                pub mod parse_error;
                pub mod parse_output;
                pub mod text_index;
+               pub mod visitor;
             };
 
             codegen
                 .write_file(&output_dir.join("mod.rs"), &content.to_string())
                 .unwrap();
         }
 
-        codegen
-            .copy_file(
-                &codegen
-                    .repo_root
-                    .join("crates/codegen/syntax_templates/src/rust/cst_visitor.rs"),
-                &output_dir.join("cst_visitor.rs"),
-            )
-            .unwrap();
-
         codegen
             .copy_file(
                 &codegen

diff --git a/crates/codegen/syntax/src/typescript_lib_code_generator.rs b/crates/codegen/syntax/src/typescript_lib_code_generator.rs
@@ -29,12 +29,15 @@ impl CodeGenerator {
                mod stream;
 
                pub mod cst;
-               pub mod cst_types;
+               pub mod cst_ts_wrappers;
+               pub mod cursor;
+               pub mod cursor_ts_wrappers;
                pub mod kinds;
                pub mod language;
                pub mod parse_error;
                pub mod parse_output;
                pub mod text_index;
+               pub mod visitor;
             };
 
             codegen
@@ -46,8 +49,17 @@ impl CodeGenerator {
             .copy_file(
                 &codegen
                     .repo_root
-                    .join("crates/codegen/syntax_templates/src/typescript/cst_types.rs"),
-                &output_dir.join("cst_types.rs"),
+                    .join("crates/codegen/syntax_templates/src/typescript/cst_ts_wrappers.rs"),
+                &output_dir.join("cst_ts_wrappers.rs"),
+            )
+            .unwrap();
+
+        codegen
+            .copy_file(
+                &codegen
+                    .repo_root
+                    .join("crates/codegen/syntax_templates/src/typescript/cursor_ts_wrappers.rs"),
+                &output_dir.join("cursor_ts_wrappers.rs"),
             )
             .unwrap();
 

diff --git a/crates/codegen/syntax_templates/src/rust/cst_visitor.rs b/crates/codegen/syntax_templates/src/rust/cst_visitor.rs
diff --git a/crates/codegen/syntax_templates/src/rust/mod.rs b/crates/codegen/syntax_templates/src/rust/mod.rs
@@ -1,5 +1,7 @@
 #[path = "../shared/cst.rs"]
 pub mod cst;
+#[path = "../shared/cursor.rs"]
+pub mod cursor;
 #[path = "../shared/language.rs"]
 pub mod language;
 #[path = "../shared/parse_error.rs"]
@@ -18,7 +20,8 @@ pub mod scanner_macros;
 pub mod stream;
 #[path = "../shared/text_index.rs"]
 pub mod text_index;
+#[path = "../shared/visitor.rs"]
+pub mod visitor;
 
-pub mod cst_visitor;
 pub mod kinds;
 pub mod parse_output;
diff --git a/crates/codegen/syntax_templates/src/rust/parse_output.rs b/crates/codegen/syntax_templates/src/rust/parse_output.rs
@@ -24,13 +24,13 @@ impl ParseOutput {
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct ParseError {
-    pub(crate) range: TextRange,
+    pub(crate) text_range: TextRange,
     pub(crate) tokens_that_would_have_allowed_more_progress: Vec<TokenKind>,
 }
 
 impl ParseError {
-    pub fn range(&self) -> &TextRange {
-        return &self.range;
+    pub fn text_range(&self) -> &TextRange {
+        return &self.text_range;
     }
 
     pub fn tokens_that_would_have_allowed_more_progress(&self) -> Vec<String> {

diff --git a/crates/codegen/syntax_templates/src/shared/cst.rs b/crates/codegen/syntax_templates/src/shared/cst.rs
@@ -3,6 +3,7 @@ use std::rc::Rc;
 use serde::Serialize;
 
 use super::{
+    cursor::Cursor,
     kinds::{RuleKind, TokenKind},
     text_index::TextIndex,
 };
@@ -52,4 +53,50 @@ impl Node {
             Self::Token(node) => (&node.text).into(),
         }
     }
+
+    pub fn cursor(&self) -> Cursor {
+        Cursor::new(self.clone())
+    }
+
+    pub fn as_token_with_kind(&self, kinds: &[TokenKind]) -> Option<&Rc<TokenNode>> {
+        if let Node::Token(token_node) = self {
+            if kinds.contains(&token_node.kind) {
+                return Some(token_node);
+            }
+        }
+        return None;
+    }
+
+    pub fn as_token_matching<F: Fn(&Rc<TokenNode>) -> bool>(
+        &self,
+        predicate: F,
+    ) -> Option<&Rc<TokenNode>> {
+        if let Node::Token(token_node) = self {
+            if predicate(&token_node) {
+                return Some(token_node);
+            }
+        }
+        return None;
+    }
+
+    pub fn as_rule_with_kind(&self, kinds: &[RuleKind]) -> Option<&Rc<RuleNode>> {
+        if let Node::Rule(rule_node) = self {
+            if kinds.contains(&rule_node.kind) {
+                return Some(rule_node);
+            }
+        }
+        return None;
+    }
+
+    pub fn as_rule_matching<F: Fn(&Rc<RuleNode>) -> bool>(
+        &self,
+        predicate: F,
+    ) -> Option<&Rc<RuleNode>> {
+        if let Node::Rule(rule_node) = self {
+            if predicate(&rule_node) {
+                return Some(rule_node);
+            }
+        }
+        return None;
+    }
 }