From ed1c8187c6fdf80f019bea2ea1cc894d032ccb4f Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 10 Jan 2023 01:49:44 +0800 Subject: [PATCH] Rewrite line_doc extract in pest_generator, avoid change AST. --- generator/src/generator.rs | 69 ++++++++++++++++++---- generator/src/lib.rs | 95 +++++++++++++++++++++++++++++- generator/tests/test.pest | 18 ++++++ grammars/src/grammars/json.pest | 1 + meta/src/ast.rs | 3 - meta/src/optimizer/concatenator.rs | 8 +-- meta/src/optimizer/factorizer.rs | 8 +-- meta/src/optimizer/lister.rs | 8 +-- meta/src/optimizer/mod.rs | 27 --------- meta/src/optimizer/restorer.rs | 19 +----- meta/src/optimizer/rotater.rs | 8 +-- meta/src/optimizer/skipper.rs | 8 +-- meta/src/optimizer/unroller.rs | 8 +-- meta/src/parser.rs | 52 ++++------------ 14 files changed, 187 insertions(+), 145 deletions(-) create mode 100644 generator/tests/test.pest diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 2f4173e9..a2b33892 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -17,13 +17,28 @@ use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; -pub fn generate( +#[derive(Debug)] +pub(crate) struct DocComment<'a> { + pub(crate) grammar_docs: Vec<&'a str>, + pub(crate) line_docs: Vec>, + pub(crate) rules: Vec, +} + +impl DocComment<'_> { + fn line_docs_for_rule<'a>(&self, rule_name: &str) -> Option { + let idx = self.rules.iter().position(|r| r.name == rule_name)?; + + self.line_docs.get(idx).map(|comments| comments.join("\n")) + } +} + +pub(crate) fn generate( name: Ident, generics: &Generics, path: Option, rules: Vec, defaults: Vec<&str>, - grammar_docs: Vec<&str>, + doc_comment: &DocComment<'_>, include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); @@ -37,7 +52,7 @@ pub fn generate( } else { quote!() }; - let rule_enum = generate_enum(&rules, grammar_docs, uses_eoi); + let rule_enum = generate_enum(&rules, doc_comment, uses_eoi); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -182,15 +197,21 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: bool) -> TokenStream { +fn generate_enum( + rules: &[OptimizedRule], + doc_comment: &DocComment<'_>, + uses_eoi: bool, +) -> TokenStream { let rules = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); - if rule.comments.is_empty() { + + let comments = doc_comment.line_docs_for_rule(&rule.name); + let comments = comments.unwrap_or_else(|| "".to_owned()); + if comments.is_empty() { quote! { #rule_name } } else { - let comments = rule.comments.join("\n"); quote! { #[doc = #comments] #rule_name @@ -198,7 +219,7 @@ fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: boo } }); - let grammar_docs = grammar_docs.join("\n"); + let grammar_docs = doc_comment.grammar_docs.join("\n"); if uses_eoi { quote! { #[doc = #grammar_docs] @@ -686,11 +707,20 @@ mod tests { name: "f".to_owned(), ty: RuleType::Normal, expr: OptimizedExpr::Ident("g".to_owned()), - comments: vec!["This is rule comment".to_owned()], }]; + let doc_comment = &DocComment { + grammar_docs: vec!["Rule doc", "hello"], + line_docs: vec![vec!["This is rule comment"]], + rules: vec![Rule { + name: "f".to_owned(), + ty: RuleType::Normal, + expr: Expr::Ident("g".to_owned()), + }], + }; + assert_eq!( - generate_enum(&rules, vec!["Rule doc", "hello"], false).to_string(), + generate_enum(&rules, doc_comment, false).to_string(), quote! { #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] @@ -988,16 +1018,31 @@ mod tests { name: "a".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Str("b".to_owned()), - comments: vec![], }, OptimizedRule { name: "if".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Ident("a".to_owned()), - comments: vec!["If statement".to_owned()], }, ]; + let doc_comment = &DocComment { + line_docs: vec![vec![], vec!["If statement"]], + grammar_docs: vec!["This is Rule doc", "This is second line"], + rules: vec![ + Rule { + name: "a".to_owned(), + ty: RuleType::Silent, + expr: Expr::Str("b".to_owned()), + }, + Rule { + name: "if".to_owned(), + ty: RuleType::Silent, + expr: Expr::Str("b".to_owned()), + }, + ], + }; + let defaults = vec!["ANY"]; let result = result_type(); let box_ty = box_type(); @@ -1005,7 +1050,7 @@ mod tests { current_dir.push("test.pest"); let test_path = current_dir.to_str().expect("path contains invalid unicode"); assert_eq!( - generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, vec!["This is Rule doc", "This is second line"], true).to_string(), + generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, doc_comment, true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path); diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 87818a6d..d431c0b9 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -37,6 +37,8 @@ mod generator; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; +use generator::DocComment; + /// Processes the derive/proc macro input and generates the corresponding parser based /// on the parsed grammar. If `include_grammar` is set to true, it'll generate an explicit /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). @@ -92,10 +94,17 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { }; let grammar_docs = consume_grammar_doc(pairs.clone()); + let line_docs = consume_line_docs(pairs.clone()); let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); let ast = unwrap_or_report(parser::consume_rules(pairs)); - let optimized = optimizer::optimize(ast); + let optimized = optimizer::optimize(ast.clone()); + + let doc_comment = DocComment { + grammar_docs, + line_docs, + rules: ast, + }; generator::generate( name, @@ -103,7 +112,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { path, optimized, defaults, - grammar_docs, + &doc_comment, include_grammar, ) } @@ -119,6 +128,32 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { docs } +fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec> { + let mut docs = vec![]; + let mut comments = vec![]; + + for pair in pairs { + if pair.as_rule() == Rule::grammar_rule { + if let Some(inner) = pair.into_inner().next() { + if inner.as_rule() == Rule::line_doc { + comments.push(inner.as_str()[3..inner.as_str().len()].trim()); + continue; + } else { + docs.push(comments); + comments = vec![]; + } + } + } + + if !comments.is_empty() { + docs.push(comments); + comments = vec![]; + } + } + + docs +} + fn read_file>(path: P) -> io::Result { let mut file = File::open(path.as_ref())?; let mut string = String::new(); @@ -177,9 +212,12 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { #[cfg(test)] mod tests { + use super::consume_line_docs; use super::parse_derive; use super::GrammarSource; + use pest_meta::parser::{self, Rule}; + #[test] fn derive_inline_file() { let definition = " @@ -247,4 +285,57 @@ mod tests { let ast = syn::parse_str(definition).unwrap(); parse_derive(ast); } + + #[test] + fn test_consume_line_docs() { + let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { + Ok(pairs) => pairs, + Err(_) => panic!("error parsing tests/test.pest"), + }; + + let line_docs = consume_line_docs(pairs); + assert_eq!( + vec![ + vec!["Matches foo str, e.g.: `foo`"], + vec!["Matches bar str,", "e.g: `bar` or `foobar`"], + vec![], + vec!["Matches dar", "Match dar description"] + ], + line_docs + ); + } + + #[test] + fn test_generate_doc() { + let input = quote! { + #[derive(Parser)] + #[grammar = "../tests/test.pest"] + pub struct TestParser; + }; + + let token = super::derive_parser(input, true); + + let expected = quote! { + #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser."] + #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + + pub enum Rule { + #[doc = "Matches foo str, e.g.: `foo`"] + r#foo, + #[doc = "Matches bar str,\ne.g: `bar` or `foobar`"] + r#bar, + r#bar1, + #[doc = "Matches dar\nMatch dar description"] + r#dar + } + }; + + assert!( + token.to_string().contains(expected.to_string().as_str()), + "{}\n\nExpected to contains:\n{}", + token, + expected + ); + } } diff --git a/generator/tests/test.pest b/generator/tests/test.pest new file mode 100644 index 00000000..28f6b88a --- /dev/null +++ b/generator/tests/test.pest @@ -0,0 +1,18 @@ +//! A parser for JSON file. +//! And this is a example for JSON parser. + +/// Matches foo str, e.g.: `foo` +foo = { "foo" } + +/// Matches bar str, +/// e.g: `bar` or `foobar` + +bar = { "bar" | "foobar" } + +bar1 = { "bar1" } + +/// Matches dar + +/// Match dar description + +dar = { "da" } \ No newline at end of file diff --git a/grammars/src/grammars/json.pest b/grammars/src/grammars/json.pest index 2692e8ab..4d0d300f 100644 --- a/grammars/src/grammars/json.pest +++ b/grammars/src/grammars/json.pest @@ -12,6 +12,7 @@ json = { SOI ~ (object | array) ~ EOI } /// Matches object, e.g.: `{ "foo": "bar" }` +/// Foobar object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" } pair = { string ~ ":" ~ value } diff --git a/meta/src/ast.rs b/meta/src/ast.rs index 781855d1..ffac8ea7 100644 --- a/meta/src/ast.rs +++ b/meta/src/ast.rs @@ -10,7 +10,6 @@ //! Types for the pest's abstract syntax tree. /// A grammar rule -#[non_exhaustive] #[derive(Clone, Debug, Eq, PartialEq)] pub struct Rule { /// The name of the rule @@ -19,8 +18,6 @@ pub struct Rule { pub ty: RuleType, /// The rule's expression pub expr: Expr, - /// Doc comments of the rule - pub(crate) comments: Vec, } /// All possible rule types diff --git a/meta/src/optimizer/concatenator.rs b/meta/src/optimizer/concatenator.rs index 3e991987..31d3aa53 100644 --- a/meta/src/optimizer/concatenator.rs +++ b/meta/src/optimizer/concatenator.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn concatenate(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -34,6 +29,5 @@ pub fn concatenate(rule: Rule) -> Rule { expr } }), - comments, } } diff --git a/meta/src/optimizer/factorizer.rs b/meta/src/optimizer/factorizer.rs index a653bb4f..5481870b 100644 --- a/meta/src/optimizer/factorizer.rs +++ b/meta/src/optimizer/factorizer.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn factor(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -52,6 +47,5 @@ pub fn factor(rule: Rule) -> Rule { expr => expr, } }), - comments, } } diff --git a/meta/src/optimizer/lister.rs b/meta/src/optimizer/lister.rs index b9f7ae82..e1988503 100644 --- a/meta/src/optimizer/lister.rs +++ b/meta/src/optimizer/lister.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn list(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -43,6 +38,5 @@ pub fn list(rule: Rule) -> Rule { expr => expr, } }), - comments, } } diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs index 89ef7cab..2038753b 100644 --- a/meta/src/optimizer/mod.rs +++ b/meta/src/optimizer/mod.rs @@ -80,7 +80,6 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule { name: rule.name, ty: rule.ty, expr: to_optimized(rule.expr), - comments: rule.comments, } } @@ -100,8 +99,6 @@ pub struct OptimizedRule { pub ty: RuleType, /// The optimized expression of the rule. pub expr: OptimizedExpr, - /// The doc comments of the rule. - pub comments: Vec, } /// The optimized version of the pest AST's `Expr`. @@ -322,7 +319,6 @@ mod tests { ), Str(String::from("d")) )), - comments: vec![], }] }; let rotated = { @@ -337,7 +333,6 @@ mod tests { Choice(Str(String::from("c")), Str(String::from("d"))) ) )), - comments: vec![], }] }; @@ -355,14 +350,12 @@ mod tests { NegPred(Choice(Str(String::from("a")), Str(String::from("b")))), Ident(String::from("ANY")) ))), - comments: vec![], }] }; let skipped = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Skip(vec![String::from("a"), String::from("b")]), - comments: vec![], }]; assert_eq!(optimize(rules), skipped); @@ -379,14 +372,12 @@ mod tests { Seq(Str(String::from("a")), Str(String::from("b"))), Seq(Str(String::from("c")), Str(String::from("d"))) )), - comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Str(String::from("abcd")), - comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -398,7 +389,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepExact(Box::new(Expr::Ident(String::from("a"))), 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -409,7 +399,6 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("a"))) )), - comments: vec![], }] }; @@ -422,7 +411,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMax(Box::new(Expr::Str("a".to_owned())), 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -433,7 +421,6 @@ mod tests { Opt(Str(String::from("a"))), Seq(Opt(Str(String::from("a"))), Opt(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -446,7 +433,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMin(Box::new(Expr::Str("a".to_owned())), 2), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -457,7 +443,6 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Rep(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -470,7 +455,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMinMax(Box::new(Expr::Str("a".to_owned())), 2, 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -488,7 +472,6 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Opt(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -506,14 +489,12 @@ mod tests { Seq(Insens(String::from("a")), Insens(String::from("b"))), Seq(Insens(String::from("c")), Insens(String::from("d"))) )), - comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Insens(String::from("abcd")), - comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -536,7 +517,6 @@ mod tests { Ident(String::from("d")) ) )), - comments: vec![], }] }; let optimized = { @@ -551,7 +531,6 @@ mod tests { Choice(Ident(String::from("c")), Ident(String::from("d"))) ) )), - comments: vec![], }] }; @@ -569,7 +548,6 @@ mod tests { Seq(Ident(String::from("a")), Ident(String::from("b"))), Ident(String::from("a")) )), - comments: vec![], }] }; let optimized = { @@ -578,7 +556,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Silent, expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))), - comments: vec![], }] }; @@ -596,7 +573,6 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("b"))) )), - comments: vec![], }] }; let optimized = { @@ -605,7 +581,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Silent, expr: box_tree!(Ident(String::from("a"))), - comments: vec![], }] }; @@ -623,7 +598,6 @@ mod tests { Rep(Seq(Ident(String::from("a")), Ident(String::from("b")))), Ident(String::from("a")) )), - comments: vec![], }] }; let optimized = { @@ -635,7 +609,6 @@ mod tests { Ident(String::from("a")), Rep(Seq(Ident(String::from("b")), Ident(String::from("a")))) )), - comments: vec![], }] }; diff --git a/meta/src/optimizer/restorer.rs b/meta/src/optimizer/restorer.rs index cccf9ebb..e128e03f 100644 --- a/meta/src/optimizer/restorer.rs +++ b/meta/src/optimizer/restorer.rs @@ -14,19 +14,9 @@ pub fn restore_on_err( rule: OptimizedRule, rules: &HashMap, ) -> OptimizedRule { - let OptimizedRule { - name, - ty, - expr, - comments, - } = rule; + let OptimizedRule { name, ty, expr } = rule; let expr = expr.map_bottom_up(|expr| wrap_branching_exprs(expr, rules)); - OptimizedRule { - name, - ty, - expr, - comments, - } + OptimizedRule { name, ty, expr } } fn wrap_branching_exprs( @@ -110,7 +100,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Opt(Str("a".to_string()))), - comments: vec![], }]; assert_eq!( @@ -125,14 +114,12 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(Push(Str("a".to_string())))), - comments: vec![], }]; let restored = OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(RestoreOnErr(Push(Str("a".to_string()))))), - comments: vec![], }; assert_eq!( @@ -147,7 +134,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Choice(Push(Str("a".to_string())), Str("a".to_string()))), - comments: vec![], }]; let restored = OptimizedRule { @@ -157,7 +143,6 @@ mod tests { RestoreOnErr(Push(Str("a".to_string()))), Str("a".to_string()) )), - comments: vec![], }; assert_eq!( diff --git a/meta/src/optimizer/rotater.rs b/meta/src/optimizer/rotater.rs index 23176d4e..7a7d8fba 100644 --- a/meta/src/optimizer/rotater.rs +++ b/meta/src/optimizer/rotater.rs @@ -35,16 +35,10 @@ pub fn rotate(rule: Rule) -> Rule { } } - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, expr: expr.map_top_down(rotate_internal), - comments, } } diff --git a/meta/src/optimizer/skipper.rs b/meta/src/optimizer/skipper.rs index 4458af04..40bc5a16 100644 --- a/meta/src/optimizer/skipper.rs +++ b/meta/src/optimizer/skipper.rs @@ -28,12 +28,7 @@ pub fn skip(rule: Rule) -> Rule { } } - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -57,6 +52,5 @@ pub fn skip(rule: Rule) -> Rule { } else { expr }, - comments, } } diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs index 419a4469..e3c360d9 100644 --- a/meta/src/optimizer/unroller.rs +++ b/meta/src/optimizer/unroller.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn unroll(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -67,6 +62,5 @@ pub fn unroll(rule: Rule) -> Rule { .unwrap(), expr => expr, }), - comments, } } diff --git a/meta/src/parser.rs b/meta/src/parser.rs index b71e171c..fabc5bdd 100644 --- a/meta/src/parser.rs +++ b/meta/src/parser.rs @@ -11,7 +11,6 @@ use std::char; use std::iter::Peekable; -use std::sync::Mutex; use pest::error::{Error, ErrorVariant}; use pest::iterators::{Pair, Pairs}; @@ -51,8 +50,6 @@ pub struct ParserRule<'i> { pub ty: RuleType, /// The rule's parser node pub node: ParserNode<'i>, - /// Doc comments of the rule - pub(crate) comments: Vec, } /// The pest grammar node @@ -170,20 +167,9 @@ pub enum ParserExpr<'i> { } fn convert_rule(rule: ParserRule<'_>) -> AstRule { - let ParserRule { - name, - ty, - node, - comments, - .. - } = rule; + let ParserRule { name, ty, node, .. } = rule; let expr = convert_node(node); - AstRule { - name, - ty, - expr, - comments, - } + AstRule { name, ty, expr } } fn convert_node(node: ParserNode<'_>) -> Expr { @@ -263,17 +249,6 @@ pub fn rename_meta_rule(rule: &Rule) -> String { } } -fn filter_line_docs(pair: &Pair<'_, Rule>, line_docs: &mut Vec) -> bool { - let mut pairs = pair.clone().into_inner(); - let pair = pairs.next().unwrap(); - if pair.as_rule() == Rule::line_doc { - line_docs.push(pair.as_str()[3..pair.as_str().len()].trim().to_string()); - false - } else { - true - } -} - fn consume_rules_with_spans( pairs: Pairs<'_, Rule>, ) -> Result>, Vec>> { @@ -281,11 +256,15 @@ fn consume_rules_with_spans( .op(Op::infix(Rule::choice_operator, Assoc::Left)) .op(Op::infix(Rule::sequence_operator, Assoc::Left)); - let line_docs: Mutex> = Mutex::new(vec![]); - pairs .filter(|pair| pair.as_rule() == Rule::grammar_rule) - .filter(|pair| filter_line_docs(pair, &mut line_docs.lock().unwrap())) + .filter(|pair| { + // To ignore `grammar_rule > line_doc` pairs + let mut pairs = pair.clone().into_inner(); + let pair = pairs.next().unwrap(); + + pair.as_rule() != Rule::line_doc + }) .map(|pair| { let mut pairs = pair.into_inner().peekable(); @@ -316,17 +295,11 @@ fn consume_rules_with_spans( let node = consume_expr(inner_nodes, &pratt)?; - // consume doc comments - let mut line_docs = line_docs.lock().unwrap(); - let comments = line_docs.clone(); - line_docs.clear(); - Ok(ParserRule { name, span, ty, node, - comments, }) }) .collect() @@ -1417,11 +1390,7 @@ mod tests { )))) )) )))))) - ), - comments: vec![ - "This is line comment".to_string(), - "This is rule".to_string(), - ], + ) },] ); } @@ -1443,7 +1412,6 @@ mod tests { Box::new(Expr::PeekSlice(-4, None)), Box::new(Expr::PeekSlice(0, Some(3))), ), - comments: vec![], }], ); }