From c7212ca86f9ec1ceb0f444e499786155e7e760c3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 29 Oct 2024 23:40:12 +0000 Subject: [PATCH] gh-125588: Teach the python PEG generator the new f-string tokens (#125589) Signed-off-by: Pablo Galindo --- Lib/test/test_peg_generator/test_pegen.py | 8 +++++ ...-10-16-13-52-48.gh-issue-125588.kCahyO.rst | 2 ++ Tools/peg_generator/pegen/grammar_parser.py | 17 ++++++++++- Tools/peg_generator/pegen/metagrammar.gram | 3 ++ Tools/peg_generator/pegen/parser.py | 30 +++++++++++++++++++ Tools/peg_generator/pegen/python_generator.py | 3 +- 6 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 54c9dce2d0c90dc..d86065213453904 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -506,6 +506,14 @@ def test_python_expr(self) -> None: val = eval(code) self.assertEqual(val, 3.0) + def test_f_string_in_action(self) -> None: + grammar = """ + start: n=NAME NEWLINE? $ { f"name -> {n.string}" } + """ + parser_class = make_parser(grammar) + node = parse_string("a", parser_class) + self.assertEqual(node.strip(), "name -> a") + def test_nullable(self) -> None: grammar_source = """ start: sign NUMBER diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst new file mode 100644 index 000000000000000..1d59a9c3c205b88 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst @@ -0,0 +1,2 @@ +The Python PEG generator can now use f-strings in the grammar actions. Patch +by Pablo Galindo diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py index bf31fe532636455..2e3a607f7209b04 100644 --- a/Tools/peg_generator/pegen/grammar_parser.py +++ b/Tools/peg_generator/pegen/grammar_parser.py @@ -575,7 +575,7 @@ def target_atoms(self) -> Optional[str]: @memoize def target_atom(self) -> Optional[str]: - # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP + # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP mark = self._mark() cut = False if ( @@ -625,6 +625,21 @@ def target_atom(self) -> Optional[str]: ): return string . string self._reset(mark) + if ( + (fstring_start := self.fstring_start()) + ): + return fstring_start . string + self._reset(mark) + if ( + (fstring_middle := self.fstring_middle()) + ): + return fstring_middle . string + self._reset(mark) + if ( + (fstring_end := self.fstring_end()) + ): + return fstring_end . string + self._reset(mark) if ( (literal := self.expect("?")) ): diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram index f22c334ca371b27..f484c4781823bc4 100644 --- a/Tools/peg_generator/pegen/metagrammar.gram +++ b/Tools/peg_generator/pegen/metagrammar.gram @@ -126,6 +126,9 @@ target_atom[str]: | NAME { name.string } | NUMBER { number.string } | STRING { string.string } + | FSTRING_START { fstring_start.string } + | FSTRING_MIDDLE { fstring_middle.string } + | FSTRING_END { fstring_end.string } | "?" { "?" } | ":" { ":" } | !"}" !"]" OP { op.string } diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index ed0aec9db2443fa..692eb9ed2417d74 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -205,6 +205,36 @@ def string(self) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None + @memoize + def fstring_start(self) -> Optional[tokenize.TokenInfo]: + FSTRING_START = getattr(token, "FSTRING_START") + if not FSTRING_START: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_START: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + if not FSTRING_MIDDLE: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_MIDDLE: + return self._tokenizer.getnext() + return None + + @memoize + def fstring_end(self) -> Optional[tokenize.TokenInfo]: + FSTRING_END = getattr(token, "FSTRING_END") + if not FSTRING_END: + return None + tok = self._tokenizer.peek() + if tok.type == FSTRING_END: + return self._tokenizer.getnext() + return None + @memoize def op(self) -> Optional[tokenize.TokenInfo]: tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 7057135a9061f63..4bb26480ebc0af8 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -99,7 +99,8 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" - if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"): + if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT", + "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"): name = name.lower() return name, f"self.{name}()" if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):