From 6a9910725106f7f392f03969cd929d309d0658c7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 26 May 2020 10:10:41 -0700 Subject: [PATCH 01/67] Parsing VM initial structure --- Makefile.pre.in | 2 ++ Parser/pegen/vm.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++ Parser/pegen/vm.h | 68 +++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 Parser/pegen/vm.c create mode 100644 Parser/pegen/vm.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 0d616d304484ce..c3aeb87f8e466f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -298,6 +298,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ PEGEN_OBJS= \ Parser/pegen/pegen.o \ + Parser/pegen/vm.o \ Parser/pegen/parse.o \ Parser/pegen/parse_string.o \ Parser/pegen/peg_api.o @@ -306,6 +307,7 @@ PEGEN_OBJS= \ PEGEN_HEADERS= \ $(srcdir)/Include/internal/pegen_interface.h \ $(srcdir)/Parser/pegen/pegen.h \ + $(srcdir)/Parser/pegen/vm.h \ $(srcdir)/Parser/pegen/parse_string.h POBJS= \ diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c new file mode 100644 index 00000000000000..5227da36238e51 --- /dev/null +++ b/Parser/pegen/vm.c @@ -0,0 +1,92 @@ +#include +#include +#include "../tokenizer.h" + +#include "pegen.h" +#include "parse_string.h" + +#include "vm.h" + +static Frame * +push_frame(Stack *stack, Rule *rule) +{ + assert(stack->top < 100); + Frame *f = &stack->frames[stack->top++]; + f->rule = rule; + return f; +} + +static Frame * +pop_frame(Stack *stack) +{ + assert(stack->top > 0); + Frame *f = &stack->frames[--stack->top]; + return f; +} + +static void * +run_vm(Parser *p, Rule rules[], int start) +{ + Stack stack = {p, 0, {{0}}}; + Frame *f = push_frame(&stack, &rules[start]); + void *v; + int oparg; + + top: + switch (f->rule->opcodes[f->iop++]) { + case OP_NAME: + v = _PyPegen_name_token(p); + break; + case OP_NUMBER: + v = _PyPegen_number_token(p); + break; + case OP_CUT: + f->cut = 1; + goto top; + case OP_TOKEN: + oparg = f->rule->opcodes[f->iop++]; + v = _PyPegen_expect_token(p, oparg); + break; + case OP_RULE: + oparg = f->rule->opcodes[f->iop++]; + f = push_frame(&stack, &rules[oparg]); + goto top; + case OP_RETURN: + oparg = f->rule->opcodes[f->iop++]; + v = (void*) 1; // dummy action result + f = pop_frame(&stack); + break; + default: + abort(); + } + + if (v) { + // f->values.append(v) + goto top; + } + if (PyErr_Occurred()) { + p->error_indicator = 1; + return NULL; + } + + fail: + if (f->cut) + goto pop; + f->iop = f->rule->alts[++f->arg]; + if (f->iop == -1) + goto pop; + goto top; + + pop: + f = pop_frame(&stack); + goto fail; +} + +void * +_PyPegen_vmparse(Parser *p) +{ + p->keywords = reserved_keywords; + p->n_keyword_lists = n_keyword_lists; + + return run_vm(p, all_rules, 0); +} diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h new file mode 100644 index 00000000000000..08791a140f0a53 --- /dev/null +++ b/Parser/pegen/vm.h @@ -0,0 +1,68 @@ +typedef enum _opcodes { + OP_NAME, + OP_NUMBER, + OP_STRING, + OP_CUT, + // The rest have an argument + OP_TOKEN, + OP_RULE, + OP_RETURN, +} Opcode; + +typedef struct _rule { + char *name; + int alts[10]; + int opcodes[100]; +} Rule; + +typedef struct _frame { + Rule *rule; + int ialt; + int iop; + int cut; +} Frame; + +typedef struct _stack { + Parser *p; + int top; + Frame frames[100]; +} Stack; + +/* Toy grammar: + +start: expr ENDMARKER +expr: term + expr | term +term: NAME | NUMBER + +*/ + +static const int n_keyword_lists = 0; +static KeywordToken *reserved_keywords[] = { +}; + +static Rule all_rules[] = { + + {"start", + {0, -1}, + { + OP_RULE, 1, OP_TOKEN, ENDMARKER, OP_RETURN, 0, + }, + }, + + {"expr", + {0, 8, -1}, + { + OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE, 1, OP_RETURN, 0, + OP_RULE, 2, OP_RETURN, 0, + }, + }, + + {"term", + {0, 3, -1}, + { + OP_NAME, OP_RETURN, 0, + OP_NUMBER, OP_RETURN, 0, + }, + }, + +}; From 8ffc315ce2b8e3240bae22718bcdabb2eb40ece3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 26 May 2020 17:01:11 -0700 Subject: [PATCH 02/67] Hook things up so we can test it --- Include/compile.h | 4 +++- Makefile.pre.in | 3 ++- Modules/_peg_parser.c | 11 ++++++++--- Parser/pegen/pegen.c | 11 ++++++++++- Parser/pegen/pegen.h | 2 ++ Parser/pegen/vm.c | 21 +++++++++++++++++++-- Parser/pegen/vm.h | 22 +++++++++++++++++++--- 7 files changed, 63 insertions(+), 11 deletions(-) diff --git a/Include/compile.h b/Include/compile.h index 12417ce805464b..c7906250406675 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -27,8 +27,10 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_IGNORE_COOKIE 0x0800 #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 +#define PyCF_VMPARSER 0x4000 #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ - PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) + PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ + PyCF_VMPARSER) #ifndef Py_LIMITED_API typedef struct { diff --git a/Makefile.pre.in b/Makefile.pre.in index c3aeb87f8e466f..e9a94f8b2b46ae 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -307,7 +307,6 @@ PEGEN_OBJS= \ PEGEN_HEADERS= \ $(srcdir)/Include/internal/pegen_interface.h \ $(srcdir)/Parser/pegen/pegen.h \ - $(srcdir)/Parser/pegen/vm.h \ $(srcdir)/Parser/pegen/parse_string.h POBJS= \ @@ -1152,6 +1151,8 @@ PYTHON_HEADERS= \ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) +$(srcdir)/Parser/pegen/vm.o: $(srcdir)/Parser/pegen/vm.h + ###################################################################### diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c index 3b27b2c9cbaa24..fedf24fea4d996 100644 --- a/Modules/_peg_parser.c +++ b/Modules/_peg_parser.c @@ -45,13 +45,14 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *keywords[] = {"string", "mode", "oldparser", NULL}; + static char *keywords[] = {"string", "mode", "oldparser", "vmparser", NULL}; char *the_string; char *mode_str = "exec"; int oldparser = 0; + int vmparser = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sp", keywords, - &the_string, &mode_str, &oldparser)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|spp", keywords, + &the_string, &mode_str, &oldparser, &vmparser)) { return NULL; } @@ -78,6 +79,10 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; flags.cf_flags = PyCF_IGNORE_COOKIE; + if (vmparser) { + printf("VMPARSER\n"); + flags.cf_flags |= 0x4000; + } mod_ty res; if (oldparser) { diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index c80f08668b07d6..8b11848adc88c9 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -1006,6 +1006,9 @@ compute_parser_flags(PyCompilerFlags *flags) if (flags->cf_flags & PyCF_TYPE_COMMENTS) { parser_flags |= PyPARSE_TYPE_COMMENTS; } + if (flags->cf_flags &PyCF_VMPARSER) { + parser_flags |= PyPARSE_VMPARSER; + } if (flags->cf_feature_version < 7) { parser_flags |= PyPARSE_ASYNC_HACKS; } @@ -1185,7 +1188,13 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen goto error; } - result = _PyPegen_run_parser(p); + if (parser_flags & PyPARSE_VMPARSER) { + printf("vmparse\n"); + result = _PyPegen_vmparser(p); + } + else { + result = _PyPegen_run_parser(p); + } _PyPegen_Parser_Free(p); error: diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h index b55a652ac8060d..d2d7649e4a6d3d 100644 --- a/Parser/pegen/pegen.h +++ b/Parser/pegen/pegen.h @@ -23,6 +23,7 @@ #define PyPARSE_BARRY_AS_BDFL 0x0020 #define PyPARSE_TYPE_COMMENTS 0x0040 #define PyPARSE_ASYNC_HACKS 0x0080 +#define PyPARSE_VMPARSER 0x0100 typedef struct _memo { int type; @@ -207,6 +208,7 @@ void _PyPegen_Parser_Free(Parser *); mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, const char *, const char *, PyCompilerFlags *, int *, PyArena *); void *_PyPegen_run_parser(Parser *); +void *_PyPegen_vmparser(Parser *); mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); void *_PyPegen_interactive_exit(Parser *); diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 5227da36238e51..49530f85e8214c 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -13,6 +13,9 @@ push_frame(Stack *stack, Rule *rule) assert(stack->top < 100); Frame *f = &stack->frames[stack->top++]; f->rule = rule; + f->ialt = 0; + f->iop = 0; + f->cut = 0; return f; } @@ -33,6 +36,8 @@ run_vm(Parser *p, Rule rules[], int start) int oparg; top: + for (int i = 0; i < stack.top; i++) printf(" "); + printf("Rule: %s; ialt=%d; iop=%d; op=%s\n", f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]]); switch (f->rule->opcodes[f->iop++]) { case OP_NAME: v = _PyPegen_name_token(p); @@ -56,23 +61,35 @@ run_vm(Parser *p, Rule rules[], int start) v = (void*) 1; // dummy action result f = pop_frame(&stack); break; + case OP_SUCCESS: + oparg = f->rule->opcodes[f->iop++]; + v = (void*) 1; // dummy action result + if (v) { + return v; + } + // fallthrough + case OP_FAILURE: + return RAISE_SYNTAX_ERROR("A syntax error"); default: abort(); } if (v) { + printf(" OK\n"); // f->values.append(v) goto top; } if (PyErr_Occurred()) { + printf(" PyErr\n"); p->error_indicator = 1; return NULL; } fail: + printf(" fail\n"); if (f->cut) goto pop; - f->iop = f->rule->alts[++f->arg]; + f->iop = f->rule->alts[++f->ialt]; if (f->iop == -1) goto pop; goto top; @@ -83,7 +100,7 @@ run_vm(Parser *p, Rule rules[], int start) } void * -_PyPegen_vmparse(Parser *p) +_PyPegen_vmparser(Parser *p) { p->keywords = reserved_keywords; p->n_keyword_lists = n_keyword_lists; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 08791a140f0a53..859322842d6114 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -3,12 +3,27 @@ typedef enum _opcodes { OP_NUMBER, OP_STRING, OP_CUT, + OP_SUCCESS, + OP_FAILURE, // The rest have an argument OP_TOKEN, OP_RULE, OP_RETURN, } Opcode; +char *opcode_names[] = { + "OP_NAME", + "OP_NUMBER", + "OP_STRING", + "OP_CUT", + "OP_SUCCESS", + "OP_FAILURE", + // The rest have an argument + "OP_TOKEN", + "OP_RULE", + "OP_RETURN", +}; + typedef struct _rule { char *name; int alts[10]; @@ -31,7 +46,7 @@ typedef struct _stack { /* Toy grammar: start: expr ENDMARKER -expr: term + expr | term +expr: term '+' expr | term term: NAME | NUMBER */ @@ -43,9 +58,10 @@ static KeywordToken *reserved_keywords[] = { static Rule all_rules[] = { {"start", - {0, -1}, + {0, 6, -1}, { - OP_RULE, 1, OP_TOKEN, ENDMARKER, OP_RETURN, 0, + OP_RULE, 1, OP_TOKEN, ENDMARKER, OP_SUCCESS, 0, + OP_FAILURE, }, }, From a9ba9466c911cb8329aed9f42baa3b69748a2c44 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 26 May 2020 20:32:49 -0700 Subject: [PATCH 03/67] Add debugging printf()s in anger; fix bugs --- Parser/pegen/vm.c | 18 +++++++++++++++--- Parser/pegen/vm.h | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 49530f85e8214c..20cf24aa210514 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -10,20 +10,23 @@ static Frame * push_frame(Stack *stack, Rule *rule) { + printf("push %s\n", rule->name); assert(stack->top < 100); Frame *f = &stack->frames[stack->top++]; f->rule = rule; f->ialt = 0; f->iop = 0; f->cut = 0; + f->mark = stack->p->mark; return f; } static Frame * pop_frame(Stack *stack) { - assert(stack->top > 0); - Frame *f = &stack->frames[--stack->top]; + assert(stack->top > 1); + Frame *f = &stack->frames[--stack->top - 1]; + printf("pop %s\n", f->rule->name); return f; } @@ -36,8 +39,13 @@ run_vm(Parser *p, Rule rules[], int start) int oparg; top: + if (p->mark == p->fill) + _PyPegen_fill_token(p); + Token *t = p->tokens[p->mark]; for (int i = 0; i < stack.top; i++) printf(" "); - printf("Rule: %s; ialt=%d; iop=%d; op=%s\n", f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]]); + printf("Rule: %s; ialt=%d; iop=%d; op=%s; p^=%s\n", + f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], + p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { case OP_NAME: v = _PyPegen_name_token(p); @@ -59,6 +67,9 @@ run_vm(Parser *p, Rule rules[], int start) case OP_RETURN: oparg = f->rule->opcodes[f->iop++]; v = (void*) 1; // dummy action result + if (!v) { + p->mark = f->mark; + } f = pop_frame(&stack); break; case OP_SUCCESS: @@ -87,6 +98,7 @@ run_vm(Parser *p, Rule rules[], int start) fail: printf(" fail\n"); + p->mark = f->mark; if (f->cut) goto pop; f->iop = f->rule->alts[++f->ialt]; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 859322842d6114..f0c10292a3534e 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -35,6 +35,7 @@ typedef struct _frame { int ialt; int iop; int cut; + int mark; } Frame; typedef struct _stack { @@ -60,7 +61,7 @@ static Rule all_rules[] = { {"start", {0, 6, -1}, { - OP_RULE, 1, OP_TOKEN, ENDMARKER, OP_SUCCESS, 0, + OP_RULE, 1, OP_TOKEN, NEWLINE, OP_SUCCESS, 0, OP_FAILURE, }, }, From 8635a4c2372e9e6e496c5e1380c9c8dc4eb6b3ca Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 09:04:46 -0700 Subject: [PATCH 04/67] Implement actions --- Parser/pegen/vm.c | 14 +++++++------- Parser/pegen/vm.h | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 20cf24aa210514..a5b976d9eb5f97 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -18,6 +18,7 @@ push_frame(Stack *stack, Rule *rule) f->iop = 0; f->cut = 0; f->mark = stack->p->mark; + f->ival = 0; return f; } @@ -39,13 +40,12 @@ run_vm(Parser *p, Rule rules[], int start) int oparg; top: - if (p->mark == p->fill) - _PyPegen_fill_token(p); - Token *t = p->tokens[p->mark]; + // if (p->mark == p->fill) + // _PyPegen_fill_token(p); for (int i = 0; i < stack.top; i++) printf(" "); printf("Rule: %s; ialt=%d; iop=%d; op=%s; p^=%s\n", f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], - p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); + p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { case OP_NAME: v = _PyPegen_name_token(p); @@ -66,7 +66,7 @@ run_vm(Parser *p, Rule rules[], int start) goto top; case OP_RETURN: oparg = f->rule->opcodes[f->iop++]; - v = (void*) 1; // dummy action result + v = call_action(p, f, oparg); if (!v) { p->mark = f->mark; } @@ -74,7 +74,7 @@ run_vm(Parser *p, Rule rules[], int start) break; case OP_SUCCESS: oparg = f->rule->opcodes[f->iop++]; - v = (void*) 1; // dummy action result + v = call_action(p, f, oparg); if (v) { return v; } @@ -87,7 +87,7 @@ run_vm(Parser *p, Rule rules[], int start) if (v) { printf(" OK\n"); - // f->values.append(v) + f->vals[f->ival++] = v; goto top; } if (PyErr_Occurred()) { diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index f0c10292a3534e..3ef01d9b503c46 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -36,6 +36,8 @@ typedef struct _frame { int iop; int cut; int mark; + int ival; + void *vals[10]; } Frame; typedef struct _stack { @@ -69,17 +71,42 @@ static Rule all_rules[] = { {"expr", {0, 8, -1}, { - OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE, 1, OP_RETURN, 0, - OP_RULE, 2, OP_RETURN, 0, + OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE, 1, OP_RETURN, 1, + OP_RULE, 2, OP_RETURN, 2, }, }, {"term", {0, 3, -1}, { - OP_NAME, OP_RETURN, 0, - OP_NUMBER, OP_RETURN, 0, + OP_NAME, OP_RETURN, 3, + OP_NUMBER, OP_RETURN, 4, }, }, }; + +static void * +call_action(Parser *p, Frame *f, int iaction) +{ + // TODO: compute lineno and col offset (put in frame?) + int _start_lineno = 1; + int _start_col_offset = 1; + int _end_lineno = 99; + int _end_col_offset = 99; + + switch (iaction) { + case 0: + return _PyPegen_make_module(p, _PyPegen_singleton_seq(p, _Py_Expr(f->vals[0], EXTRA))); + case 1: + return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); + case 2: + return f->vals[0]; + case 3: + return f->vals[0]; + case 4: + return f->vals[0]; + default: + assert(0); + } +} From 1a34aebdb5e75f567a25eeec6d71659ab8d09572 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 09:18:58 -0700 Subject: [PATCH 05/67] Support optional tokens --- Parser/pegen/vm.c | 16 ++++++++++++++++ Parser/pegen/vm.h | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index a5b976d9eb5f97..5e67e1ec05b777 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -60,10 +60,26 @@ run_vm(Parser *p, Rule rules[], int start) oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); break; + case OP_TOKEN_OPTIONAL: + oparg = f->rule->opcodes[f->iop++]; + v = _PyPegen_expect_token(p, oparg); + if (!v) { + if (PyErr_Occurred()) { + printf(" PyErr\n"); + p->error_indicator = 1; + return NULL; + } + f->vals[f->ival++] = NULL; + goto top; + } + break; case OP_RULE: oparg = f->rule->opcodes[f->iop++]; f = push_frame(&stack, &rules[oparg]); goto top; + case OP_RULE_OPTIONAL: + printf("OP_RULE_OPTIONAL not yet supported\n"); + abort(); case OP_RETURN: oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 3ef01d9b503c46..d0758714ebc027 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -7,7 +7,9 @@ typedef enum _opcodes { OP_FAILURE, // The rest have an argument OP_TOKEN, + OP_TOKEN_OPTIONAL, OP_RULE, + OP_RULE_OPTIONAL, OP_RETURN, } Opcode; @@ -20,7 +22,9 @@ char *opcode_names[] = { "OP_FAILURE", // The rest have an argument "OP_TOKEN", + "OP_TOKEN_OPTIONAL", "OP_RULE", + "OP_RULE_OPTIONAL", "OP_RETURN", }; @@ -71,7 +75,7 @@ static Rule all_rules[] = { {"expr", {0, 8, -1}, { - OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE, 1, OP_RETURN, 1, + OP_RULE, 2, OP_TOKEN_OPTIONAL, PLUS, OP_RULE, 1, OP_RETURN, 1, OP_RULE, 2, OP_RETURN, 2, }, }, From b4a22928d9491a3e36e81b0ff4d4d2f484a892d5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 15:00:51 -0700 Subject: [PATCH 06/67] Support optional rules --- Parser/pegen/vm.c | 31 +++++++++++++------------------ Parser/pegen/vm.h | 13 +++++++++---- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 5e67e1ec05b777..ae10d0c6666f3f 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -10,7 +10,7 @@ static Frame * push_frame(Stack *stack, Rule *rule) { - printf("push %s\n", rule->name); + printf(" push %s\n", rule->name); assert(stack->top < 100); Frame *f = &stack->frames[stack->top++]; f->rule = rule; @@ -27,7 +27,7 @@ pop_frame(Stack *stack) { assert(stack->top > 1); Frame *f = &stack->frames[--stack->top - 1]; - printf("pop %s\n", f->rule->name); + printf(" pop %s\n", f->rule->name); return f; } @@ -40,12 +40,12 @@ run_vm(Parser *p, Rule rules[], int start) int oparg; top: - // if (p->mark == p->fill) - // _PyPegen_fill_token(p); + if (p->mark == p->fill) + _PyPegen_fill_token(p); for (int i = 0; i < stack.top; i++) printf(" "); - printf("Rule: %s; ialt=%d; iop=%d; op=%s; p^=%s\n", + printf("Rule: %s; ialt=%d; iop=%d; op=%s; p^='%s'\n", f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], - p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); + p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { case OP_NAME: v = _PyPegen_name_token(p); @@ -63,29 +63,19 @@ run_vm(Parser *p, Rule rules[], int start) case OP_TOKEN_OPTIONAL: oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); - if (!v) { - if (PyErr_Occurred()) { - printf(" PyErr\n"); - p->error_indicator = 1; - return NULL; - } + if (!v && !PyErr_Occurred()) { f->vals[f->ival++] = NULL; goto top; } break; case OP_RULE: + case OP_RULE_OPTIONAL: // The magic is at label 'pop' below oparg = f->rule->opcodes[f->iop++]; f = push_frame(&stack, &rules[oparg]); goto top; - case OP_RULE_OPTIONAL: - printf("OP_RULE_OPTIONAL not yet supported\n"); - abort(); case OP_RETURN: oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); - if (!v) { - p->mark = f->mark; - } f = pop_frame(&stack); break; case OP_SUCCESS: @@ -124,6 +114,11 @@ run_vm(Parser *p, Rule rules[], int start) pop: f = pop_frame(&stack); + assert(f->rule->opcodes[f->iop - 2] == OP_RULE_OPTIONAL || f->rule->opcodes[f->iop - 2] == OP_RULE); + if (f->rule->opcodes[f->iop - 2] == OP_RULE_OPTIONAL) { + f->vals[f->ival++] = NULL; + goto top; + } goto fail; } diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index d0758714ebc027..3393f1dfab3188 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -50,10 +50,12 @@ typedef struct _stack { Frame frames[100]; } Stack; -/* Toy grammar: +/* Toy grammar + +NOTE: [expr] is optional to test OP_RULE_OPTIONAL. start: expr ENDMARKER -expr: term '+' expr | term +expr: term '+' [expr] | term term: NAME | NUMBER */ @@ -75,7 +77,7 @@ static Rule all_rules[] = { {"expr", {0, 8, -1}, { - OP_RULE, 2, OP_TOKEN_OPTIONAL, PLUS, OP_RULE, 1, OP_RETURN, 1, + OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE_OPTIONAL, 1, OP_RETURN, 1, OP_RULE, 2, OP_RETURN, 2, }, }, @@ -103,7 +105,10 @@ call_action(Parser *p, Frame *f, int iaction) case 0: return _PyPegen_make_module(p, _PyPegen_singleton_seq(p, _Py_Expr(f->vals[0], EXTRA))); case 1: - return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); + if (f->vals[2]) + return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); + else + return f->vals[0]; case 2: return f->vals[0]; case 3: From b1f2a03d157418935ea174fc1eca942fa03f3706 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 17:18:05 -0700 Subject: [PATCH 07/67] Add vmreadme.md; OP_SUCCESS has an argument --- Parser/pegen/vm.h | 4 +- Parser/pegen/vmreadme.md | 116 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 Parser/pegen/vmreadme.md diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 3393f1dfab3188..9e4681011a7a90 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -3,7 +3,6 @@ typedef enum _opcodes { OP_NUMBER, OP_STRING, OP_CUT, - OP_SUCCESS, OP_FAILURE, // The rest have an argument OP_TOKEN, @@ -11,6 +10,7 @@ typedef enum _opcodes { OP_RULE, OP_RULE_OPTIONAL, OP_RETURN, + OP_SUCCESS, } Opcode; char *opcode_names[] = { @@ -18,7 +18,6 @@ char *opcode_names[] = { "OP_NUMBER", "OP_STRING", "OP_CUT", - "OP_SUCCESS", "OP_FAILURE", // The rest have an argument "OP_TOKEN", @@ -26,6 +25,7 @@ char *opcode_names[] = { "OP_RULE", "OP_RULE_OPTIONAL", "OP_RETURN", + "OP_SUCCESS", }; typedef struct _rule { diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md new file mode 100644 index 00000000000000..b2a55ba4458890 --- /dev/null +++ b/Parser/pegen/vmreadme.md @@ -0,0 +1,116 @@ +Pegen Virtual Machine +===================== + +The Pegen VM is an alternative for the recursive-descent Pegen parser. +The idea is that the grammar (including actions) is identical, but +execution does not use the C stack frame. The hope is that this might +be faster, but this is far from given. If it is clear that it will +*not* be faster, we should stop working on this project. + +The runtime uses the same `Parser` structure as the recursive-descent +Pegen parser, and the same helper functions +(e.g., `_PyPegen_singleton_seq`). + +The VM uses a stack to hold state during parsing. The grammar is +represented by a few read-only tables. This design may be familiar +from that of CPython's "ceval" VM. The actions are represented by a +function containing a giant switch with one case for each action. + +The grammar tables and the action function are meant to be generated +by a parser generator similar to the current one. Because of the +actions, the generator needs to generate C code. + +The primary VM state is a stack of `Frame` structures. Each frame +represents a particular attempt to parse a rule at a given point in +the input. The only state separate from the stack is a pointer to the +`Parser` structure. + +The main state in a frame is as follows: + +- `Rule *rule` -- points to the rule being parsed +- `int mark` -- the input position at the start of the rule invocation +- `int ialt` -- indicates which alternative is currently being tried +- `int iop` -- indicates where we are in the current alternative +- `int cut` -- whether a "cut" was executed in the current alternative + +(We'll need state related to line numbers and column offsets as well.) + +Note that `rule` and `mark` don't change after the frame is initialized. + +In addition each frame has a "value stack" where successfully +recognized tokens and rules are stored. This uses: + +- `int ival` -- number of values stored so far +- `void *vals[]` -- values stored (the type is `Token *` or an AST node type) + +A `Rule` structure has the following fields: + +- `char *name` -- rule name, for debugging (e.g., `"start"`) +- `int alts[]` -- index into `opcodes` array for each alternative, + terminated by `-1` +- `int opcodes[]` -- array of opcodes and their arguments + +All rules are collected in a single array; the index in this array +is used by operations that reference other rules. + +The `opcodes` array is a sequence of operation codes and arguments. +Some opcodes (e.g., `OP_TOKEN`) are followed by an argument; others +(e.g., `OP_NAME`) are not. Both are representable as integers. + +Opcodes +------- + +Most operations can succeed or fail, and produce a vaue if they +succeed. + +If an operation succeeds, the value is appended to the frame's values +stack, and the VM proceeds to the next opcode. + +If an operation fails, the VM resets the input to the frame's mark, +and then proceeds to the next alternative of the frame's rule, if +there is one, and the frame's `cut` flag is not set. If the frame's +`cut` flag is set, or if its rule has no more alternatives, the frame +is popped off the frame stack and the VM proceeds with failure there. + +Some operations manipulate other frame fields. + +Calls into the support runtime can also produce *errors* -- when an +error is detected, the VM exits, immediately returning `NULL`. + +The following opcodes take no argument. + +- `OP_NAME` -- call `_PyPegen_name_token()`; fail if it returns + `NULL`, otherwise succeeds with the return value. + +- `OP_NUMBER` -- call `_PyPegen_number_token()`; same as `OP_NAME`. + +- `OP_STRING` -- call `_PyPegen_string_token()`; same as `OP_NAME`. + +- `OP_CUT` -- set the frame's `cut` flag; always succeeds, without a + value. + +- `OP_FAILURE` -- report a syntax error and exit the VM. + +These opcodes are followed by a single integer argument. + +- `OP_TOKEN` -- call `_PyPegen_expect_token()` with the argument; + processing is the same as for `OP_NAME`. + +- `OP_TOKEN_OPTIONAL` -- like `OP_TOKEN` but instead of failing, + succeed with a `NULL` value. + +- `OP_RULE` -- push a new frame onto the stack, initializing it with + the given rule, the current input position (mark), at the first + alternative and opcode. Then proceed to the first operation of the + new frame. + +- `OP_RULE_OPTIONAL` -- this is to `OP_RULE` as `OP_TOKEN_OPTIONAL` is + to `OP_TOKEN`. + +- `OP_RETURN` -- call the action given by the argument, then pop the + frame off the stack. Execution then proceeds (in the frame newly + revealed by that pop operation) as if the previous operation + succeeded or failed with the return value of the action. + +- `OP_SUCCESS` -- call the action given by the argument, and exit the + VM with its return value as result. From 50aa86805bd228ec3329278c3fcfe1b59e31110a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 19:03:09 -0700 Subject: [PATCH 08/67] Do optional items differently (with a postfix op) --- Parser/pegen/vm.c | 20 +++++++++----------- Parser/pegen/vm.h | 10 ++++------ Parser/pegen/vmreadme.md | 9 +++------ 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index ae10d0c6666f3f..5ff82e6d8c9dee 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -56,20 +56,13 @@ run_vm(Parser *p, Rule rules[], int start) case OP_CUT: f->cut = 1; goto top; + case OP_OPTIONAL: + goto top; case OP_TOKEN: oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); break; - case OP_TOKEN_OPTIONAL: - oparg = f->rule->opcodes[f->iop++]; - v = _PyPegen_expect_token(p, oparg); - if (!v && !PyErr_Occurred()) { - f->vals[f->ival++] = NULL; - goto top; - } - break; case OP_RULE: - case OP_RULE_OPTIONAL: // The magic is at label 'pop' below oparg = f->rule->opcodes[f->iop++]; f = push_frame(&stack, &rules[oparg]); goto top; @@ -101,6 +94,11 @@ run_vm(Parser *p, Rule rules[], int start) p->error_indicator = 1; return NULL; } + if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { + printf(" GA!\n"); + f->vals[f->ival++] = NULL; + goto top; + } fail: printf(" fail\n"); @@ -114,8 +112,8 @@ run_vm(Parser *p, Rule rules[], int start) pop: f = pop_frame(&stack); - assert(f->rule->opcodes[f->iop - 2] == OP_RULE_OPTIONAL || f->rule->opcodes[f->iop - 2] == OP_RULE); - if (f->rule->opcodes[f->iop - 2] == OP_RULE_OPTIONAL) { + if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { + printf(" GAGA!\n"); f->vals[f->ival++] = NULL; goto top; } diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 9e4681011a7a90..53845db3221028 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -4,11 +4,10 @@ typedef enum _opcodes { OP_STRING, OP_CUT, OP_FAILURE, + OP_OPTIONAL, // The rest have an argument OP_TOKEN, - OP_TOKEN_OPTIONAL, OP_RULE, - OP_RULE_OPTIONAL, OP_RETURN, OP_SUCCESS, } Opcode; @@ -19,11 +18,10 @@ char *opcode_names[] = { "OP_STRING", "OP_CUT", "OP_FAILURE", + "OP_OPTIONAL", // The rest have an argument "OP_TOKEN", - "OP_TOKEN_OPTIONAL", "OP_RULE", - "OP_RULE_OPTIONAL", "OP_RETURN", "OP_SUCCESS", }; @@ -75,9 +73,9 @@ static Rule all_rules[] = { }, {"expr", - {0, 8, -1}, + {0, 9, -1}, { - OP_RULE, 2, OP_TOKEN, PLUS, OP_RULE_OPTIONAL, 1, OP_RETURN, 1, + OP_RULE, 2, OP_TOKEN, PLUS, OP_OPTIONAL, OP_RULE, 1, OP_RETURN, 1, OP_RULE, 2, OP_RETURN, 2, }, }, diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index b2a55ba4458890..5f3337104f0980 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -89,6 +89,9 @@ The following opcodes take no argument. - `OP_CUT` -- set the frame's `cut` flag; always succeeds, without a value. +- `OP_OPTIONAL` -- modifies the *previous* operation to treat a `NULL` + result as a success. + - `OP_FAILURE` -- report a syntax error and exit the VM. These opcodes are followed by a single integer argument. @@ -96,17 +99,11 @@ These opcodes are followed by a single integer argument. - `OP_TOKEN` -- call `_PyPegen_expect_token()` with the argument; processing is the same as for `OP_NAME`. -- `OP_TOKEN_OPTIONAL` -- like `OP_TOKEN` but instead of failing, - succeed with a `NULL` value. - - `OP_RULE` -- push a new frame onto the stack, initializing it with the given rule, the current input position (mark), at the first alternative and opcode. Then proceed to the first operation of the new frame. -- `OP_RULE_OPTIONAL` -- this is to `OP_RULE` as `OP_TOKEN_OPTIONAL` is - to `OP_TOKEN`. - - `OP_RETURN` -- call the action given by the argument, then pop the frame off the stack. Execution then proceeds (in the frame newly revealed by that pop operation) as if the previous operation From 320266b91cdb73a6ee376803456a917b7ffb3a65 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 27 May 2020 21:40:23 -0700 Subject: [PATCH 09/67] Compute start/end line/col numbers; add some ideas to vmreadme.md --- Parser/pegen/vm.h | 11 ++++++----- Parser/pegen/vmreadme.md | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 53845db3221028..653db4ede46507 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -93,11 +93,12 @@ static Rule all_rules[] = { static void * call_action(Parser *p, Frame *f, int iaction) { - // TODO: compute lineno and col offset (put in frame?) - int _start_lineno = 1; - int _start_col_offset = 1; - int _end_lineno = 99; - int _end_col_offset = 99; + Token *t = p->tokens[f->mark]; + int _start_lineno = t->lineno; + int _start_col_offset = t->col_offset; + t = p->tokens[p->mark - 1]; + int _end_lineno = t->end_lineno; + int _end_col_offset = t->end_col_offset; switch (iaction) { case 0: diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 5f3337104f0980..a70711346cb927 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -111,3 +111,22 @@ These opcodes are followed by a single integer argument. - `OP_SUCCESS` -- call the action given by the argument, and exit the VM with its return value as result. + +Ideas +----- + +We need to extend the VM to support loops and lookaheads. + +- Standard loops (`a*`, `a+`, `a.b+`) -- add a new opcode that resets + the instruction pointer without destroying the captured value; need + a new frame field to store the collected values. (Probably needs + several opcodes.) + +- Positive lookahead (`&a`) -- add a new opcode to save the mark in a + new frame field, and another opcode that restores the mark from + there. + +- Negative lookahead (`!a`) -- add another opcode that catches failure + and turns it into success, restoring the saved mark. + +- Left-recursion -- hmm, tricky... From 5bf3f9c59b1bb326463b8a8ad7257a3e00ab97ca Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 28 May 2020 09:09:54 -0700 Subject: [PATCH 10/67] Tighten the code; add some speculation to vmreadme --- Parser/pegen/vm.c | 9 +++---- Parser/pegen/vmreadme.md | 56 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 5ff82e6d8c9dee..d3d16741a771b4 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -94,13 +94,15 @@ run_vm(Parser *p, Rule rules[], int start) p->error_indicator = 1; return NULL; } + + fail: if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { printf(" GA!\n"); f->vals[f->ival++] = NULL; + f->iop++; // Skip over the OP_OPTIONAL opcode goto top; } - fail: printf(" fail\n"); p->mark = f->mark; if (f->cut) @@ -112,11 +114,6 @@ run_vm(Parser *p, Rule rules[], int start) pop: f = pop_frame(&stack); - if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { - printf(" GAGA!\n"); - f->vals[f->ival++] = NULL; - goto top; - } goto fail; } diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index a70711346cb927..a265c72b539352 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -130,3 +130,59 @@ We need to extend the VM to support loops and lookaheads. and turns it into success, restoring the saved mark. - Left-recursion -- hmm, tricky... + +Loop opcodes +------------ + +``` +case START_LOOP: // At start of first alternative + f->collection = malloc(0); + f->collected = 0; + goto top; + +case CONTINUE_LOOP: // At end of first alternative (replaces RETURN) + oparg = f->rule->opcodes[f->iop++]; + v = call_action(p, f, oparg); + if (v) { + f->collection = realloc(f->collection, (f->collected + 1)*sizeof(void*)); + if (!f->collection) return NULL; + f->collection[f->collected++] = v; + goto top; + } + break; + +case COLLECT_LOOP: // At start of second (and last) alternative + v = collection from malloc'ed array to asdl_seq>; + free(f->collection); + // Continue same as RETURN opcode +``` + +(Need some more stuff for `a+` and `a.b+`.) + + +Lookahead opcodes +----------------- + +``` +case SAVE_MARK: + f->save_mark = p->mark; + goto top; + +case POS_LOOKAHEAD: + p->mark - f->save_mark; + goto top; + +case NEG_LOOKAHEAD: + goto fail; + +/* Later, when v == NULL */ +if (f->rule->opcodes[f->iop] == NEG_LOOKAHEAD) { + p->mark - f->save_mark; + goto top; +} +/* Also at the other fail check, under pop */ +``` + +If we initialize save_mark to the same value as mark, we can avoid a +`SAVE_MARK` opcode at the start of alternatives -- this is a common +pattern. From 816db752e4ad4cecd9bd25eb7bcbc670b2625c8a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 28 May 2020 09:28:33 -0700 Subject: [PATCH 11/67] Add OP_NOOP; add enums for rules & actions --- Parser/pegen/vm.c | 2 + Parser/pegen/vm.h | 79 +++++++++++++++++++++++++++++++--------- Parser/pegen/vmreadme.md | 5 ++- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index d3d16741a771b4..e75c0bf5a120e5 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -47,6 +47,8 @@ run_vm(Parser *p, Rule rules[], int start) f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { + case OP_NOOP: + goto top; case OP_NAME: v = _PyPegen_name_token(p); break; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 653db4ede46507..6f4bad8c3ea535 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -1,28 +1,36 @@ typedef enum _opcodes { + OP_NOOP, OP_NAME, OP_NUMBER, OP_STRING, OP_CUT, - OP_FAILURE, OP_OPTIONAL, + OP_LOOP_START, + OP_LOOP_COLLECT, + OP_FAILURE, // The rest have an argument OP_TOKEN, OP_RULE, OP_RETURN, + OP_LOOP_CONTINUE, OP_SUCCESS, } Opcode; char *opcode_names[] = { + "OP_NOOP", "OP_NAME", "OP_NUMBER", "OP_STRING", "OP_CUT", - "OP_FAILURE", "OP_OPTIONAL", + "OP_LOOP_START", + "OP_LOOP_COLLECT", + "OP_FAILURE", // The rest have an argument "OP_TOKEN", "OP_RULE", "OP_RETURN", + "OP_LOOP_CONTINUE", "OP_SUCCESS", }; @@ -52,8 +60,9 @@ typedef struct _stack { NOTE: [expr] is optional to test OP_RULE_OPTIONAL. -start: expr ENDMARKER -expr: term '+' [expr] | term +start: stmt ENDMARKER +stmt: expr NEWLINE +expr: term '+' expr | term term: NAME | NUMBER */ @@ -62,29 +71,63 @@ static const int n_keyword_lists = 0; static KeywordToken *reserved_keywords[] = { }; +enum { + R_START, + R_STMT, + R_EXPR, + R_TERM, +}; + +enum { + A_START_0, + A_STMT_0, + A_EXPR_0, + A_EXPR_1, + A_TERM_0, + A_TERM_1, +}; + static Rule all_rules[] = { {"start", - {0, 6, -1}, + {0, 4, -1}, { - OP_RULE, 1, OP_TOKEN, NEWLINE, OP_SUCCESS, 0, + OP_RULE, R_STMT, + OP_SUCCESS, A_START_0, + OP_FAILURE, }, }, + {"stmt", + {0, -1}, + { + OP_RULE, R_EXPR, + OP_RETURN, A_STMT_0, + }, + }, + {"expr", - {0, 9, -1}, + {0, 8, -1}, { - OP_RULE, 2, OP_TOKEN, PLUS, OP_OPTIONAL, OP_RULE, 1, OP_RETURN, 1, - OP_RULE, 2, OP_RETURN, 2, + OP_RULE, R_TERM, + OP_TOKEN, PLUS, + OP_RULE, R_EXPR, + OP_RETURN, A_EXPR_0, + + OP_RULE, R_TERM, + OP_RETURN, A_EXPR_1, }, }, {"term", {0, 3, -1}, { - OP_NAME, OP_RETURN, 3, - OP_NUMBER, OP_RETURN, 4, + OP_NAME, + OP_RETURN, A_TERM_0, + + OP_NUMBER, + OP_RETURN, A_TERM_1, }, }, @@ -101,18 +144,20 @@ call_action(Parser *p, Frame *f, int iaction) int _end_col_offset = t->end_col_offset; switch (iaction) { - case 0: - return _PyPegen_make_module(p, _PyPegen_singleton_seq(p, _Py_Expr(f->vals[0], EXTRA))); - case 1: + case A_START_0: + return _PyPegen_make_module(p, f->vals[0]); + case A_STMT_0: + return _PyPegen_singleton_seq(p, _Py_Expr(f->vals[0], EXTRA)); + case A_EXPR_0: if (f->vals[2]) return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); else return f->vals[0]; - case 2: + case A_EXPR_1: return f->vals[0]; - case 3: + case A_TERM_0: return f->vals[0]; - case 4: + case A_TERM_1: return f->vals[0]; default: assert(0); diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index a265c72b539352..5d77fb81331054 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -79,6 +79,8 @@ error is detected, the VM exits, immediately returning `NULL`. The following opcodes take no argument. +- `OP_NOOP` -- succeed without a value. (Used for opcode padding.) + - `OP_NAME` -- call `_PyPegen_name_token()`; fail if it returns `NULL`, otherwise succeeds with the return value. @@ -86,8 +88,7 @@ The following opcodes take no argument. - `OP_STRING` -- call `_PyPegen_string_token()`; same as `OP_NAME`. -- `OP_CUT` -- set the frame's `cut` flag; always succeeds, without a - value. +- `OP_CUT` -- set the frame's `cut` flag; succeeds without a value. - `OP_OPTIONAL` -- modifies the *previous* operation to treat a `NULL` result as a success. From d2a980fa338a011d285a74698570f7f3b24ebc2a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 28 May 2020 17:28:38 -0700 Subject: [PATCH 12/67] Implement loops --- Parser/pegen/vm.c | 58 ++++++++++++++++-- Parser/pegen/vm.h | 44 ++++++++++---- Parser/pegen/vmreadme.md | 125 +++++++++++++++++++++------------------ 3 files changed, 151 insertions(+), 76 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index e75c0bf5a120e5..0b086983fb3407 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -14,10 +14,12 @@ push_frame(Stack *stack, Rule *rule) assert(stack->top < 100); Frame *f = &stack->frames[stack->top++]; f->rule = rule; + f->mark = stack->p->mark; f->ialt = 0; f->iop = 0; f->cut = 0; - f->mark = stack->p->mark; + f->ncollected = 0; + f->collection = NULL; f->ival = 0; return f; } @@ -26,11 +28,28 @@ static Frame * pop_frame(Stack *stack) { assert(stack->top > 1); - Frame *f = &stack->frames[--stack->top - 1]; + Frame *f = &stack->frames[--stack->top]; // Frame being popped + if (f->collection) { + PyMem_Free(f->collection); + } + f = &stack->frames[stack->top - 1]; // New top of stack printf(" pop %s\n", f->rule->name); return f; } +static asdl_seq * +make_asdl_seq(Parser *p, void *collection[], int ncollected) +{ + asdl_seq *seq = _Py_asdl_seq_new(ncollected, p->arena); + if (!seq) { + return NULL; + } + for (int i = 0; i < ncollected; i++) { + asdl_seq_SET(seq, i, collection[i]); + } + return seq; +} + static void * run_vm(Parser *p, Rule rules[], int start) { @@ -43,8 +62,8 @@ run_vm(Parser *p, Rule rules[], int start) if (p->mark == p->fill) _PyPegen_fill_token(p); for (int i = 0; i < stack.top; i++) printf(" "); - printf("Rule: %s; ialt=%d; iop=%d; op=%s; p^='%s'\n", - f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], + printf("Rule: %s; ialt=%d; iop=%d; op=%s; mark=%d; token=%d; p^='%s'\n", + f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], p->mark, p->tokens[p->mark]->type, p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { case OP_NOOP: @@ -60,6 +79,33 @@ run_vm(Parser *p, Rule rules[], int start) goto top; case OP_OPTIONAL: goto top; + case OP_LOOP_START: + f->ncollected = 0; + f->collection = PyMem_Malloc(0); + if (!f->collection) { + return PyErr_NoMemory(); + } + goto top; + case OP_LOOP_ITERATE: + f->mark = p->mark; + assert(f->ival == 1); + v = f->vals[0]; + assert(v); + f->collection = PyMem_Realloc(f->collection, (f->ncollected + 1) * sizeof(void *)); + if (!f->collection) { + return NULL; + } + f->collection[f->ncollected++] = v; + f->iop = f->rule->alts[f->ialt] + 1; // Skip OP_LOOP_START operation + f->ival = 0; + goto top; + case OP_LOOP_COLLECT: + v = make_asdl_seq(p, f->collection, f->ncollected); + f = pop_frame(&stack); + if (!v) { + return PyErr_NoMemory(); + } + break; case OP_TOKEN: oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); @@ -76,10 +122,10 @@ run_vm(Parser *p, Rule rules[], int start) case OP_SUCCESS: oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); - if (v) { + if (v || PyErr_Occurred()) { return v; } - // fallthrough + return RAISE_SYNTAX_ERROR("Final action failed"); case OP_FAILURE: return RAISE_SYNTAX_ERROR("A syntax error"); default: diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 6f4bad8c3ea535..de5438fac17c53 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -6,13 +6,13 @@ typedef enum _opcodes { OP_CUT, OP_OPTIONAL, OP_LOOP_START, - OP_LOOP_COLLECT, + OP_LOOP_ITERATE, OP_FAILURE, // The rest have an argument OP_TOKEN, OP_RULE, OP_RETURN, - OP_LOOP_CONTINUE, + OP_LOOP_COLLECT, OP_SUCCESS, } Opcode; @@ -24,13 +24,13 @@ char *opcode_names[] = { "OP_CUT", "OP_OPTIONAL", "OP_LOOP_START", - "OP_LOOP_COLLECT", + "OP_LOOP_ITERATE", "OP_FAILURE", // The rest have an argument "OP_TOKEN", "OP_RULE", "OP_RETURN", - "OP_LOOP_CONTINUE", + "OP_LOOP_COLLECT", "OP_SUCCESS", }; @@ -42,10 +42,12 @@ typedef struct _rule { typedef struct _frame { Rule *rule; + int mark; int ialt; int iop; int cut; - int mark; + int ncollected; + void **collection; int ival; void *vals[10]; } Frame; @@ -58,9 +60,9 @@ typedef struct _stack { /* Toy grammar -NOTE: [expr] is optional to test OP_RULE_OPTIONAL. +NOTE: [expr] is right-recursive. -start: stmt ENDMARKER +start: stmt* ENDMARKER stmt: expr NEWLINE expr: term '+' expr | term term: NAME | NUMBER @@ -73,6 +75,7 @@ static KeywordToken *reserved_keywords[] = { enum { R_START, + R_STMTS, R_STMT, R_EXPR, R_TERM, @@ -90,19 +93,32 @@ enum { static Rule all_rules[] = { {"start", - {0, 4, -1}, + {0, 6, -1}, { - OP_RULE, R_STMT, + OP_RULE, R_STMTS, + OP_TOKEN, ENDMARKER, OP_SUCCESS, A_START_0, OP_FAILURE, }, }, + {"stmt*", + {0, 4, -1}, + { + OP_LOOP_START, + OP_RULE, R_STMT, + OP_LOOP_ITERATE, + + OP_LOOP_COLLECT, + }, + }, + {"stmt", {0, -1}, { OP_RULE, R_EXPR, + OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, }, }, @@ -139,7 +155,13 @@ call_action(Parser *p, Frame *f, int iaction) Token *t = p->tokens[f->mark]; int _start_lineno = t->lineno; int _start_col_offset = t->col_offset; - t = p->tokens[p->mark - 1]; + if (p->mark > 0) { + t = p->tokens[p->mark - 1]; + } + else { + // Debug anomaly rather than crashing + printf("p->mark=%d, p->fill=%d, f->mark=%d\n", p->mark, p->fill, f->mark); + } int _end_lineno = t->end_lineno; int _end_col_offset = t->end_col_offset; @@ -147,7 +169,7 @@ call_action(Parser *p, Frame *f, int iaction) case A_START_0: return _PyPegen_make_module(p, f->vals[0]); case A_STMT_0: - return _PyPegen_singleton_seq(p, _Py_Expr(f->vals[0], EXTRA)); + return _Py_Expr(f->vals[0], EXTRA); case A_EXPR_0: if (f->vals[2]) return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 5d77fb81331054..a5095f8d4d8680 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -33,7 +33,7 @@ The main state in a frame is as follows: - `int iop` -- indicates where we are in the current alternative - `int cut` -- whether a "cut" was executed in the current alternative -(We'll need state related to line numbers and column offsets as well.) +State related to loops is described below. Note that `rule` and `mark` don't change after the frame is initialized. @@ -57,8 +57,8 @@ The `opcodes` array is a sequence of operation codes and arguments. Some opcodes (e.g., `OP_TOKEN`) are followed by an argument; others (e.g., `OP_NAME`) are not. Both are representable as integers. -Opcodes -------- +Operations +---------- Most operations can succeed or fail, and produce a vaue if they succeed. @@ -77,6 +77,8 @@ Some operations manipulate other frame fields. Calls into the support runtime can also produce *errors* -- when an error is detected, the VM exits, immediately returning `NULL`. +### General operations + The following opcodes take no argument. - `OP_NOOP` -- succeed without a value. (Used for opcode padding.) @@ -93,35 +95,69 @@ The following opcodes take no argument. - `OP_OPTIONAL` -- modifies the *previous* operation to treat a `NULL` result as a success. +These operations are followed by a single integer argument. + +- `OP_TOKEN(type)` -- call `_PyPegen_expect_token()` with the `type` + argument; processing is the same as for `OP_NAME`. + +- `OP_RULE(rule)` -- push a new frame onto the stack, initializing it + with the give rule (by index), the current input position (mark), + at the first alternative and opcode. Then proceed to the first + operation of the new frame. + +- `OP_RETURN(action)` -- call the action given by the argument, then + pop the frame off the stack. Execution then proceeds (in the frame + newly revealed by that pop operation) as if the previous operation + succeeded or failed with the return value of the action. + +### Operations for start rules only + +- `OP_SUCCESS(action)` -- call the action given by the argument, and + exit the VM with its return value as result. + - `OP_FAILURE` -- report a syntax error and exit the VM. -These opcodes are followed by a single integer argument. +### Looping operations -- `OP_TOKEN` -- call `_PyPegen_expect_token()` with the argument; - processing is the same as for `OP_NAME`. +For a loop such as `a*`, a synthetic rule must be created with the +following structure: -- `OP_RULE` -- push a new frame onto the stack, initializing it with - the given rule, the current input position (mark), at the first - alternative and opcode. Then proceed to the first operation of the - new frame. +``` +# First alternative: +OP_LOOP_START + +OP_LOOP_ITERATE -- `OP_RETURN` -- call the action given by the argument, then pop the - frame off the stack. Execution then proceeds (in the frame newly - revealed by that pop operation) as if the previous operation - succeeded or failed with the return value of the action. +# Second alternative: +OP_LOOP_COLLECT +``` + +The values being collected are stored in a `malloc`-ed array named +`collections` that is grown as needed. This uses the following +fields: + +- `ncollected` -- the number of collected values. +- `collection` -- `malloc`-ed array of `void *` values representing + the collected values. -- `OP_SUCCESS` -- call the action given by the argument, and exit the - VM with its return value as result. +The operations are defined as follows: + +- `OP_LOOP_START` -- initialize the `collections` array. + +- `OP_LOOP_ITERATE` -- append the current value to the `collections` + array, save the current input position, and start the next iteration + of the loop (resetting the instruction pointer). + +- `OP_LOOP_COLLECT` -- restore the input position from the last saved + position and pop the frame off the stack, producing a new value that + is an `asdl_seq *` containing the collected values. + +(TODO: additional operations to support `a+` and `b.a+`.) Ideas ----- -We need to extend the VM to support loops and lookaheads. - -- Standard loops (`a*`, `a+`, `a.b+`) -- add a new opcode that resets - the instruction pointer without destroying the captured value; need - a new frame field to store the collected values. (Probably needs - several opcodes.) +We also need to extend the VM to support lookaheads. - Positive lookahead (`&a`) -- add a new opcode to save the mark in a new frame field, and another opcode that restores the mark from @@ -132,58 +168,29 @@ We need to extend the VM to support loops and lookaheads. - Left-recursion -- hmm, tricky... -Loop opcodes ------------- - -``` -case START_LOOP: // At start of first alternative - f->collection = malloc(0); - f->collected = 0; - goto top; - -case CONTINUE_LOOP: // At end of first alternative (replaces RETURN) - oparg = f->rule->opcodes[f->iop++]; - v = call_action(p, f, oparg); - if (v) { - f->collection = realloc(f->collection, (f->collected + 1)*sizeof(void*)); - if (!f->collection) return NULL; - f->collection[f->collected++] = v; - goto top; - } - break; - -case COLLECT_LOOP: // At start of second (and last) alternative - v = collection from malloc'ed array to asdl_seq>; - free(f->collection); - // Continue same as RETURN opcode -``` - -(Need some more stuff for `a+` and `a.b+`.) - - Lookahead opcodes ----------------- ``` -case SAVE_MARK: - f->save_mark = p->mark; +case OP_SAVE_MARK: + f->savemark = p->mark; goto top; -case POS_LOOKAHEAD: - p->mark - f->save_mark; +case OP_POS_LOOKAHEAD: + p->mark - f->savemark; goto top; -case NEG_LOOKAHEAD: +case OP_NEG_LOOKAHEAD: goto fail; /* Later, when v == NULL */ -if (f->rule->opcodes[f->iop] == NEG_LOOKAHEAD) { - p->mark - f->save_mark; +if (f->rule->opcodes[f->iop] == OP_NEG_LOOKAHEAD) { + p->mark - f->savemark; goto top; } /* Also at the other fail check, under pop */ ``` -If we initialize save_mark to the same value as mark, we can avoid a +If we initialize savemark to the same value as mark, we can avoid a `SAVE_MARK` opcode at the start of alternatives -- this is a common pattern. From 2befb12d1d488dd9f79aca49403ecf0d849c2f9d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 28 May 2020 19:57:48 -0700 Subject: [PATCH 13/67] Add a few more rules to the grammar --- Parser/pegen/vm.c | 7 +++++-- Parser/pegen/vm.h | 42 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 0b086983fb3407..17719f25d5fd7b 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -62,8 +62,11 @@ run_vm(Parser *p, Rule rules[], int start) if (p->mark == p->fill) _PyPegen_fill_token(p); for (int i = 0; i < stack.top; i++) printf(" "); - printf("Rule: %s; ialt=%d; iop=%d; op=%s; mark=%d; token=%d; p^='%s'\n", - f->rule->name, f->ialt, f->iop, opcode_names[f->rule->opcodes[f->iop]], p->mark, p->tokens[p->mark]->type, + int opc = f->rule->opcodes[f->iop]; + printf("Rule: %s; ialt=%d; iop=%d; op=%s; arg=%d, mark=%d; token=%d; p^='%s'\n", + f->rule->name, f->ialt, f->iop, opcode_names[opc], + opc >= OP_TOKEN ? f->rule->opcodes[f->iop + 1] : -1, + p->mark, p->tokens[p->mark]->type, p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); switch (f->rule->opcodes[f->iop++]) { case OP_NOOP: diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index de5438fac17c53..8eee05d8649266 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -65,7 +65,8 @@ NOTE: [expr] is right-recursive. start: stmt* ENDMARKER stmt: expr NEWLINE expr: term '+' expr | term -term: NAME | NUMBER +term: factor '*' term | factor +factor: '(' expr ')' | NUMBER | NAME */ @@ -79,6 +80,7 @@ enum { R_STMT, R_EXPR, R_TERM, + R_FACTOR, }; enum { @@ -88,6 +90,9 @@ enum { A_EXPR_1, A_TERM_0, A_TERM_1, + A_FACTOR_0, + A_FACTOR_1, + A_FACTOR_2, }; static Rule all_rules[] = { @@ -137,16 +142,34 @@ static Rule all_rules[] = { }, {"term", - {0, 3, -1}, + {0, 8, -1}, { - OP_NAME, + OP_RULE, R_FACTOR, + OP_TOKEN, STAR, + OP_RULE, R_TERM, OP_RETURN, A_TERM_0, - OP_NUMBER, + OP_RULE, R_FACTOR, OP_RETURN, A_TERM_1, }, }, + {"factor", + {0, 8, -1}, + { + OP_TOKEN, LPAR, + OP_RULE, R_EXPR, + OP_TOKEN, RPAR, + OP_RETURN, A_FACTOR_0, + + OP_NUMBER, + OP_RETURN, A_FACTOR_1, + + OP_NAME, + OP_RETURN, A_FACTOR_2, + }, + }, + }; static void * @@ -171,16 +194,19 @@ call_action(Parser *p, Frame *f, int iaction) case A_STMT_0: return _Py_Expr(f->vals[0], EXTRA); case A_EXPR_0: - if (f->vals[2]) - return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); - else - return f->vals[0]; + return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); case A_EXPR_1: return f->vals[0]; case A_TERM_0: return f->vals[0]; case A_TERM_1: return f->vals[0]; + case A_FACTOR_0: + return f->vals[0]; + case A_FACTOR_1: + return f->vals[0]; + case A_FACTOR_2: + return f->vals[0]; default: assert(0); } From 175a127ba6f854bf882b0554931f032cf483358a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 13:07:30 -0700 Subject: [PATCH 14/67] Drop debug printf()s, more flexibility in parse_string() --- Modules/_peg_parser.c | 18 ++++++++++++------ Parser/pegen/pegen.c | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c index fedf24fea4d996..bb6caaf2fef878 100644 --- a/Modules/_peg_parser.c +++ b/Modules/_peg_parser.c @@ -45,14 +45,15 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *keywords[] = {"string", "mode", "oldparser", "vmparser", NULL}; + static char *keywords[] = {"string", "mode", "oldparser", "vmparser", "ast", NULL}; char *the_string; char *mode_str = "exec"; int oldparser = 0; int vmparser = 0; + int ast = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|spp", keywords, - &the_string, &mode_str, &oldparser, &vmparser)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sppp", keywords, + &the_string, &mode_str, &oldparser, &vmparser, &ast)) { return NULL; } @@ -80,8 +81,7 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; flags.cf_flags = PyCF_IGNORE_COOKIE; if (vmparser) { - printf("VMPARSER\n"); - flags.cf_flags |= 0x4000; + flags.cf_flags |= PyCF_VMPARSER; } mod_ty res; @@ -94,7 +94,13 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) if (res == NULL) { goto error; } - result = PyAST_mod2obj(res); + if (ast) { + result = PyAST_mod2obj(res); + } + else { + result = Py_None; + Py_INCREF(result); + } error: PyArena_Free(arena); diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 8b11848adc88c9..93b5ea15065798 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -1189,7 +1189,6 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } if (parser_flags & PyPARSE_VMPARSER) { - printf("vmparse\n"); result = _PyPegen_vmparser(p); } else { From f3f1665de9e2a7cc70d44c212c1541fea0cb71b1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 13:08:27 -0700 Subject: [PATCH 15/67] Add memoization, some debug niceties --- Parser/pegen/vm.c | 73 ++++++++++++++++++++++++++++++++--------------- Parser/pegen/vm.h | 19 +++++++++--- 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 17719f25d5fd7b..79d151d8a04968 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -7,11 +7,20 @@ #include "vm.h" +#undef D +#define DEBUG 0 + +#if DEBUG +#define D(x) x +#else +#define D(x) +#endif + static Frame * push_frame(Stack *stack, Rule *rule) { - printf(" push %s\n", rule->name); - assert(stack->top < 100); + D(printf(" push %s\n", rule->name)); + assert(stack->top < MAXFRAMES); Frame *f = &stack->frames[stack->top++]; f->rule = rule; f->mark = stack->p->mark; @@ -25,15 +34,16 @@ push_frame(Stack *stack, Rule *rule) } static Frame * -pop_frame(Stack *stack) +pop_frame(Stack *stack, void *v) { assert(stack->top > 1); Frame *f = &stack->frames[--stack->top]; // Frame being popped if (f->collection) { PyMem_Free(f->collection); } + _PyPegen_insert_memo(stack->p, f->mark, f->rule->type, v); f = &stack->frames[stack->top - 1]; // New top of stack - printf(" pop %s\n", f->rule->name); + D(printf(" pop %s\n", f->rule->name)); return f; } @@ -57,18 +67,22 @@ run_vm(Parser *p, Rule rules[], int start) Frame *f = push_frame(&stack, &rules[start]); void *v; int oparg; + int opc; top: - if (p->mark == p->fill) - _PyPegen_fill_token(p); - for (int i = 0; i < stack.top; i++) printf(" "); - int opc = f->rule->opcodes[f->iop]; - printf("Rule: %s; ialt=%d; iop=%d; op=%s; arg=%d, mark=%d; token=%d; p^='%s'\n", - f->rule->name, f->ialt, f->iop, opcode_names[opc], - opc >= OP_TOKEN ? f->rule->opcodes[f->iop + 1] : -1, - p->mark, p->tokens[p->mark]->type, - p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); - switch (f->rule->opcodes[f->iop++]) { + opc = f->rule->opcodes[f->iop]; + if (DEBUG) { + if (p->mark == p->fill) + _PyPegen_fill_token(p); + for (int i = 0; i < stack.top; i++) printf(" "); + printf("Rule: %s; ialt=%d; iop=%d; op=%s; arg=%d, mark=%d; token=%d; p^='%s'\n", + f->rule->name, f->ialt, f->iop, opcode_names[opc], + opc >= OP_TOKEN ? f->rule->opcodes[f->iop + 1] : -1, + p->mark, p->tokens[p->mark]->type, + p->fill > p-> mark ? PyBytes_AsString(p->tokens[p->mark]->bytes) : ""); + } + f->iop++; + switch (opc) { case OP_NOOP: goto top; case OP_NAME: @@ -104,7 +118,7 @@ run_vm(Parser *p, Rule rules[], int start) goto top; case OP_LOOP_COLLECT: v = make_asdl_seq(p, f->collection, f->ncollected); - f = pop_frame(&stack); + f = pop_frame(&stack, v); if (!v) { return PyErr_NoMemory(); } @@ -115,12 +129,22 @@ run_vm(Parser *p, Rule rules[], int start) break; case OP_RULE: oparg = f->rule->opcodes[f->iop++]; - f = push_frame(&stack, &rules[oparg]); + Rule *rule = &rules[oparg]; + int memo = _PyPegen_is_memoized(p, rule->type, &v); + if (memo) { + if (memo < 0) { + return NULL; + } + else { + break; // The result is v + } + } + f = push_frame(&stack, rule); goto top; case OP_RETURN: oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); - f = pop_frame(&stack); + f = pop_frame(&stack, v); break; case OP_SUCCESS: oparg = f->rule->opcodes[f->iop++]; @@ -132,29 +156,32 @@ run_vm(Parser *p, Rule rules[], int start) case OP_FAILURE: return RAISE_SYNTAX_ERROR("A syntax error"); default: - abort(); + printf("opc=%d\n", opc); + assert(0); } if (v) { - printf(" OK\n"); + D(printf(" OK\n")); + assert(f->ival < MAXVALS); f->vals[f->ival++] = v; goto top; } if (PyErr_Occurred()) { - printf(" PyErr\n"); + D(printf(" PyErr\n")); p->error_indicator = 1; return NULL; } fail: if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { - printf(" GA!\n"); + D(printf(" GA!\n")); + assert(f->ival < MAXVALS); f->vals[f->ival++] = NULL; f->iop++; // Skip over the OP_OPTIONAL opcode goto top; } - printf(" fail\n"); + D(printf(" fail\n")); p->mark = f->mark; if (f->cut) goto pop; @@ -164,7 +191,7 @@ run_vm(Parser *p, Rule rules[], int start) goto top; pop: - f = pop_frame(&stack); + f = pop_frame(&stack, NULL); goto fail; } diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 8eee05d8649266..b51e55a865e912 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -36,10 +36,13 @@ char *opcode_names[] = { typedef struct _rule { char *name; + int type; int alts[10]; int opcodes[100]; } Rule; +#define MAXVALS 10 + typedef struct _frame { Rule *rule; int mark; @@ -49,13 +52,15 @@ typedef struct _frame { int ncollected; void **collection; int ival; - void *vals[10]; + void *vals[MAXVALS]; } Frame; +#define MAXFRAMES 100 + typedef struct _stack { Parser *p; int top; - Frame frames[100]; + Frame frames[MAXFRAMES]; } Stack; /* Toy grammar @@ -98,6 +103,7 @@ enum { static Rule all_rules[] = { {"start", + R_START, {0, 6, -1}, { OP_RULE, R_STMTS, @@ -109,6 +115,7 @@ static Rule all_rules[] = { }, {"stmt*", + R_STMTS, {0, 4, -1}, { OP_LOOP_START, @@ -120,6 +127,7 @@ static Rule all_rules[] = { }, {"stmt", + R_STMT, {0, -1}, { OP_RULE, R_EXPR, @@ -129,6 +137,7 @@ static Rule all_rules[] = { }, {"expr", + R_EXPR, {0, 8, -1}, { OP_RULE, R_TERM, @@ -142,6 +151,7 @@ static Rule all_rules[] = { }, {"term", + R_TERM, {0, 8, -1}, { OP_RULE, R_FACTOR, @@ -155,7 +165,8 @@ static Rule all_rules[] = { }, {"factor", - {0, 8, -1}, + R_FACTOR, + {0, 8, 11, -1}, { OP_TOKEN, LPAR, OP_RULE, R_EXPR, @@ -202,7 +213,7 @@ call_action(Parser *p, Frame *f, int iaction) case A_TERM_1: return f->vals[0]; case A_FACTOR_0: - return f->vals[0]; + return f->vals[1]; case A_FACTOR_1: return f->vals[0]; case A_FACTOR_2: From bda35171b1be421cf2f7760d92e4e17a07ce2ff9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 13:32:57 -0700 Subject: [PATCH 16/67] Inline helper functions --- Parser/pegen/vm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 79d151d8a04968..4e30ae657e4980 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -16,7 +16,7 @@ #define D(x) #endif -static Frame * +static inline Frame * push_frame(Stack *stack, Rule *rule) { D(printf(" push %s\n", rule->name)); @@ -33,7 +33,7 @@ push_frame(Stack *stack, Rule *rule) return f; } -static Frame * +static inline Frame * pop_frame(Stack *stack, void *v) { assert(stack->top > 1); @@ -47,7 +47,7 @@ pop_frame(Stack *stack, void *v) return f; } -static asdl_seq * +static inline asdl_seq * make_asdl_seq(Parser *p, void *collection[], int ncollected) { asdl_seq *seq = _Py_asdl_seq_new(ncollected, p->arena); From 881d7563c25aa8da8de1d04b6d6cc58bd38e9e34 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 15:47:53 -0700 Subject: [PATCH 17/67] Explain OP_OPTIONAL better --- Parser/pegen/vmreadme.md | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index a5095f8d4d8680..bc4d9247710488 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -64,7 +64,7 @@ Most operations can succeed or fail, and produce a vaue if they succeed. If an operation succeeds, the value is appended to the frame's values -stack, and the VM proceeds to the next opcode. +array (`vals`), and the VM proceeds to the next opcode. If an operation fails, the VM resets the input to the frame's mark, and then proceeds to the next alternative of the frame's rule, if @@ -92,8 +92,8 @@ The following opcodes take no argument. - `OP_CUT` -- set the frame's `cut` flag; succeeds without a value. -- `OP_OPTIONAL` -- modifies the *previous* operation to treat a `NULL` - result as a success. +- `OP_OPTIONAL` -- always succeeds, but modifies the *previous* + operation to treat a `NULL` result as a success. (See below.) These operations are followed by a single integer argument. @@ -154,6 +154,36 @@ The operations are defined as follows: (TODO: additional operations to support `a+` and `b.a+`.) +More about `OP_OPTIONAL` +------------------------ + +The `OP_OPTIONAL` flag is a "postfix" operation. It must *follow* any +operation that may produce a result. The normal way of the VM is that +if the result is `NULL` this is treated as a failure by the VM. But +before treating `NULL` as a failure, the VM checks whether the next +operation is `OP_OPTIONAL`. If so, it treats `NULL` as a success. In +this case a `NULL` is appended to the `vals` array and control flows +to the next operation. + +When the operation preceding `OP_OPTIONAL` succeeds, `OP_OPTIONAL` is +executed as a regular operation, and always succeed. + +Constraints on operation order +------------------------------ + +Note that for operations that succeed with a value or fail, there is +always a next operation. These operations are `OP_NAME`, `OP_NUMBER`, +`OP_STRING`, `OP_TOKEN`, and `OP_RULE`. + +These operations always succeed: `OP_NOOP`, `OP_CUT`, `OP_OPTIONAL`, +`OP_START`. + +These operations must be last in their alternative: `OP_RETURN`, +`OP_SUCCESS`, `OP_FAILURE`, `OP_LOOP_ITERATE`, `OP_LOOP_COLLECT`. + +These operations must be first in their alternative: `OP_LOOP_START`, +`OP_FAILURE`. + Ideas ----- From 94cfb95c946af88fdafe29451774017ec6395a1d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 29 May 2020 23:51:37 +0100 Subject: [PATCH 18/67] Skeleton of code generator Execute as "python -m pegen.vm_generator" in the Tools folder --- Tools/peg_generator/data/simple.gram | 12 +++ Tools/peg_generator/pegen/vm_generator.py | 97 +++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 Tools/peg_generator/data/simple.gram create mode 100644 Tools/peg_generator/pegen/vm_generator.py diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram new file mode 100644 index 00000000000000..4d0415581f6728 --- /dev/null +++ b/Tools/peg_generator/data/simple.gram @@ -0,0 +1,12 @@ +start: stmt* ENDMARKER {_PyPegen_make_module(p, f->vals[0])} +stmt: expr NEWLINE {_Py_Expr(f->vals[0], EXTRA)} +expr: + | term '+' expr {_Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA)} + | term {f->vals[0]} +term: + | factor '*' term {f->vals[0]} + | factor {f->vals[0]} +factor: + | '(' expr ')' {f->vals[1]} + | NUMBER {f->vals[0]} + | NAME {f->vals[0]} \ No newline at end of file diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py new file mode 100644 index 00000000000000..0cc57395feb5b0 --- /dev/null +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -0,0 +1,97 @@ +from pegen.build import build_parser +import contextlib +import token +import sys + +from typing import Any, Dict, Optional, IO, Text, Tuple + +from pegen import grammar +from pegen.grammar import ( + Alt, + Cut, + Gather, + GrammarVisitor, + Group, + Lookahead, + NamedItem, + NameLeaf, + NegativeLookahead, + Opt, + PositiveLookahead, + Repeat0, + Repeat1, + Rhs, + Rule, + StringLeaf, +) + +from pegen.parser_generator import ParserGenerator + + +class CParserGenerator(ParserGenerator, GrammarVisitor): + def __init__( + self, + grammar: grammar.Grammar, + ): + super().__init__(grammar, token.tok_name, sys.stdout) + + def generate(self, filename: str) -> None: + while self.todo: + self.print("static Rule all_rules[] = {") + for rulename, rule in list(self.todo.items()): + del self.todo[rulename] + with self.indent(): + self.visit(rule) + self.print("}") + + def visit_Rule(self, node: Rule) -> None: + is_loop = node.is_loop() + is_gather = node.is_gather() + rhs = node.flatten() + if node.left_recursive: + raise ValueError("No left recursiveness") + self.print(f'{{"{node.name}",') + self.print(f' R_{node.name.upper()},') + self.print(' {TODO_INDEX_INTO_OPCODES_FOR_EVERY_ALT, -1},') + self.print(' {') + with self.indent(): + self.current_rule = node #TODO: make this a context manager + self.visit(rhs, is_loop=is_loop, is_gather=is_gather) + self.print('},') + + def visit_NamedItem(self, node: NamedItem) -> None: + self.visit(node.item) + + def _get_rule_opcode(self, name: str) -> str: + return f"R_{name.upper()}" + + def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + name = node.value + if name in ("NAME", "NUMBER", "STRING"): + self.print(f"OP_{name}") + elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): + self.print(f'OP_TOKEN, {name},') + else: + self.print(f'OP_RULE, {self._get_rule_opcode(name)},') + + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + tok_str = node.value.replace("'", "") + tok_num = token.EXACT_TOKEN_TYPES[tok_str] + tok_name = token.tok_name[tok_num] + self.print(f"OP_TOKEN, {tok_name}") + + def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: + for index, alt in enumerate(node.alts): + self.visit(alt, is_loop=is_loop, is_gather=is_gather) + if alt.action: + self.print(f"OP_RETURN, A_{self.current_rule.name.upper()}_{index},") + self.print(f"") + + def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: + for item in node.items: + self.visit(item) + +GRAMMAR_FILE = ".data/simple.gram" +grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) +p = CParserGenerator(grammar) +p.generate("") \ No newline at end of file From 4ba6c6136c004c3fa10dcf882d9c68675ff4b4ac Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 15:59:37 -0700 Subject: [PATCH 19/67] Simplify structure of OP_SUCCESS --- Parser/pegen/vm.c | 12 ++++-------- Parser/pegen/vm.h | 18 +++++++++++++++--- Parser/pegen/vmreadme.md | 7 ++++--- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 4e30ae657e4980..62ed9416691e26 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -61,10 +61,10 @@ make_asdl_seq(Parser *p, void *collection[], int ncollected) } static void * -run_vm(Parser *p, Rule rules[], int start) +run_vm(Parser *p, Rule rules[], int root) { Stack stack = {p, 0, {{0}}}; - Frame *f = push_frame(&stack, &rules[start]); + Frame *f = push_frame(&stack, &rules[root]); void *v; int oparg; int opc; @@ -147,12 +147,8 @@ run_vm(Parser *p, Rule rules[], int start) f = pop_frame(&stack, v); break; case OP_SUCCESS: - oparg = f->rule->opcodes[f->iop++]; - v = call_action(p, f, oparg); - if (v || PyErr_Occurred()) { - return v; - } - return RAISE_SYNTAX_ERROR("Final action failed"); + v = f->vals[0]; + return v; case OP_FAILURE: return RAISE_SYNTAX_ERROR("A syntax error"); default: diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index b51e55a865e912..0a7e3a72c1f540 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -7,13 +7,13 @@ typedef enum _opcodes { OP_OPTIONAL, OP_LOOP_START, OP_LOOP_ITERATE, + OP_SUCCESS, OP_FAILURE, // The rest have an argument OP_TOKEN, OP_RULE, OP_RETURN, OP_LOOP_COLLECT, - OP_SUCCESS, } Opcode; char *opcode_names[] = { @@ -25,13 +25,13 @@ char *opcode_names[] = { "OP_OPTIONAL", "OP_LOOP_START", "OP_LOOP_ITERATE", + "OP_SUCCESS", "OP_FAILURE", // The rest have an argument "OP_TOKEN", "OP_RULE", "OP_RETURN", "OP_LOOP_COLLECT", - "OP_SUCCESS", }; typedef struct _rule { @@ -80,6 +80,7 @@ static KeywordToken *reserved_keywords[] = { }; enum { + R_ROOT, R_START, R_STMTS, R_STMT, @@ -102,13 +103,24 @@ enum { static Rule all_rules[] = { + {"root", + R_ROOT, + {0, 3, -1}, + { + OP_RULE, R_START, + OP_SUCCESS, + + OP_FAILURE, + }, + }, + {"start", R_START, {0, 6, -1}, { OP_RULE, R_STMTS, OP_TOKEN, ENDMARKER, - OP_SUCCESS, A_START_0, + OP_RETURN, A_START_0, OP_FAILURE, }, diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index bc4d9247710488..0a66945776f74e 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -112,10 +112,11 @@ These operations are followed by a single integer argument. ### Operations for start rules only -- `OP_SUCCESS(action)` -- call the action given by the argument, and - exit the VM with its return value as result. +- `OP_SUCCESS` -- exit the VM with the first collected value as + result. -- `OP_FAILURE` -- report a syntax error and exit the VM. +- `OP_FAILURE` -- report a syntax error and exit the VM with a NULL + result. ### Looping operations From db628e1f784ca3395bd524580a5b9c90811696c4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 16:08:50 -0700 Subject: [PATCH 20/67] Move opcodes around --- Parser/pegen/vm.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 0a7e3a72c1f540..8a844a8233a86c 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -1,37 +1,37 @@ typedef enum _opcodes { OP_NOOP, + OP_CUT, + OP_OPTIONAL, OP_NAME, OP_NUMBER, OP_STRING, - OP_CUT, - OP_OPTIONAL, OP_LOOP_START, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, OP_SUCCESS, OP_FAILURE, // The rest have an argument OP_TOKEN, OP_RULE, OP_RETURN, - OP_LOOP_COLLECT, } Opcode; char *opcode_names[] = { "OP_NOOP", + "OP_CUT", + "OP_OPTIONAL", "OP_NAME", "OP_NUMBER", "OP_STRING", - "OP_CUT", - "OP_OPTIONAL", "OP_LOOP_START", "OP_LOOP_ITERATE", + "OP_LOOP_COLLECT", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument "OP_TOKEN", "OP_RULE", "OP_RETURN", - "OP_LOOP_COLLECT", }; typedef struct _rule { From 7798629e0b3f8f344ba5e986de42c93b67c74166 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 16:09:07 -0700 Subject: [PATCH 21/67] Add a 'grammar' for operations --- Parser/pegen/vmreadme.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 0a66945776f74e..614628affbbfb9 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -185,6 +185,34 @@ These operations must be last in their alternative: `OP_RETURN`, These operations must be first in their alternative: `OP_LOOP_START`, `OP_FAILURE`. +Grammar for lists of operations +------------------------------- + +This shows the constraints on how operations can be used together. + +``` +root_rule: success_alt failure_alt + +success_alt: rule_op OP_SUCCESS +failure_alt: OP_FAILURE + +normal_rule: alt+ + +loop_rule: loop_start_alt loop_collect_alt + +loop_start_alt: OP_LOOP_START regular_op OP_LOOP_ITERATE +loop_collect_alt: OP_LOOP_COLLECT + +alt: any_op+ return_op + +any_op: regular_op [OP_OPTIONAL] | special_op +regular_op: short_op | long_op +short_op: OP_NAME | OP_NUMBER | OP_STRING +long_op: OP_TOKEN | OP_RULE +special_op: OP_NOOP | OP_CUT +return_op: OP_RETURN +``` + Ideas ----- From 412c741bc05d356b7c5ef1bac906b1846e3b2615 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 16:30:50 -0700 Subject: [PATCH 22/67] Move generated part of vm.h into vmparse.h --- Makefile.pre.in | 2 +- Parser/pegen/vm.c | 9 ++ Parser/pegen/vm.h | 189 ++--------------------------------------- Parser/pegen/vmparse.h | 171 +++++++++++++++++++++++++++++++++++++ 4 files changed, 187 insertions(+), 184 deletions(-) create mode 100644 Parser/pegen/vmparse.h diff --git a/Makefile.pre.in b/Makefile.pre.in index e9a94f8b2b46ae..92544f32c034b6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1151,7 +1151,7 @@ PYTHON_HEADERS= \ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) -$(srcdir)/Parser/pegen/vm.o: $(srcdir)/Parser/pegen/vm.h +$(srcdir)/Parser/pegen/vm.o: $(srcdir)/Parser/pegen/vm.h $(srcdir)/Parser/pegen/vmparse.h ###################################################################### diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 62ed9416691e26..4dc83b1f13b0ce 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -6,6 +6,7 @@ #include "parse_string.h" #include "vm.h" +#include "vmparse.h" // Generated parser tables #undef D #define DEBUG 0 @@ -16,6 +17,14 @@ #define D(x) #endif +#define MAXFRAMES 100 + +typedef struct _stack { + Parser *p; + int top; + Frame frames[MAXFRAMES]; +} Stack; + static inline Frame * push_frame(Stack *stack, Rule *rule) { diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 8a844a8233a86c..387eb9bdeafa3e 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -16,7 +16,7 @@ typedef enum _opcodes { OP_RETURN, } Opcode; -char *opcode_names[] = { +static char *opcode_names[] = { "OP_NOOP", "OP_CUT", "OP_OPTIONAL", @@ -34,11 +34,14 @@ char *opcode_names[] = { "OP_RETURN", }; +#define MAXALTS 10 +#define MAXOPCODES 100 + typedef struct _rule { char *name; int type; - int alts[10]; - int opcodes[100]; + int alts[MAXALTS]; + int opcodes[MAXOPCODES]; } Rule; #define MAXVALS 10 @@ -54,183 +57,3 @@ typedef struct _frame { int ival; void *vals[MAXVALS]; } Frame; - -#define MAXFRAMES 100 - -typedef struct _stack { - Parser *p; - int top; - Frame frames[MAXFRAMES]; -} Stack; - -/* Toy grammar - -NOTE: [expr] is right-recursive. - -start: stmt* ENDMARKER -stmt: expr NEWLINE -expr: term '+' expr | term -term: factor '*' term | factor -factor: '(' expr ')' | NUMBER | NAME - -*/ - -static const int n_keyword_lists = 0; -static KeywordToken *reserved_keywords[] = { -}; - -enum { - R_ROOT, - R_START, - R_STMTS, - R_STMT, - R_EXPR, - R_TERM, - R_FACTOR, -}; - -enum { - A_START_0, - A_STMT_0, - A_EXPR_0, - A_EXPR_1, - A_TERM_0, - A_TERM_1, - A_FACTOR_0, - A_FACTOR_1, - A_FACTOR_2, -}; - -static Rule all_rules[] = { - - {"root", - R_ROOT, - {0, 3, -1}, - { - OP_RULE, R_START, - OP_SUCCESS, - - OP_FAILURE, - }, - }, - - {"start", - R_START, - {0, 6, -1}, - { - OP_RULE, R_STMTS, - OP_TOKEN, ENDMARKER, - OP_RETURN, A_START_0, - - OP_FAILURE, - }, - }, - - {"stmt*", - R_STMTS, - {0, 4, -1}, - { - OP_LOOP_START, - OP_RULE, R_STMT, - OP_LOOP_ITERATE, - - OP_LOOP_COLLECT, - }, - }, - - {"stmt", - R_STMT, - {0, -1}, - { - OP_RULE, R_EXPR, - OP_TOKEN, NEWLINE, - OP_RETURN, A_STMT_0, - }, - }, - - {"expr", - R_EXPR, - {0, 8, -1}, - { - OP_RULE, R_TERM, - OP_TOKEN, PLUS, - OP_RULE, R_EXPR, - OP_RETURN, A_EXPR_0, - - OP_RULE, R_TERM, - OP_RETURN, A_EXPR_1, - }, - }, - - {"term", - R_TERM, - {0, 8, -1}, - { - OP_RULE, R_FACTOR, - OP_TOKEN, STAR, - OP_RULE, R_TERM, - OP_RETURN, A_TERM_0, - - OP_RULE, R_FACTOR, - OP_RETURN, A_TERM_1, - }, - }, - - {"factor", - R_FACTOR, - {0, 8, 11, -1}, - { - OP_TOKEN, LPAR, - OP_RULE, R_EXPR, - OP_TOKEN, RPAR, - OP_RETURN, A_FACTOR_0, - - OP_NUMBER, - OP_RETURN, A_FACTOR_1, - - OP_NAME, - OP_RETURN, A_FACTOR_2, - }, - }, - -}; - -static void * -call_action(Parser *p, Frame *f, int iaction) -{ - Token *t = p->tokens[f->mark]; - int _start_lineno = t->lineno; - int _start_col_offset = t->col_offset; - if (p->mark > 0) { - t = p->tokens[p->mark - 1]; - } - else { - // Debug anomaly rather than crashing - printf("p->mark=%d, p->fill=%d, f->mark=%d\n", p->mark, p->fill, f->mark); - } - int _end_lineno = t->end_lineno; - int _end_col_offset = t->end_col_offset; - - switch (iaction) { - case A_START_0: - return _PyPegen_make_module(p, f->vals[0]); - case A_STMT_0: - return _Py_Expr(f->vals[0], EXTRA); - case A_EXPR_0: - return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); - case A_EXPR_1: - return f->vals[0]; - case A_TERM_0: - return f->vals[0]; - case A_TERM_1: - return f->vals[0]; - case A_FACTOR_0: - return f->vals[1]; - case A_FACTOR_1: - return f->vals[0]; - case A_FACTOR_2: - return f->vals[0]; - default: - assert(0); - } -} diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h new file mode 100644 index 00000000000000..be8b84b9923b50 --- /dev/null +++ b/Parser/pegen/vmparse.h @@ -0,0 +1,171 @@ +/* Toy grammar + +NOTE: [expr] is right-recursive. + +start: stmt* ENDMARKER +stmt: expr NEWLINE +expr: term '+' expr | term +term: factor '*' term | factor +factor: '(' expr ')' | NUMBER | NAME + +*/ + +static const int n_keyword_lists = 0; +static KeywordToken *reserved_keywords[] = { +}; + +enum { + R_ROOT, + R_START, + R_STMTS, + R_STMT, + R_EXPR, + R_TERM, + R_FACTOR, +}; + +enum { + A_START_0, + A_STMT_0, + A_EXPR_0, + A_EXPR_1, + A_TERM_0, + A_TERM_1, + A_FACTOR_0, + A_FACTOR_1, + A_FACTOR_2, +}; + +static Rule all_rules[] = { + + {"root", + R_ROOT, + {0, 3, -1}, + { + OP_RULE, R_START, + OP_SUCCESS, + + OP_FAILURE, + }, + }, + + {"start", + R_START, + {0, 6, -1}, + { + OP_RULE, R_STMTS, + OP_TOKEN, ENDMARKER, + OP_RETURN, A_START_0, + + OP_FAILURE, + }, + }, + + {"stmt*", + R_STMTS, + {0, 4, -1}, + { + OP_LOOP_START, + OP_RULE, R_STMT, + OP_LOOP_ITERATE, + + OP_LOOP_COLLECT, + }, + }, + + {"stmt", + R_STMT, + {0, -1}, + { + OP_RULE, R_EXPR, + OP_TOKEN, NEWLINE, + OP_RETURN, A_STMT_0, + }, + }, + + {"expr", + R_EXPR, + {0, 8, -1}, + { + OP_RULE, R_TERM, + OP_TOKEN, PLUS, + OP_RULE, R_EXPR, + OP_RETURN, A_EXPR_0, + + OP_RULE, R_TERM, + OP_RETURN, A_EXPR_1, + }, + }, + + {"term", + R_TERM, + {0, 8, -1}, + { + OP_RULE, R_FACTOR, + OP_TOKEN, STAR, + OP_RULE, R_TERM, + OP_RETURN, A_TERM_0, + + OP_RULE, R_FACTOR, + OP_RETURN, A_TERM_1, + }, + }, + + {"factor", + R_FACTOR, + {0, 8, 11, -1}, + { + OP_TOKEN, LPAR, + OP_RULE, R_EXPR, + OP_TOKEN, RPAR, + OP_RETURN, A_FACTOR_0, + + OP_NUMBER, + OP_RETURN, A_FACTOR_1, + + OP_NAME, + OP_RETURN, A_FACTOR_2, + }, + }, + +}; + +static void * +call_action(Parser *p, Frame *f, int iaction) +{ + Token *t = p->tokens[f->mark]; + int _start_lineno = t->lineno; + int _start_col_offset = t->col_offset; + if (p->mark > 0) { + t = p->tokens[p->mark - 1]; + } + else { + // Debug anomaly rather than crashing + printf("p->mark=%d, p->fill=%d, f->mark=%d\n", p->mark, p->fill, f->mark); + } + int _end_lineno = t->end_lineno; + int _end_col_offset = t->end_col_offset; + + switch (iaction) { + case A_START_0: + return _PyPegen_make_module(p, f->vals[0]); + case A_STMT_0: + return _Py_Expr(f->vals[0], EXTRA); + case A_EXPR_0: + return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); + case A_EXPR_1: + return f->vals[0]; + case A_TERM_0: + return f->vals[0]; + case A_TERM_1: + return f->vals[0]; + case A_FACTOR_0: + return f->vals[1]; + case A_FACTOR_1: + return f->vals[0]; + case A_FACTOR_2: + return f->vals[0]; + default: + assert(0); + } +} From ef1fabdaf4cf1b80266d2d5f3d468886084d6e40 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 30 May 2020 01:01:16 +0100 Subject: [PATCH 23/67] Clean skeleton of vm_generator --- Tools/peg_generator/pegen/vm_generator.py | 86 +++++++++++++++-------- 1 file changed, 58 insertions(+), 28 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 0cc57395feb5b0..ca5fcdd3e32f7c 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -1,11 +1,12 @@ -from pegen.build import build_parser import contextlib -import token import sys - -from typing import Any, Dict, Optional, IO, Text, Tuple +import token +from collections import defaultdict +from itertools import accumulate +from typing import Dict, Iterator, List, Optional from pegen import grammar +from pegen.build import build_parser from pegen.grammar import ( Alt, Cut, @@ -24,17 +25,27 @@ Rule, StringLeaf, ) - from pegen.parser_generator import ParserGenerator class CParserGenerator(ParserGenerator, GrammarVisitor): def __init__( - self, - grammar: grammar.Grammar, + self, grammar: grammar.Grammar, ): super().__init__(grammar, token.tok_name, sys.stdout) + self.opcode_buffer: Optional[List[str]] = None + + @contextlib.contextmanager + def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: + self.opcode_buffer = buffer + yield + self.opcode_buffer = None + + def add_opcode(self, opcode: str) -> None: + assert self.opcode_buffer is not None + self.opcode_buffer.append(opcode) + def generate(self, filename: str) -> None: while self.todo: self.print("static Rule all_rules[] = {") @@ -51,47 +62,66 @@ def visit_Rule(self, node: Rule) -> None: if node.left_recursive: raise ValueError("No left recursiveness") self.print(f'{{"{node.name}",') - self.print(f' R_{node.name.upper()},') - self.print(' {TODO_INDEX_INTO_OPCODES_FOR_EVERY_ALT, -1},') - self.print(' {') + self.print(f" R_{node.name.upper()},") + self.current_rule = node # TODO: make this a context manager with self.indent(): - self.current_rule = node #TODO: make this a context manager self.visit(rhs, is_loop=is_loop, is_gather=is_gather) - self.print('},') + self.print("},") def visit_NamedItem(self, node: NamedItem) -> None: self.visit(node.item) - + def _get_rule_opcode(self, name: str) -> str: return f"R_{name.upper()}" - - def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + + def visit_NameLeaf(self, node: NameLeaf) -> None: name = node.value if name in ("NAME", "NUMBER", "STRING"): - self.print(f"OP_{name}") + self.add_opcode(f"OP_{name}") elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): - self.print(f'OP_TOKEN, {name},') + self.add_opcode("OP_TOKEN") + self.add_opcode(name) else: - self.print(f'OP_RULE, {self._get_rule_opcode(name)},') + self.add_opcode("OP_RULE") + self.add_opcode(self._get_rule_opcode(name)) - def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + def visit_StringLeaf(self, node: StringLeaf) -> None: + self.add_opcode("OP_TOKEN") tok_str = node.value.replace("'", "") tok_num = token.EXACT_TOKEN_TYPES[tok_str] tok_name = token.tok_name[tok_num] - self.print(f"OP_TOKEN, {tok_name}") + self.add_opcode(tok_name) def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: + opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) for index, alt in enumerate(node.alts): - self.visit(alt, is_loop=is_loop, is_gather=is_gather) - if alt.action: - self.print(f"OP_RETURN, A_{self.current_rule.name.upper()}_{index},") - self.print(f"") + with self.set_opcode_buffer(opcodes_by_alt[index]): + self.visit(alt, is_loop=is_loop, is_gather=is_gather) + if alt.action: + self.add_opcode("OP_RETURN") + self.add_opcode(f"A_{self.current_rule.name.upper()}_{index}") + + *indexes, _ = accumulate(map(len, opcodes_by_alt.values())) + indexes = [0, *indexes, -1] + self.print(f" {{{', '.join(map(str, indexes))}}},") + + self.print(" {") + with self.indent(): + for index, opcodes in opcodes_by_alt.items(): + self.print(", ".join(opcodes) + ",") + self.print() + self.print(" },") def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: for item in node.items: self.visit(item) -GRAMMAR_FILE = ".data/simple.gram" -grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) -p = CParserGenerator(grammar) -p.generate("") \ No newline at end of file + +def main() -> None: + grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) + p = CParserGenerator(grammar) + p.generate("") + + +if __name__ == "__main__": + main() From 8eab7e0dcfc70badebd821fcc5e2606337b274f5 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 30 May 2020 03:09:08 +0300 Subject: [PATCH 24/67] Better formatting of generated file; remove unneeded indentation --- Tools/peg_generator/pegen/vm_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index ca5fcdd3e32f7c..40b32612188b1c 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -64,8 +64,7 @@ def visit_Rule(self, node: Rule) -> None: self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") self.current_rule = node # TODO: make this a context manager - with self.indent(): - self.visit(rhs, is_loop=is_loop, is_gather=is_gather) + self.visit(rhs, is_loop=is_loop, is_gather=is_gather) self.print("},") def visit_NamedItem(self, node: NamedItem) -> None: From 4a5f8236c52bd7e1046522f8f669321fc010907d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 17:09:24 -0700 Subject: [PATCH 25/67] Add OP_LOOP_COLLECT_NONEMPTY -- used for a+ --- Parser/pegen/vm.c | 6 ++++++ Parser/pegen/vm.h | 2 ++ Parser/pegen/vmreadme.md | 12 ++++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 4dc83b1f13b0ce..1011af9389ab93 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -125,6 +125,12 @@ run_vm(Parser *p, Rule rules[], int root) f->iop = f->rule->alts[f->ialt] + 1; // Skip OP_LOOP_START operation f->ival = 0; goto top; + case OP_LOOP_COLLECT_NONEMPTY: + if (!f->ncollected) { + v = NULL; + f = pop_frame(&stack, v); + break; + } case OP_LOOP_COLLECT: v = make_asdl_seq(p, f->collection, f->ncollected); f = pop_frame(&stack, v); diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 387eb9bdeafa3e..dc940afeb839eb 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -8,6 +8,7 @@ typedef enum _opcodes { OP_LOOP_START, OP_LOOP_ITERATE, OP_LOOP_COLLECT, + OP_LOOP_COLLECT_NONEMPTY, OP_SUCCESS, OP_FAILURE, // The rest have an argument @@ -26,6 +27,7 @@ static char *opcode_names[] = { "OP_LOOP_START", "OP_LOOP_ITERATE", "OP_LOOP_COLLECT", + "OP_LOOP_COLLECT_NONEMPTY", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 614628affbbfb9..ef3d7381d49985 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -130,7 +130,7 @@ OP_LOOP_START OP_LOOP_ITERATE # Second alternative: -OP_LOOP_COLLECT +Either OP_LOOP_COLLECT or OP_LOOP_COLLECT_NONEMPTY ``` The values being collected are stored in a `malloc`-ed array named @@ -153,7 +153,10 @@ The operations are defined as follows: position and pop the frame off the stack, producing a new value that is an `asdl_seq *` containing the collected values. -(TODO: additional operations to support `a+` and `b.a+`.) +- `OP_LOOP_COLLECT_NONEMPTY` -- like `OP_LOOP_COLLECT` but fails if no + values are collected. + +(TODO: additional operations to support `b.a+`.) More about `OP_OPTIONAL` ------------------------ @@ -180,7 +183,8 @@ These operations always succeed: `OP_NOOP`, `OP_CUT`, `OP_OPTIONAL`, `OP_START`. These operations must be last in their alternative: `OP_RETURN`, -`OP_SUCCESS`, `OP_FAILURE`, `OP_LOOP_ITERATE`, `OP_LOOP_COLLECT`. +`OP_SUCCESS`, `OP_FAILURE`, `OP_LOOP_ITERATE`, `OP_LOOP_COLLECT`, +`OP_LOOP_COLLECT_NONEMPTY`. These operations must be first in their alternative: `OP_LOOP_START`, `OP_FAILURE`. @@ -201,7 +205,7 @@ normal_rule: alt+ loop_rule: loop_start_alt loop_collect_alt loop_start_alt: OP_LOOP_START regular_op OP_LOOP_ITERATE -loop_collect_alt: OP_LOOP_COLLECT +loop_collect_alt: OP_LOOP_COLLECT | OP_LOOP_COLLECT_NONEMPTY alt: any_op+ return_op From 3cee73c11746171e4165f02131440a5a0f516bfa Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 17:37:48 -0700 Subject: [PATCH 26/67] Expand description of root rules --- Parser/pegen/vmreadme.md | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index ef3d7381d49985..d9e33a6091c7ab 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -50,7 +50,7 @@ A `Rule` structure has the following fields: terminated by `-1` - `int opcodes[]` -- array of opcodes and their arguments -All rules are collected in a single array; the index in this array +All rules are combined in a single array; the index in this array is used by operations that reference other rules. The `opcodes` array is a sequence of operation codes and arguments. @@ -110,15 +110,25 @@ These operations are followed by a single integer argument. newly revealed by that pop operation) as if the previous operation succeeded or failed with the return value of the action. -### Operations for start rules only +### Operations for root rules only -- `OP_SUCCESS` -- exit the VM with the first collected value as - result. +A grammar must have one or more *root rules*. A root rule is a +synthetic rule that uses `OP_SUCCESS` and `OP_FAILURE` operations to +report overall success or failure. Only root rules may use these +operations, and they must be used in the right format. + +Each root rule must have exactly two alternatives. The first +alternative must be a single operation (generally `OP_RULE`) that +stores a value in the values array, followed by `OP_SUCCESS`. The +second alternative must be the single operation `OP_FAILURE`. + +- `OP_SUCCESS` -- exit the VM with the first value from the `vals` + array as the result. - `OP_FAILURE` -- report a syntax error and exit the VM with a NULL result. -### Looping operations +### Operations for loop rules only For a loop such as `a*`, a synthetic rule must be created with the following structure: @@ -143,7 +153,8 @@ fields: The operations are defined as follows: -- `OP_LOOP_START` -- initialize the `collections` array. +- `OP_LOOP_START` -- initialize the `collections` array. (TODO: Maybe + we don't need this?) - `OP_LOOP_ITERATE` -- append the current value to the `collections` array, save the current input position, and start the next iteration @@ -197,7 +208,7 @@ This shows the constraints on how operations can be used together. ``` root_rule: success_alt failure_alt -success_alt: rule_op OP_SUCCESS +success_alt: regular_op OP_SUCCESS failure_alt: OP_FAILURE normal_rule: alt+ From 9b96df08e4605695c2ba53f5bd1917e5c25c3ad7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 30 May 2020 01:52:00 +0100 Subject: [PATCH 27/67] Initial support for repeat_0 --- Tools/peg_generator/pegen/vm_generator.py | 51 +++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 40b32612188b1c..ad9bed265278a2 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -3,7 +3,7 @@ import token from collections import defaultdict from itertools import accumulate -from typing import Dict, Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Optional from pegen import grammar from pegen.build import build_parser @@ -18,6 +18,7 @@ NameLeaf, NegativeLookahead, Opt, + Plain, PositiveLookahead, Repeat0, Repeat1, @@ -28,6 +29,26 @@ from pegen.parser_generator import ParserGenerator +class CCallMakerVisitor(GrammarVisitor): + def __init__( + self, parser_generator: ParserGenerator, + ): + self.gen = parser_generator + self.cache: Dict[Any, Any] = {} + + def visit_Repeat0(self, node: Repeat0) -> None: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, False) + self.cache[node] = name + + def visit_Repeat1(self, node: Repeat1) -> None: + if node in self.cache: + return self.cache[node] + name = self.gen.name_loop(node.node, True) + self.cache[node] = name + + class CParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, @@ -35,6 +56,7 @@ def __init__( super().__init__(grammar, token.tok_name, sys.stdout) self.opcode_buffer: Optional[List[str]] = None + self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(self) @contextlib.contextmanager def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: @@ -47,6 +69,7 @@ def add_opcode(self, opcode: str) -> None: self.opcode_buffer.append(opcode) def generate(self, filename: str) -> None: + self.collect_todo() while self.todo: self.print("static Rule all_rules[] = {") for rulename, rule in list(self.todo.items()): @@ -91,15 +114,35 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: tok_name = token.tok_name[tok_num] self.add_opcode(tok_name) - def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: - opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) + def handle_loop_rhs(self, node: Rhs, opcodes_by_alt: Dict[int, List[str]]) -> None: + with self.set_opcode_buffer(opcodes_by_alt[0]): + self.add_opcode("OP_LOOP_START") + self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) + with self.set_opcode_buffer(opcodes_by_alt[len(node.alts)]): + self.add_opcode("OP_LOOP_COLLECT") + + def handle_default_rhs( + self, + node: Rhs, + opcodes_by_alt: Dict[int, List[str]], + is_loop: bool = False, + is_gather: bool = False, + ) -> None: for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[index]): - self.visit(alt, is_loop=is_loop, is_gather=is_gather) + self.visit(alt, is_loop=False, is_gather=False) if alt.action: self.add_opcode("OP_RETURN") self.add_opcode(f"A_{self.current_rule.name.upper()}_{index}") + if is_loop: + self.add_opcode("OP_LOOP_ITERATE") + def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: + opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) + if is_loop: + self.handle_loop_rhs(node, opcodes_by_alt) + else: + self.handle_default_rhs(node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather) *indexes, _ = accumulate(map(len, opcodes_by_alt.values())) indexes = [0, *indexes, -1] self.print(f" {{{', '.join(map(str, indexes))}}},") From 2f44ee951bf1ac745905a079bbe21098d92ead21 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 30 May 2020 01:56:58 +0100 Subject: [PATCH 28/67] Fix name rules for repeat0 nodes --- Tools/peg_generator/pegen/vm_generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index ad9bed265278a2..00d769381cd229 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -158,6 +158,11 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: for item in node.items: self.visit(item) + def visit_Repeat0(self, node: Repeat0) -> None: + name = self.callmakervisitor.visit(node) + self.add_opcode("OP_RULE") + self.add_opcode(self._get_rule_opcode(name)) + def main() -> None: grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) From 6dc70928363f013a187b494be6d7d8a2382a7e29 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 20:46:29 -0700 Subject: [PATCH 29/67] Eliminate OP_LOOP_START --- Parser/pegen/vm.c | 11 ++--------- Parser/pegen/vm.h | 2 -- Parser/pegen/vmparse.h | 3 +-- Parser/pegen/vmreadme.md | 13 ++++--------- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 1011af9389ab93..0cabbe53dc018f 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -105,13 +105,6 @@ run_vm(Parser *p, Rule rules[], int root) goto top; case OP_OPTIONAL: goto top; - case OP_LOOP_START: - f->ncollected = 0; - f->collection = PyMem_Malloc(0); - if (!f->collection) { - return PyErr_NoMemory(); - } - goto top; case OP_LOOP_ITERATE: f->mark = p->mark; assert(f->ival == 1); @@ -119,10 +112,10 @@ run_vm(Parser *p, Rule rules[], int root) assert(v); f->collection = PyMem_Realloc(f->collection, (f->ncollected + 1) * sizeof(void *)); if (!f->collection) { - return NULL; + return PyErr_NoMemory(); } f->collection[f->ncollected++] = v; - f->iop = f->rule->alts[f->ialt] + 1; // Skip OP_LOOP_START operation + f->iop = f->rule->alts[f->ialt]; f->ival = 0; goto top; case OP_LOOP_COLLECT_NONEMPTY: diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index dc940afeb839eb..8ad63eacd9a8f7 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -5,7 +5,6 @@ typedef enum _opcodes { OP_NAME, OP_NUMBER, OP_STRING, - OP_LOOP_START, OP_LOOP_ITERATE, OP_LOOP_COLLECT, OP_LOOP_COLLECT_NONEMPTY, @@ -24,7 +23,6 @@ static char *opcode_names[] = { "OP_NAME", "OP_NUMBER", "OP_STRING", - "OP_LOOP_START", "OP_LOOP_ITERATE", "OP_LOOP_COLLECT", "OP_LOOP_COLLECT_NONEMPTY", diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index be8b84b9923b50..31b3c6ea82ca36 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -63,9 +63,8 @@ static Rule all_rules[] = { {"stmt*", R_STMTS, - {0, 4, -1}, + {0, 3, -1}, { - OP_LOOP_START, OP_RULE, R_STMT, OP_LOOP_ITERATE, diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index d9e33a6091c7ab..5c541375eea680 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -130,12 +130,11 @@ second alternative must be the single operation `OP_FAILURE`. ### Operations for loop rules only -For a loop such as `a*`, a synthetic rule must be created with the -following structure: +For a loop such as `a*` or `a+`, a synthetic rule must be created with +the following structure: ``` # First alternative: -OP_LOOP_START OP_LOOP_ITERATE @@ -153,9 +152,6 @@ fields: The operations are defined as follows: -- `OP_LOOP_START` -- initialize the `collections` array. (TODO: Maybe - we don't need this?) - - `OP_LOOP_ITERATE` -- append the current value to the `collections` array, save the current input position, and start the next iteration of the loop (resetting the instruction pointer). @@ -197,8 +193,7 @@ These operations must be last in their alternative: `OP_RETURN`, `OP_SUCCESS`, `OP_FAILURE`, `OP_LOOP_ITERATE`, `OP_LOOP_COLLECT`, `OP_LOOP_COLLECT_NONEMPTY`. -These operations must be first in their alternative: `OP_LOOP_START`, -`OP_FAILURE`. +This operation must be first in its alternative: `OP_FAILURE`. Grammar for lists of operations ------------------------------- @@ -215,7 +210,7 @@ normal_rule: alt+ loop_rule: loop_start_alt loop_collect_alt -loop_start_alt: OP_LOOP_START regular_op OP_LOOP_ITERATE +loop_start_alt: regular_op OP_LOOP_ITERATE loop_collect_alt: OP_LOOP_COLLECT | OP_LOOP_COLLECT_NONEMPTY alt: any_op+ return_op From 9e7e12ed806cb00df51a833efc86fc364535fbe3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 20:54:56 -0700 Subject: [PATCH 30/67] Do fewer reallocs (at the cost of an extra int per frame) --- Parser/pegen/vm.c | 10 +++++++--- Parser/pegen/vm.h | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 0cabbe53dc018f..caf293fcd6b6c2 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -37,6 +37,7 @@ push_frame(Stack *stack, Rule *rule) f->iop = 0; f->cut = 0; f->ncollected = 0; + f->capacity = 0; f->collection = NULL; f->ival = 0; return f; @@ -110,9 +111,12 @@ run_vm(Parser *p, Rule rules[], int root) assert(f->ival == 1); v = f->vals[0]; assert(v); - f->collection = PyMem_Realloc(f->collection, (f->ncollected + 1) * sizeof(void *)); - if (!f->collection) { - return PyErr_NoMemory(); + if (f->ncollected >= f->capacity) { + f->capacity = (f->ncollected + 1) * 2; // 2, 6, 14, 30, 62, ... (2**i - 2) + f->collection = PyMem_Realloc(f->collection, (f->capacity) * sizeof(void *)); + if (!f->collection) { + return PyErr_NoMemory(); + } } f->collection[f->ncollected++] = v; f->iop = f->rule->alts[f->ialt]; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 8ad63eacd9a8f7..399c08973a82f7 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -53,6 +53,7 @@ typedef struct _frame { int iop; int cut; int ncollected; + int capacity; void **collection; int ival; void *vals[MAXVALS]; From 52f5a75829c1acec90d535b7362d2f76e7903301 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 29 May 2020 21:24:20 -0700 Subject: [PATCH 31/67] Speculate how to implement a.b+ --- Parser/pegen/vm.c | 1 + Parser/pegen/vmreadme.md | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index caf293fcd6b6c2..647686d4987597 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -128,6 +128,7 @@ run_vm(Parser *p, Rule rules[], int root) f = pop_frame(&stack, v); break; } + // Fallthrough! case OP_LOOP_COLLECT: v = make_asdl_seq(p, f->collection, f->ncollected); f = pop_frame(&stack, v); diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 5c541375eea680..7e77a98f555368 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -226,6 +226,31 @@ return_op: OP_RETURN Ideas ----- +### "Special" loops + +We still need opcodes for `a.b+` (e.g. `','.expr+` is a +comma-separated list of expressions). We could generate code like this: + +``` +# first alternative: + + +OP_LOOP_ITERATE + +# second alternative: + +OP_LOOP_COLLECT_NONEMPTY +``` + +- The `OP_LOOP_ITERATE` opcode would ignore the value collected for + the delimiter. + +- The `OP_LOOP_COLLECT_NONEMPTY` opcode would add the final collected + value to the collection, if one was collected. (Alternatively, we + could use a new opcode, e.g. `OP_LOOP_COLLECT_SPECIAL`.) + +### Lookaheads + We also need to extend the VM to support lookaheads. - Positive lookahead (`&a`) -- add a new opcode to save the mark in a @@ -235,7 +260,9 @@ We also need to extend the VM to support lookaheads. - Negative lookahead (`!a`) -- add another opcode that catches failure and turns it into success, restoring the saved mark. -- Left-recursion -- hmm, tricky... +### Left-recursion + +Hmm, tricky... Will think about this later. Lookahead opcodes ----------------- From 3b9323795a2475a9e0d1fa4315714b4a7fd2d6c2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 30 May 2020 10:22:14 -0700 Subject: [PATCH 32/67] Make memo rule types distinct from token types --- Parser/pegen/vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 647686d4987597..bddf8f40177bbc 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -51,7 +51,7 @@ pop_frame(Stack *stack, void *v) if (f->collection) { PyMem_Free(f->collection); } - _PyPegen_insert_memo(stack->p, f->mark, f->rule->type, v); + _PyPegen_insert_memo(stack->p, f->mark, f->rule->type + 1000, v); f = &stack->frames[stack->top - 1]; // New top of stack D(printf(" pop %s\n", f->rule->name)); return f; @@ -143,7 +143,7 @@ run_vm(Parser *p, Rule rules[], int root) case OP_RULE: oparg = f->rule->opcodes[f->iop++]; Rule *rule = &rules[oparg]; - int memo = _PyPegen_is_memoized(p, rule->type, &v); + int memo = _PyPegen_is_memoized(p, rule->type + 1000, &v); if (memo) { if (memo < 0) { return NULL; From 33522ae8d0837003323da9afb418db4cb3ede80b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 30 May 2020 10:23:12 -0700 Subject: [PATCH 33/67] Fix small issues in vmreadme.pm --- Parser/pegen/vmreadme.md | 57 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 7e77a98f555368..fe27349d355376 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -2,23 +2,24 @@ Pegen Virtual Machine ===================== The Pegen VM is an alternative for the recursive-descent Pegen parser. -The idea is that the grammar (including actions) is identical, but -execution does not use the C stack frame. The hope is that this might -be faster, but this is far from given. If it is clear that it will -*not* be faster, we should stop working on this project. +The grammar (including actions) is identical, but execution does not +use the C stack. We expect this to be faster, and initial +measurements seem to bear this out. But we need to keep diligent, and +if it ever becomes clear that it will *not* be faster, we should stop +working on this project. The runtime uses the same `Parser` structure as the recursive-descent Pegen parser, and the same helper functions (e.g., `_PyPegen_singleton_seq`). The VM uses a stack to hold state during parsing. The grammar is -represented by a few read-only tables. This design may be familiar -from that of CPython's "ceval" VM. The actions are represented by a -function containing a giant switch with one case for each action. +represented by a few read-only tables. The actions are represented by +a function containing a giant switch with one case per action. (An +optimization here could be to combine identical actions.) The grammar tables and the action function are meant to be generated by a parser generator similar to the current one. Because of the -actions, the generator needs to generate C code. +actions, it needs to generate C code. The primary VM state is a stack of `Frame` structures. Each frame represents a particular attempt to parse a rule at a given point in @@ -35,17 +36,20 @@ The main state in a frame is as follows: State related to loops is described below. -Note that `rule` and `mark` don't change after the frame is initialized. +Note that `rule` doesn't change after the frame is initialized. Also, +`mark` normally doesn't change, except for loop operations. -In addition each frame has a "value stack" where successfully -recognized tokens and rules are stored. This uses: +Each frame also has an array of values where successfully recognized +tokens and rules are stores. This uses: - `int ival` -- number of values stored so far -- `void *vals[]` -- values stored (the type is `Token *` or an AST node type) +- `void *vals[]` -- values stored (the type is `Token *` or an AST + node type; may be NULL) A `Rule` structure has the following fields: - `char *name` -- rule name, for debugging (e.g., `"start"`) +- `int type` -- rule type, used for memo lookup - `int alts[]` -- index into `opcodes` array for each alternative, terminated by `-1` - `int opcodes[]` -- array of opcodes and their arguments @@ -67,15 +71,16 @@ If an operation succeeds, the value is appended to the frame's values array (`vals`), and the VM proceeds to the next opcode. If an operation fails, the VM resets the input to the frame's mark, -and then proceeds to the next alternative of the frame's rule, if -there is one, and the frame's `cut` flag is not set. If the frame's -`cut` flag is set, or if its rule has no more alternatives, the frame -is popped off the frame stack and the VM proceeds with failure there. +and resets the value array. It then proceeds to the next alternative +of the frame's rule, if there is one and the frame's `cut` flag is not +set. If the frame's `cut` flag is set, or if its rule has no more +alternatives, the frame is popped off the frame stack and the VM +proceeds with failure there. Some operations manipulate other frame fields. -Calls into the support runtime can also produce *errors* -- when an -error is detected, the VM exits, immediately returning `NULL`. +Calls into the support runtime can produce *errors* -- when an error +is detected, the VM exits immediately, returning `NULL`. ### General operations @@ -90,12 +95,12 @@ The following opcodes take no argument. - `OP_STRING` -- call `_PyPegen_string_token()`; same as `OP_NAME`. -- `OP_CUT` -- set the frame's `cut` flag; succeeds without a value. +- `OP_CUT` -- set the frame's `cut` flag; succeed without a value. -- `OP_OPTIONAL` -- always succeeds, but modifies the *previous* +- `OP_OPTIONAL` -- succeed without a value; modifies the *previous* operation to treat a `NULL` result as a success. (See below.) -These operations are followed by a single integer argument. +The following operations are followed by a single integer argument. - `OP_TOKEN(type)` -- call `_PyPegen_expect_token()` with the `type` argument; processing is the same as for `OP_NAME`. @@ -163,8 +168,6 @@ The operations are defined as follows: - `OP_LOOP_COLLECT_NONEMPTY` -- like `OP_LOOP_COLLECT` but fails if no values are collected. -(TODO: additional operations to support `b.a+`.) - More about `OP_OPTIONAL` ------------------------ @@ -228,17 +231,17 @@ Ideas ### "Special" loops -We still need opcodes for `a.b+` (e.g. `','.expr+` is a +We still need opcodes for `b.a+` (e.g. `','.expr+` is a comma-separated list of expressions). We could generate code like this: ``` # first alternative: - + OP_LOOP_ITERATE # second alternative: - + OP_LOOP_COLLECT_NONEMPTY ``` @@ -289,4 +292,4 @@ if (f->rule->opcodes[f->iop] == OP_NEG_LOOKAHEAD) { If we initialize savemark to the same value as mark, we can avoid a `SAVE_MARK` opcode at the start of alternatives -- this is a common -pattern. +pattern. (OTOH, it costs an extra store for each frame.) From cbd45a59880de83c0f4374afc38a6fe21c796b51 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 11:04:31 -0700 Subject: [PATCH 34/67] Add generation of root rules (very coarssely) --- Tools/peg_generator/pegen/vm_generator.py | 50 ++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 00d769381cd229..9ae93c65ea2d54 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -29,7 +29,13 @@ from pegen.parser_generator import ParserGenerator -class CCallMakerVisitor(GrammarVisitor): +class RootRule(Rule): + def __init__(self, name: str, startrulename: str): + super().__init__(name, "void *", Rhs([]), None) + self.startrulename = startrulename + + +class VMCallMakerVisitor(GrammarVisitor): def __init__( self, parser_generator: ParserGenerator, ): @@ -49,14 +55,14 @@ def visit_Repeat1(self, node: Repeat1) -> None: self.cache[node] = name -class CParserGenerator(ParserGenerator, GrammarVisitor): +class VMParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, ): super().__init__(grammar, token.tok_name, sys.stdout) self.opcode_buffer: Optional[List[str]] = None - self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(self) + self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor(self) @contextlib.contextmanager def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: @@ -69,6 +75,7 @@ def add_opcode(self, opcode: str) -> None: self.opcode_buffer.append(opcode) def generate(self, filename: str) -> None: + self.add_root_rules() self.collect_todo() while self.todo: self.print("static Rule all_rules[] = {") @@ -78,6 +85,40 @@ def generate(self, filename: str) -> None: self.visit(rule) self.print("}") + def add_root_rules(self) -> None: + assert "root" not in self.todo + assert "root" not in self.rules + root = RootRule("root", "start") # TODO: determine start rules dynamically + # Odd way of updating todo/rules, to make the root rule appear first. + # TODO: Is this necessary? + self.todo = {"root": root} | self.todo + self.rules = {"root": root} | self.rules + + def visit_RootRule(self, node: RootRule) -> None: + # TODO: Refactor visit_Rule() so we can share code. + self.print(f'{{"{node.name}",') + self.print(f" R_{node.name.upper()},") + + opcodes_by_alt: Dict[int, List[str]] = {0: [], 1: []} + with self.set_opcode_buffer(opcodes_by_alt[0]): + self.add_opcode("OP_RULE") + self.add_opcode(f"R_{node.startrulename.upper()}") + self.add_opcode("OP_SUCCESS") + with self.set_opcode_buffer(opcodes_by_alt[1]): + self.add_opcode("OP_FAILURE") + + indexes = [0, len(opcodes_by_alt[0]), -1] + + self.print(f" {{{', '.join(map(str, indexes))}}},") + + self.print(" {") + with self.indent(): + for index, opcodes in opcodes_by_alt.items(): + self.print(", ".join(opcodes) + ",") + self.print(" },") + + self.print("},") + def visit_Rule(self, node: Rule) -> None: is_loop = node.is_loop() is_gather = node.is_gather() @@ -151,7 +192,6 @@ def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) - with self.indent(): for index, opcodes in opcodes_by_alt.items(): self.print(", ".join(opcodes) + ",") - self.print() self.print(" },") def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: @@ -166,7 +206,7 @@ def visit_Repeat0(self, node: Repeat0) -> None: def main() -> None: grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) - p = CParserGenerator(grammar) + p = VMParserGenerator(grammar) p.generate("") From 1a6531df9b5bcfa6a2f214ea0b23d99f02559066 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 14:11:10 -0700 Subject: [PATCH 35/67] Add enum for rule types (R_) --- Tools/peg_generator/pegen/vm_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 9ae93c65ea2d54..36920dd1e5b15c 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -77,6 +77,12 @@ def add_opcode(self, opcode: str) -> None: def generate(self, filename: str) -> None: self.add_root_rules() self.collect_todo() + self.print("enum {") + with self.indent(): + for rulename in self.todo: + self.print(f"R_{rulename.upper()},") + self.print("};") + self.print() while self.todo: self.print("static Rule all_rules[] = {") for rulename, rule in list(self.todo.items()): From 0226dd58af9d4640139d8131d3c3f83b51d5f127 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 14:37:49 -0700 Subject: [PATCH 36/67] Generate actions (primitively) --- Tools/peg_generator/pegen/vm_generator.py | 56 +++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 36920dd1e5b15c..40f1b95c993c39 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -28,6 +28,32 @@ ) from pegen.parser_generator import ParserGenerator +CALL_ACTION_HEAD = """ +// TODO: Fill this +static const int n_keyword_lists = 0; +static KeywordToken *reserved_keywords[] = { +}; + +static void * +call_action(Parser *p, Frame *f, int iaction) +{ + assert(p->mark > 0); + Token *t = p->tokens[f->mark]; + int _start_lineno = t->lineno; + int _start_col_offset = t->col_offset; + t = p->tokens[p->mark - 1]; + int _end_lineno = t->end_lineno; + int _end_col_offset = t->end_col_offset; + + switch (iaction) { +""" + +CALL_ACTION_TAIL = """\ + default: + assert(0); + } +} +""" class RootRule(Rule): def __init__(self, name: str, startrulename: str): @@ -77,19 +103,44 @@ def add_opcode(self, opcode: str) -> None: def generate(self, filename: str) -> None: self.add_root_rules() self.collect_todo() + self.gather_actions() + self.print("enum {") with self.indent(): for rulename in self.todo: self.print(f"R_{rulename.upper()},") self.print("};") self.print() + + self.print("enum {") + with self.indent(): + for actionname, action in self.actions.items(): + self.print(f"{actionname}, // {action}") + self.print("};") + self.print() + while self.todo: self.print("static Rule all_rules[] = {") for rulename, rule in list(self.todo.items()): del self.todo[rulename] with self.indent(): self.visit(rule) - self.print("}") + self.print("};") + + self.printblock(CALL_ACTION_HEAD) + with self.indent(): + for actionname, action in self.actions.items(): + self.print(f"case {actionname}:") + self.print(f" return {action};") + self.printblock(CALL_ACTION_TAIL) + + def gather_actions(self) -> None: + self.actions: Dict[str, str] = {} + for rulename, rule in self.todo.items(): + if not rule.is_loop(): + for index, alt in enumerate(rule.rhs.alts): + actionname = f"A_{rulename.upper()}_{index}" + self.actions[actionname] = alt.action def add_root_rules(self) -> None: assert "root" not in self.todo @@ -162,8 +213,6 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: self.add_opcode(tok_name) def handle_loop_rhs(self, node: Rhs, opcodes_by_alt: Dict[int, List[str]]) -> None: - with self.set_opcode_buffer(opcodes_by_alt[0]): - self.add_opcode("OP_LOOP_START") self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) with self.set_opcode_buffer(opcodes_by_alt[len(node.alts)]): self.add_opcode("OP_LOOP_COLLECT") @@ -178,6 +227,7 @@ def handle_default_rhs( for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[index]): self.visit(alt, is_loop=False, is_gather=False) + assert not (alt.action and is_loop) # A loop rule can't have actions if alt.action: self.add_opcode("OP_RETURN") self.add_opcode(f"A_{self.current_rule.name.upper()}_{index}") From c64ff2fa8c818ea79ba885abb4d936ef9073676c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 1 Jun 2020 01:28:41 +0300 Subject: [PATCH 37/67] Implement code generation for keywords --- Tools/peg_generator/data/simple.gram | 6 ++- Tools/peg_generator/pegen/vm_generator.py | 61 +++++++++++++++++++---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 4d0415581f6728..f512d3ec8aaa78 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -1,5 +1,9 @@ start: stmt* ENDMARKER {_PyPegen_make_module(p, f->vals[0])} -stmt: expr NEWLINE {_Py_Expr(f->vals[0], EXTRA)} +stmt: + | if_stmt NEWLINE { f->vals[0] } + | expr NEWLINE {_Py_Expr(f->vals[0], EXTRA)} +if_stmt: + | 'if' NAME ':' expr { _Py_If(f->vals[1], f->vals[2], NULL, EXTRA) } expr: | term '+' expr {_Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA)} | term {f->vals[0]} diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 40f1b95c993c39..20b5ce606111ef 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -1,9 +1,11 @@ +import ast import contextlib import sys +import re import token from collections import defaultdict from itertools import accumulate -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Optional, Tuple from pegen import grammar from pegen.build import build_parser @@ -29,11 +31,6 @@ from pegen.parser_generator import ParserGenerator CALL_ACTION_HEAD = """ -// TODO: Fill this -static const int n_keyword_lists = 0; -static KeywordToken *reserved_keywords[] = { -}; - static void * call_action(Parser *p, Frame *f, int iaction) { @@ -67,6 +64,18 @@ def __init__( ): self.gen = parser_generator self.cache: Dict[Any, Any] = {} + self.keyword_cache: Dict[str, int] = {} + + def keyword_helper(self, keyword: str) -> None: + if keyword not in self.keyword_cache: + self.keyword_cache[keyword] = self.gen.keyword_type() + return self.keyword_cache[keyword] + + def visit_StringLeaf(self, node: StringLeaf) -> None: + val = ast.literal_eval(node.value) + if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword + return self.keyword_helper(val) + return token.EXACT_TOKEN_TYPES[val] def visit_Repeat0(self, node: Repeat0) -> None: if node in self.cache: @@ -104,6 +113,7 @@ def generate(self, filename: str) -> None: self.add_root_rules() self.collect_todo() self.gather_actions() + self._setup_keywords() self.print("enum {") with self.indent(): @@ -134,6 +144,39 @@ def generate(self, filename: str) -> None: self.print(f" return {action};") self.printblock(CALL_ACTION_TAIL) + def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: + groups: Dict[int, List[Tuple[str, int]]] = {} + for keyword_str, keyword_type in self.callmakervisitor.keyword_cache.items(): + length = len(keyword_str) + if length in groups: + groups[length].append((keyword_str, keyword_type)) + else: + groups[length] = [(keyword_str, keyword_type)] + return groups + + def _setup_keywords(self) -> None: + keyword_cache = self.callmakervisitor.keyword_cache + n_keyword_lists = ( + len(max(keyword_cache.keys(), key=len)) + 1 if len(keyword_cache) > 0 else 0 + ) + self.print(f"static const int n_keyword_lists = {n_keyword_lists};") + groups = self._group_keywords_by_length() + self.print("static KeywordToken *reserved_keywords[] = {") + with self.indent(): + num_groups = max(groups) + 1 if groups else 1 + for keywords_length in range(num_groups): + if keywords_length not in groups.keys(): + self.print("NULL,") + else: + self.print("(KeywordToken[]) {") + with self.indent(): + for keyword_str, keyword_type in groups[keywords_length]: + self.print(f'{{"{keyword_str}", {keyword_type}}},') + self.print("{NULL, -1},") + self.print("},") + self.print("};") + self.print() + def gather_actions(self) -> None: self.actions: Dict[str, str] = {} for rulename, rule in self.todo.items(): @@ -206,11 +249,9 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: self.add_opcode(self._get_rule_opcode(name)) def visit_StringLeaf(self, node: StringLeaf) -> None: + token_type = self.callmakervisitor.visit(node) self.add_opcode("OP_TOKEN") - tok_str = node.value.replace("'", "") - tok_num = token.EXACT_TOKEN_TYPES[tok_str] - tok_name = token.tok_name[tok_num] - self.add_opcode(tok_name) + self.add_opcode(str(token_type)) def handle_loop_rhs(self, node: Rhs, opcodes_by_alt: Dict[int, List[str]]) -> None: self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) From e2c4a363001b40d4f73220d2c15046cba3c6da3b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 1 Jun 2020 01:59:28 +0300 Subject: [PATCH 38/67] Refactor add_opcode to optionally accept a second argument oparg --- Tools/peg_generator/pegen/vm_generator.py | 24 ++++++++++------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 20b5ce606111ef..8d2299ffe32297 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -5,7 +5,7 @@ import token from collections import defaultdict from itertools import accumulate -from typing import Any, Dict, Iterator, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from pegen import grammar from pegen.build import build_parser @@ -105,9 +105,11 @@ def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: yield self.opcode_buffer = None - def add_opcode(self, opcode: str) -> None: + def add_opcode(self, opcode: str, oparg: Optional[Union[int, str]] = None) -> None: assert self.opcode_buffer is not None self.opcode_buffer.append(opcode) + if oparg is not None: + self.opcode_buffer.append(str(oparg)) def generate(self, filename: str) -> None: self.add_root_rules() @@ -201,8 +203,7 @@ def visit_RootRule(self, node: RootRule) -> None: opcodes_by_alt: Dict[int, List[str]] = {0: [], 1: []} with self.set_opcode_buffer(opcodes_by_alt[0]): - self.add_opcode("OP_RULE") - self.add_opcode(f"R_{node.startrulename.upper()}") + self.add_opcode("OP_RULE", f"R_{node.startrulename.upper()}") self.add_opcode("OP_SUCCESS") with self.set_opcode_buffer(opcodes_by_alt[1]): self.add_opcode("OP_FAILURE") @@ -242,16 +243,13 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: if name in ("NAME", "NUMBER", "STRING"): self.add_opcode(f"OP_{name}") elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): - self.add_opcode("OP_TOKEN") - self.add_opcode(name) + self.add_opcode("OP_TOKEN", name) else: - self.add_opcode("OP_RULE") - self.add_opcode(self._get_rule_opcode(name)) + self.add_opcode("OP_RULE", self._get_rule_opcode(name)) def visit_StringLeaf(self, node: StringLeaf) -> None: token_type = self.callmakervisitor.visit(node) - self.add_opcode("OP_TOKEN") - self.add_opcode(str(token_type)) + self.add_opcode("OP_TOKEN", token_type) def handle_loop_rhs(self, node: Rhs, opcodes_by_alt: Dict[int, List[str]]) -> None: self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) @@ -270,8 +268,7 @@ def handle_default_rhs( self.visit(alt, is_loop=False, is_gather=False) assert not (alt.action and is_loop) # A loop rule can't have actions if alt.action: - self.add_opcode("OP_RETURN") - self.add_opcode(f"A_{self.current_rule.name.upper()}_{index}") + self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") if is_loop: self.add_opcode("OP_LOOP_ITERATE") @@ -297,8 +294,7 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: def visit_Repeat0(self, node: Repeat0) -> None: name = self.callmakervisitor.visit(node) - self.add_opcode("OP_RULE") - self.add_opcode(self._get_rule_opcode(name)) + self.add_opcode("OP_RULE", self._get_rule_opcode(name)) def main() -> None: From 21d8b83d159206f0f8ab6e4c030ccd6d03e2a882 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 15:40:37 -0700 Subject: [PATCH 39/67] Translate item names in actions; use the generated vmparse.h! --- Parser/pegen/vm.c | 2 +- Parser/pegen/vmparse.h | 179 +++++++++------------- Tools/peg_generator/data/simple.gram | 22 +-- Tools/peg_generator/pegen/vm_generator.py | 70 +++++++-- 4 files changed, 142 insertions(+), 131 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index bddf8f40177bbc..95c07e469342f6 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -210,5 +210,5 @@ _PyPegen_vmparser(Parser *p) p->keywords = reserved_keywords; p->n_keyword_lists = n_keyword_lists; - return run_vm(p, all_rules, 0); + return run_vm(p, all_rules, R_ROOT); } diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 31b3c6ea82ca36..9aad7e3cf1cb36 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -1,169 +1,134 @@ -/* Toy grammar - -NOTE: [expr] is right-recursive. - -start: stmt* ENDMARKER -stmt: expr NEWLINE -expr: term '+' expr | term -term: factor '*' term | factor -factor: '(' expr ')' | NUMBER | NAME - -*/ - -static const int n_keyword_lists = 0; +static const int n_keyword_lists = 3; static KeywordToken *reserved_keywords[] = { + NULL, + NULL, + (KeywordToken[]) { + {"if", 500}, + {NULL, -1}, + }, }; enum { - R_ROOT, - R_START, - R_STMTS, - R_STMT, - R_EXPR, - R_TERM, - R_FACTOR, + R_START, + R_STMT, + R_IF_STMT, + R_EXPR, + R_TERM, + R_FACTOR, + R_ROOT, + R__LOOP0_1, }; enum { - A_START_0, - A_STMT_0, - A_EXPR_0, - A_EXPR_1, - A_TERM_0, - A_TERM_1, - A_FACTOR_0, - A_FACTOR_1, - A_FACTOR_2, + A_START_0, + A_STMT_0, + A_STMT_1, + A_IF_STMT_0, + A_EXPR_0, + A_EXPR_1, + A_TERM_0, + A_TERM_1, + A_FACTOR_0, + A_FACTOR_1, + A_FACTOR_2, }; static Rule all_rules[] = { - - {"root", - R_ROOT, - {0, 3, -1}, - { - OP_RULE, R_START, - OP_SUCCESS, - - OP_FAILURE, - }, - }, - {"start", R_START, - {0, 6, -1}, + {0, -1}, { - OP_RULE, R_STMTS, - OP_TOKEN, ENDMARKER, - OP_RETURN, A_START_0, - - OP_FAILURE, + OP_RULE, R__LOOP0_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_START_0, }, }, - - {"stmt*", - R_STMTS, - {0, 3, -1}, + {"stmt", + R_STMT, + {0, 6, -1}, { - OP_RULE, R_STMT, - OP_LOOP_ITERATE, - - OP_LOOP_COLLECT, + OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, + OP_RULE, R_IF_STMT, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_1, }, }, - - {"stmt", - R_STMT, + {"if_stmt", + R_IF_STMT, {0, -1}, { - OP_RULE, R_EXPR, - OP_TOKEN, NEWLINE, - OP_RETURN, A_STMT_0, + OP_TOKEN, 500, OP_NAME, OP_TOKEN, 11, OP_RULE, R_EXPR, OP_RETURN, A_IF_STMT_0, }, }, - {"expr", R_EXPR, {0, 8, -1}, { - OP_RULE, R_TERM, - OP_TOKEN, PLUS, - OP_RULE, R_EXPR, - OP_RETURN, A_EXPR_0, - - OP_RULE, R_TERM, - OP_RETURN, A_EXPR_1, + OP_RULE, R_TERM, OP_TOKEN, 14, OP_RULE, R_EXPR, OP_RETURN, A_EXPR_0, + OP_RULE, R_TERM, OP_RETURN, A_EXPR_1, }, }, - {"term", R_TERM, {0, 8, -1}, { - OP_RULE, R_FACTOR, - OP_TOKEN, STAR, - OP_RULE, R_TERM, - OP_RETURN, A_TERM_0, - - OP_RULE, R_FACTOR, - OP_RETURN, A_TERM_1, + OP_RULE, R_FACTOR, OP_TOKEN, 16, OP_RULE, R_TERM, OP_RETURN, A_TERM_0, + OP_RULE, R_FACTOR, OP_RETURN, A_TERM_1, }, }, - {"factor", R_FACTOR, {0, 8, 11, -1}, { - OP_TOKEN, LPAR, - OP_RULE, R_EXPR, - OP_TOKEN, RPAR, - OP_RETURN, A_FACTOR_0, - - OP_NUMBER, - OP_RETURN, A_FACTOR_1, - - OP_NAME, - OP_RETURN, A_FACTOR_2, + OP_TOKEN, 7, OP_RULE, R_EXPR, OP_TOKEN, 8, OP_RETURN, A_FACTOR_0, + OP_NUMBER, OP_RETURN, A_FACTOR_1, + OP_NAME, OP_RETURN, A_FACTOR_2, + }, + }, + {"root", + R_ROOT, + {0, 3, -1}, + { + OP_RULE, R_START, OP_SUCCESS, + OP_FAILURE, + }, + }, + {"_loop0_1", + R__LOOP0_1, + {0, 3, -1}, + { + OP_RULE, R_STMT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, }, }, - }; static void * call_action(Parser *p, Frame *f, int iaction) { + assert(p->mark > 0); Token *t = p->tokens[f->mark]; int _start_lineno = t->lineno; int _start_col_offset = t->col_offset; - if (p->mark > 0) { - t = p->tokens[p->mark - 1]; - } - else { - // Debug anomaly rather than crashing - printf("p->mark=%d, p->fill=%d, f->mark=%d\n", p->mark, p->fill, f->mark); - } + t = p->tokens[p->mark - 1]; int _end_lineno = t->end_lineno; int _end_col_offset = t->end_col_offset; switch (iaction) { case A_START_0: - return _PyPegen_make_module(p, f->vals[0]); + return _PyPegen_make_module ( p , f->vals[0] ); case A_STMT_0: - return _Py_Expr(f->vals[0], EXTRA); - case A_EXPR_0: - return _Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA); + return _Py_Expr ( f->vals[0] , EXTRA ); + case A_STMT_1: case A_EXPR_1: - return f->vals[0]; - case A_TERM_0: - return f->vals[0]; case A_TERM_1: - return f->vals[0]; - case A_FACTOR_0: - return f->vals[1]; case A_FACTOR_1: - return f->vals[0]; case A_FACTOR_2: return f->vals[0]; + case A_IF_STMT_0: + return _Py_If ( f->vals[1] , f->vals[3] , NULL , EXTRA ); + case A_EXPR_0: + return _Py_BinOp ( f->vals[0] , Add , f->vals[2] , EXTRA ); + case A_TERM_0: + return _Py_BinOp ( f->vals[0] , Mult , f->vals[2] , EXTRA ); + case A_FACTOR_0: + return f->vals[1]; default: assert(0); } diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index f512d3ec8aaa78..a345e8e82061a4 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -1,16 +1,16 @@ -start: stmt* ENDMARKER {_PyPegen_make_module(p, f->vals[0])} +start: a=stmt* ENDMARKER { _PyPegen_make_module(p, a) } stmt: - | if_stmt NEWLINE { f->vals[0] } - | expr NEWLINE {_Py_Expr(f->vals[0], EXTRA)} + | a=expr NEWLINE { _Py_Expr(a, EXTRA) } + | a=if_stmt NEWLINE { a } if_stmt: - | 'if' NAME ':' expr { _Py_If(f->vals[1], f->vals[2], NULL, EXTRA) } + | 'if' a=NAME ':' b=expr { _Py_If(a, b, NULL, EXTRA) } expr: - | term '+' expr {_Py_BinOp(f->vals[0], Add, f->vals[2], EXTRA)} - | term {f->vals[0]} + | a=term '+' b=expr { _Py_BinOp(a, Add, b, EXTRA) } + | term term: - | factor '*' term {f->vals[0]} - | factor {f->vals[0]} + | a=factor '*' b=term { _Py_BinOp(a, Mult, b, EXTRA) } + | factor factor: - | '(' expr ')' {f->vals[1]} - | NUMBER {f->vals[0]} - | NAME {f->vals[0]} \ No newline at end of file + | '(' a=expr ')' { a } + | NUMBER + | NAME diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 8d2299ffe32297..47d11d064ae776 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -3,6 +3,7 @@ import sys import re import token +import tokenize from collections import defaultdict from itertools import accumulate from typing import Any, Dict, Iterator, List, Optional, Tuple, Union @@ -15,6 +16,7 @@ Gather, GrammarVisitor, Group, + Leaf, Lookahead, NamedItem, NameLeaf, @@ -22,6 +24,7 @@ Opt, Plain, PositiveLookahead, + Repeat, Repeat0, Repeat1, Rhs, @@ -75,7 +78,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: val = ast.literal_eval(node.value) if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword return self.keyword_helper(val) - return token.EXACT_TOKEN_TYPES[val] + return token.EXACT_TOKEN_TYPES[val] # type: ignore def visit_Repeat0(self, node: Repeat0) -> None: if node in self.cache: @@ -127,7 +130,7 @@ def generate(self, filename: str) -> None: self.print("enum {") with self.indent(): for actionname, action in self.actions.items(): - self.print(f"{actionname}, // {action}") + self.print(f"{actionname},") self.print("};") self.print() @@ -141,9 +144,7 @@ def generate(self, filename: str) -> None: self.printblock(CALL_ACTION_HEAD) with self.indent(): - for actionname, action in self.actions.items(): - self.print(f"case {actionname}:") - self.print(f" return {action};") + self.print_action_cases() self.printblock(CALL_ACTION_TAIL) def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: @@ -179,22 +180,67 @@ def _setup_keywords(self) -> None: self.print("};") self.print() + def print_action_cases(self) -> None: + unique_actions: Dict[str, str] = defaultdict(list) + for actionname, action in self.actions.items(): + unique_actions[action].append(actionname) + for action, actionnames in unique_actions.items(): + for actionname in actionnames: + self.print(f"case {actionname}:") + self.print(f" return {action};") + def gather_actions(self) -> None: self.actions: Dict[str, str] = {} for rulename, rule in self.todo.items(): if not rule.is_loop(): for index, alt in enumerate(rule.rhs.alts): actionname = f"A_{rulename.upper()}_{index}" - self.actions[actionname] = alt.action + self.actions[actionname] = self.translate_action(alt) + + def translate_action(self, alt: Alt) -> str: + # Given an alternative like this: + # | a=NAME '=' b=expr { foo(p, a, b) } + # return a string like this: + # "foo(p, f->vals[0], f->vals[2])" + # As a special case, if there's no action, return f->vals[0]. + name_to_index, index = self.map_alt_names_to_vals_index(alt) + if not alt.action: + assert index == 1 + return "f->vals[0]" + # Sadly, the action is given as a string, so tokenize it back. + # We must not substitute item names when preceded by '.' or '->'. + # Note that Python tokenizes '->' as two separate tokens. + res = [] + prevs = "" + for stuff in tokenize.generate_tokens(iter([alt.action]).__next__): + _, s, _, _, _ = stuff + if prevs not in (".", "->"): + i = name_to_index.get(s) + if i is not None: + s = f"f->vals[{i}]" + if prevs == "-" and s == ">": + prevs = "->" + else: + prevs = s + res.append(s) + return " ".join(res).strip() + + def map_alt_names_to_vals_index(self, alt: Alt) -> Tuple[Dict[str, int], int]: + index = 0 + map: Dict[str, int] = {} + for nameditem in alt.items: + if nameditem.name: + map[nameditem.name] = index + if isinstance(nameditem.item, (Leaf, Group, Opt, Repeat)): + index += 1 + return map, index def add_root_rules(self) -> None: assert "root" not in self.todo assert "root" not in self.rules root = RootRule("root", "start") # TODO: determine start rules dynamically - # Odd way of updating todo/rules, to make the root rule appear first. - # TODO: Is this necessary? - self.todo = {"root": root} | self.todo - self.rules = {"root": root} | self.rules + self.todo["root"] = root + self.rules["root"] = root def visit_RootRule(self, node: RootRule) -> None: # TODO: Refactor visit_Rule() so we can share code. @@ -267,10 +313,10 @@ def handle_default_rhs( with self.set_opcode_buffer(opcodes_by_alt[index]): self.visit(alt, is_loop=False, is_gather=False) assert not (alt.action and is_loop) # A loop rule can't have actions - if alt.action: - self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") if is_loop: self.add_opcode("OP_LOOP_ITERATE") + else: + self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) From 6fe4f0e475c499a27789ce7a6bec6b1fc0798c24 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 16:18:22 -0700 Subject: [PATCH 40/67] Fix mypy (in vm_generator) --- Tools/peg_generator/pegen/vm_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 47d11d064ae776..2826b874caad5a 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -69,16 +69,16 @@ def __init__( self.cache: Dict[Any, Any] = {} self.keyword_cache: Dict[str, int] = {} - def keyword_helper(self, keyword: str) -> None: + def keyword_helper(self, keyword: str) -> int: if keyword not in self.keyword_cache: self.keyword_cache[keyword] = self.gen.keyword_type() return self.keyword_cache[keyword] - def visit_StringLeaf(self, node: StringLeaf) -> None: + def visit_StringLeaf(self, node: StringLeaf) -> int: val = ast.literal_eval(node.value) if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword return self.keyword_helper(val) - return token.EXACT_TOKEN_TYPES[val] # type: ignore + return token.EXACT_TOKEN_TYPES[val] # type: ignore [attr-defined] def visit_Repeat0(self, node: Repeat0) -> None: if node in self.cache: @@ -181,7 +181,7 @@ def _setup_keywords(self) -> None: self.print() def print_action_cases(self) -> None: - unique_actions: Dict[str, str] = defaultdict(list) + unique_actions: Dict[str, List[str]] = defaultdict(list) for actionname, action in self.actions.items(): unique_actions[action].append(actionname) for action, actionnames in unique_actions.items(): From 6205002a2c6cd32adce3efbb857992b4a3cfd7c2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 16:24:24 -0700 Subject: [PATCH 41/67] Avoid name conflict for 'f' --- Parser/pegen/vmparse.h | 30 +++++++++++------------ Tools/peg_generator/pegen/vm_generator.py | 26 ++++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 9aad7e3cf1cb36..bb164d3a53e512 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -100,35 +100,35 @@ static Rule all_rules[] = { }; static void * -call_action(Parser *p, Frame *f, int iaction) +call_action(Parser *p, Frame *_f, int _iaction) { assert(p->mark > 0); - Token *t = p->tokens[f->mark]; - int _start_lineno = t->lineno; - int _start_col_offset = t->col_offset; - t = p->tokens[p->mark - 1]; - int _end_lineno = t->end_lineno; - int _end_col_offset = t->end_col_offset; + Token *_t = p->tokens[_f->mark]; + int _start_lineno = _t->lineno; + int _start_col_offset = _t->col_offset; + _t = p->tokens[p->mark - 1]; + int _end_lineno = _t->end_lineno; + int _end_col_offset = _t->end_col_offset; - switch (iaction) { + switch (_iaction) { case A_START_0: - return _PyPegen_make_module ( p , f->vals[0] ); + return _PyPegen_make_module ( p , _f->vals[0] ); case A_STMT_0: - return _Py_Expr ( f->vals[0] , EXTRA ); + return _Py_Expr ( _f->vals[0] , EXTRA ); case A_STMT_1: case A_EXPR_1: case A_TERM_1: case A_FACTOR_1: case A_FACTOR_2: - return f->vals[0]; + return _f->vals[0]; case A_IF_STMT_0: - return _Py_If ( f->vals[1] , f->vals[3] , NULL , EXTRA ); + return _Py_If ( _f->vals[1] , _f->vals[3] , NULL , EXTRA ); case A_EXPR_0: - return _Py_BinOp ( f->vals[0] , Add , f->vals[2] , EXTRA ); + return _Py_BinOp ( _f->vals[0] , Add , _f->vals[2] , EXTRA ); case A_TERM_0: - return _Py_BinOp ( f->vals[0] , Mult , f->vals[2] , EXTRA ); + return _Py_BinOp ( _f->vals[0] , Mult , _f->vals[2] , EXTRA ); case A_FACTOR_0: - return f->vals[1]; + return _f->vals[1]; default: assert(0); } diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 2826b874caad5a..72e550bb83d79c 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -35,17 +35,17 @@ CALL_ACTION_HEAD = """ static void * -call_action(Parser *p, Frame *f, int iaction) +call_action(Parser *p, Frame *_f, int _iaction) { assert(p->mark > 0); - Token *t = p->tokens[f->mark]; - int _start_lineno = t->lineno; - int _start_col_offset = t->col_offset; - t = p->tokens[p->mark - 1]; - int _end_lineno = t->end_lineno; - int _end_col_offset = t->end_col_offset; - - switch (iaction) { + Token *_t = p->tokens[_f->mark]; + int _start_lineno = _t->lineno; + int _start_col_offset = _t->col_offset; + _t = p->tokens[p->mark - 1]; + int _end_lineno = _t->end_lineno; + int _end_col_offset = _t->end_col_offset; + + switch (_iaction) { """ CALL_ACTION_TAIL = """\ @@ -201,12 +201,12 @@ def translate_action(self, alt: Alt) -> str: # Given an alternative like this: # | a=NAME '=' b=expr { foo(p, a, b) } # return a string like this: - # "foo(p, f->vals[0], f->vals[2])" - # As a special case, if there's no action, return f->vals[0]. + # "foo(p, _f->vals[0], _f->vals[2])" + # As a special case, if there's no action, return _f->vals[0]. name_to_index, index = self.map_alt_names_to_vals_index(alt) if not alt.action: assert index == 1 - return "f->vals[0]" + return "_f->vals[0]" # Sadly, the action is given as a string, so tokenize it back. # We must not substitute item names when preceded by '.' or '->'. # Note that Python tokenizes '->' as two separate tokens. @@ -217,7 +217,7 @@ def translate_action(self, alt: Alt) -> str: if prevs not in (".", "->"): i = name_to_index.get(s) if i is not None: - s = f"f->vals[{i}]" + s = f"_f->vals[{i}]" if prevs == "-" and s == ">": prevs = "->" else: From f72c7b6467e16784352bc81a9ca9d7f59825d4eb Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 17:02:59 -0700 Subject: [PATCH 42/67] Generate code for repeat1 loops --- Parser/pegen/vmparse.h | 10 ++++----- Tools/peg_generator/data/simple.gram | 2 +- Tools/peg_generator/pegen/vm_generator.py | 27 +++++++++++++++++------ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index bb164d3a53e512..d61b25ba6c23ee 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -16,7 +16,7 @@ enum { R_TERM, R_FACTOR, R_ROOT, - R__LOOP0_1, + R__LOOP1_1, }; enum { @@ -38,7 +38,7 @@ static Rule all_rules[] = { R_START, {0, -1}, { - OP_RULE, R__LOOP0_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_START_0, + OP_RULE, R__LOOP1_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_START_0, }, }, {"stmt", @@ -89,12 +89,12 @@ static Rule all_rules[] = { OP_FAILURE, }, }, - {"_loop0_1", - R__LOOP0_1, + {"_loop1_1", + R__LOOP1_1, {0, 3, -1}, { OP_RULE, R_STMT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + OP_LOOP_COLLECT_NONEMPTY, }, }, }; diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index a345e8e82061a4..9a27c6782d2cb4 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -1,4 +1,4 @@ -start: a=stmt* ENDMARKER { _PyPegen_make_module(p, a) } +start: a=stmt+ ENDMARKER { _PyPegen_make_module(p, a) } stmt: | a=expr NEWLINE { _Py_Expr(a, EXTRA) } | a=if_stmt NEWLINE { a } diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 72e550bb83d79c..b685d09409de12 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -268,6 +268,7 @@ def visit_RootRule(self, node: RootRule) -> None: def visit_Rule(self, node: Rule) -> None: is_loop = node.is_loop() + is_loop1 = node.name.startswith("_loop1") is_gather = node.is_gather() rhs = node.flatten() if node.left_recursive: @@ -275,7 +276,7 @@ def visit_Rule(self, node: Rule) -> None: self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") self.current_rule = node # TODO: make this a context manager - self.visit(rhs, is_loop=is_loop, is_gather=is_gather) + self.visit(rhs, is_loop=is_loop, is_loop1=is_loop1, is_gather=is_gather) self.print("},") def visit_NamedItem(self, node: NamedItem) -> None: @@ -297,31 +298,39 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: token_type = self.callmakervisitor.visit(node) self.add_opcode("OP_TOKEN", token_type) - def handle_loop_rhs(self, node: Rhs, opcodes_by_alt: Dict[int, List[str]]) -> None: + def handle_loop_rhs( + self, node: Rhs, opcodes_by_alt: Dict[int, List[str]], collect_opcode: str, + ) -> None: self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) with self.set_opcode_buffer(opcodes_by_alt[len(node.alts)]): - self.add_opcode("OP_LOOP_COLLECT") + self.add_opcode(collect_opcode) def handle_default_rhs( self, node: Rhs, opcodes_by_alt: Dict[int, List[str]], is_loop: bool = False, + is_loop1: bool = False, is_gather: bool = False, ) -> None: for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[index]): - self.visit(alt, is_loop=False, is_gather=False) + self.visit(alt, is_loop=False, is_loop1=False, is_gather=False) assert not (alt.action and is_loop) # A loop rule can't have actions if is_loop: self.add_opcode("OP_LOOP_ITERATE") else: self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") - def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: + def visit_Rhs( + self, node: Rhs, is_loop: bool = False, is_loop1: bool = False, is_gather: bool = False, + ) -> None: opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) if is_loop: - self.handle_loop_rhs(node, opcodes_by_alt) + opcode = "OP_LOOP_COLLECT" + if is_loop1: + opcode += "_NONEMPTY" + self.handle_loop_rhs(node, opcodes_by_alt, opcode) else: self.handle_default_rhs(node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather) *indexes, _ = accumulate(map(len, opcodes_by_alt.values())) @@ -334,7 +343,7 @@ def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) - self.print(", ".join(opcodes) + ",") self.print(" },") - def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: + def visit_Alt(self, node: Alt, is_loop: bool, is_loop1: bool, is_gather: bool) -> None: for item in node.items: self.visit(item) @@ -342,6 +351,10 @@ def visit_Repeat0(self, node: Repeat0) -> None: name = self.callmakervisitor.visit(node) self.add_opcode("OP_RULE", self._get_rule_opcode(name)) + def visit_Repeat1(self, node: Repeat0) -> None: + name = self.callmakervisitor.visit(node) + self.add_opcode("OP_RULE", self._get_rule_opcode(name)) + def main() -> None: grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) From fc3d4c40d0eaf81e9268ec3060e7b305eef20c30 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 17:15:39 -0700 Subject: [PATCH 43/67] Implement delimited loops (b.a+) --- Parser/pegen/vm.c | 16 +++++++++++++++- Parser/pegen/vm.h | 2 ++ Parser/pegen/vmreadme.md | 31 +++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 95c07e469342f6..3a98028ff22ef8 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -108,7 +108,7 @@ run_vm(Parser *p, Rule rules[], int root) goto top; case OP_LOOP_ITERATE: f->mark = p->mark; - assert(f->ival == 1); + assert(f->ival == 1 || f->ival == 2); v = f->vals[0]; assert(v); if (f->ncollected >= f->capacity) { @@ -122,8 +122,21 @@ run_vm(Parser *p, Rule rules[], int root) f->iop = f->rule->alts[f->ialt]; f->ival = 0; goto top; + case OP_LOOP_COLLECT_DELIMITED: + /* Collect one item */ + assert(f->ival == 1); + if (f->ncollected >= f->capacity) { + f->capacity = f->ncollected + 1; // We know there won't be any more + f->collection = PyMem_Realloc(f->collection, (f->capacity) * sizeof(void *)); + if (!f->collection) { + return PyErr_NoMemory(); + } + } + f->collection[f->ncollected++] = v; + // Fallthrough! case OP_LOOP_COLLECT_NONEMPTY: if (!f->ncollected) { + printf("Nothing collected for %s\n", f->rule->name); v = NULL; f = pop_frame(&stack, v); break; @@ -197,6 +210,7 @@ run_vm(Parser *p, Rule rules[], int root) f->iop = f->rule->alts[++f->ialt]; if (f->iop == -1) goto pop; + f->ival = 0; goto top; pop: diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 399c08973a82f7..f4e8c33102bc55 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -8,6 +8,7 @@ typedef enum _opcodes { OP_LOOP_ITERATE, OP_LOOP_COLLECT, OP_LOOP_COLLECT_NONEMPTY, + OP_LOOP_COLLECT_DELIMITED, OP_SUCCESS, OP_FAILURE, // The rest have an argument @@ -26,6 +27,7 @@ static char *opcode_names[] = { "OP_LOOP_ITERATE", "OP_LOOP_COLLECT", "OP_LOOP_COLLECT_NONEMPTY", + "OP_LOOP_COLLECT_DELIMITED", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index fe27349d355376..8b99f8be3273d6 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -144,7 +144,7 @@ the following structure: OP_LOOP_ITERATE # Second alternative: -Either OP_LOOP_COLLECT or OP_LOOP_COLLECT_NONEMPTY + ``` The values being collected are stored in a `malloc`-ed array named @@ -168,6 +168,26 @@ The operations are defined as follows: - `OP_LOOP_COLLECT_NONEMPTY` -- like `OP_LOOP_COLLECT` but fails if no values are collected. +For a "delimited" loop, written in the metagrammar as `b.a+` (one or +more `a` items separated by the delimiter `b`), the format is +slightly different: + +``` +# First alternative: + + +OP_LOOP_ITERATE + +# Second alternative: + +OP_LOOP_COLLECT_DELIMITED +``` + +The new operation is: + +- `OP_LOOP_COLLECT_DELIMITED` -- Add the first value from the values array + to the collection and then do everything that `OP_LOOP_COLLECT does. + More about `OP_OPTIONAL` ------------------------ @@ -204,6 +224,8 @@ Grammar for lists of operations This shows the constraints on how operations can be used together. ``` +rule: root_rule | normal_rule | loop_rule | delimited_rule + root_rule: success_alt failure_alt success_alt: regular_op OP_SUCCESS @@ -216,6 +238,11 @@ loop_rule: loop_start_alt loop_collect_alt loop_start_alt: regular_op OP_LOOP_ITERATE loop_collect_alt: OP_LOOP_COLLECT | OP_LOOP_COLLECT_NONEMPTY +delimited_rule: delimited_start_alt delimited_collect_alt + +delimited_start_alt: regular_op regular_op OP_LOOP_ITERATE +delimited_collect_alt: OP_LOOP_COLLECT_DELIMITED + alt: any_op+ return_op any_op: regular_op [OP_OPTIONAL] | special_op @@ -250,7 +277,7 @@ OP_LOOP_COLLECT_NONEMPTY - The `OP_LOOP_COLLECT_NONEMPTY` opcode would add the final collected value to the collection, if one was collected. (Alternatively, we - could use a new opcode, e.g. `OP_LOOP_COLLECT_SPECIAL`.) + could use a new opcode, e.g. `OP_LOOP_COLLECT_DELIMITED`.) ### Lookaheads From 6c5046897efd889ed02babd23bc7690bcda512a2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 31 May 2020 22:18:10 -0700 Subject: [PATCH 44/67] Generate code for delimited loop --- Parser/pegen/vmparse.h | 25 +++++++++++++++--- Tools/peg_generator/data/simple.gram | 1 + Tools/peg_generator/pegen/vm_generator.py | 32 +++++++++++++++++++---- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index d61b25ba6c23ee..27299a9acd0d5e 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -17,6 +17,7 @@ enum { R_FACTOR, R_ROOT, R__LOOP1_1, + R__GATHER_2, }; enum { @@ -31,6 +32,9 @@ enum { A_FACTOR_0, A_FACTOR_1, A_FACTOR_2, + A_FACTOR_3, + A__GATHER_2_0, + A__GATHER_2_1, }; static Rule all_rules[] = { @@ -74,11 +78,12 @@ static Rule all_rules[] = { }, {"factor", R_FACTOR, - {0, 8, 11, -1}, + {0, 8, 16, 19, -1}, { OP_TOKEN, 7, OP_RULE, R_EXPR, OP_TOKEN, 8, OP_RETURN, A_FACTOR_0, - OP_NUMBER, OP_RETURN, A_FACTOR_1, - OP_NAME, OP_RETURN, A_FACTOR_2, + OP_TOKEN, 9, OP_RULE, R__GATHER_2, OP_TOKEN, 10, OP_RETURN, A_FACTOR_1, + OP_NUMBER, OP_RETURN, A_FACTOR_2, + OP_NAME, OP_RETURN, A_FACTOR_3, }, }, {"root", @@ -97,6 +102,14 @@ static Rule all_rules[] = { OP_LOOP_COLLECT_NONEMPTY, }, }, + {"_gather_2", + R__GATHER_2, + {0, 5, -1}, + { + OP_RULE, R_EXPR, OP_TOKEN, 12, OP_LOOP_ITERATE, + OP_RULE, R_EXPR, OP_LOOP_COLLECT_DELIMITED, + }, + }, }; static void * @@ -118,8 +131,10 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_STMT_1: case A_EXPR_1: case A_TERM_1: - case A_FACTOR_1: case A_FACTOR_2: + case A_FACTOR_3: + case A__GATHER_2_0: + case A__GATHER_2_1: return _f->vals[0]; case A_IF_STMT_0: return _Py_If ( _f->vals[1] , _f->vals[3] , NULL , EXTRA ); @@ -129,6 +144,8 @@ call_action(Parser *p, Frame *_f, int _iaction) return _Py_BinOp ( _f->vals[0] , Mult , _f->vals[2] , EXTRA ); case A_FACTOR_0: return _f->vals[1]; + case A_FACTOR_1: + return _Py_List ( _f->vals[1] , Load , EXTRA ); default: assert(0); } diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 9a27c6782d2cb4..dec41289e64dfb 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -12,5 +12,6 @@ term: | factor factor: | '(' a=expr ')' { a } + | '[' a=','.expr+ ']' { _Py_List(a, Load, EXTRA) } | NUMBER | NAME diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index b685d09409de12..c0a6a52b809af2 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -92,6 +92,12 @@ def visit_Repeat1(self, node: Repeat1) -> None: name = self.gen.name_loop(node.node, True) self.cache[node] = name + def visit_Gather(self, node: Gather) -> None: + if node in self.cache: + return self.cache[node] + name = self.gen.name_gather(node) + self.cache[node] = name + class VMParserGenerator(ParserGenerator, GrammarVisitor): def __init__( @@ -114,6 +120,14 @@ def add_opcode(self, opcode: str, oparg: Optional[Union[int, str]] = None) -> No if oparg is not None: self.opcode_buffer.append(str(oparg)) + def name_gather(self, node: Gather) -> str: + self.counter += 1 + name = f"_gather_{self.counter}" + alt0 = Alt([NamedItem(None, node.node), NamedItem(None, node.separator)]) + alt1 = Alt([NamedItem(None, node.node)]) + self.todo[name] = Rule(name, None, Rhs([alt0, alt1])) + return name + def generate(self, filename: str) -> None: self.add_root_rules() self.collect_todo() @@ -205,7 +219,8 @@ def translate_action(self, alt: Alt) -> str: # As a special case, if there's no action, return _f->vals[0]. name_to_index, index = self.map_alt_names_to_vals_index(alt) if not alt.action: - assert index == 1 + # TODO: Restore the assert, but expect index == 2 in Gather + ##assert index == 1, "Alternative with >1 item must have an action" return "_f->vals[0]" # Sadly, the action is given as a string, so tokenize it back. # We must not substitute item names when preceded by '.' or '->'. @@ -316,9 +331,12 @@ def handle_default_rhs( for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[index]): self.visit(alt, is_loop=False, is_loop1=False, is_gather=False) - assert not (alt.action and is_loop) # A loop rule can't have actions - if is_loop: - self.add_opcode("OP_LOOP_ITERATE") + assert not (alt.action and (is_loop or is_gather)) # A loop rule can't have actions + if is_loop or is_gather: + if index == 0: + self.add_opcode("OP_LOOP_ITERATE") + else: + self.add_opcode("OP_LOOP_COLLECT_DELIMITED") else: self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") @@ -351,7 +369,11 @@ def visit_Repeat0(self, node: Repeat0) -> None: name = self.callmakervisitor.visit(node) self.add_opcode("OP_RULE", self._get_rule_opcode(name)) - def visit_Repeat1(self, node: Repeat0) -> None: + def visit_Repeat1(self, node: Repeat1) -> None: + name = self.callmakervisitor.visit(node) + self.add_opcode("OP_RULE", self._get_rule_opcode(name)) + + def visit_Gather(self, node: Gather) -> None: name = self.callmakervisitor.visit(node) self.add_opcode("OP_RULE", self._get_rule_opcode(name)) From a10babb8c9e5487ca5af90781fc60dcc786192fc Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 1 Jun 2020 17:09:02 +0300 Subject: [PATCH 45/67] Implement soft keywords (hand-written and code generation) (#129) --- Parser/pegen/vm.c | 4 +++ Parser/pegen/vm.h | 2 ++ Parser/pegen/vmparse.h | 18 ++++++++-- Tools/peg_generator/data/simple.gram | 1 + Tools/peg_generator/pegen/vm_generator.py | 43 +++++++++++++++++++---- 5 files changed, 58 insertions(+), 10 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 3a98028ff22ef8..34710c9574f3ea 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -153,6 +153,10 @@ run_vm(Parser *p, Rule rules[], int root) oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); break; + case OP_SOFT_KEYWORD: + oparg = f->rule->opcodes[f->iop++]; + v = _PyPegen_expect_soft_keyword(p, soft_keywords[oparg]); + break; case OP_RULE: oparg = f->rule->opcodes[f->iop++]; Rule *rule = &rules[oparg]; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index f4e8c33102bc55..692d9ce2cd012b 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -12,6 +12,7 @@ typedef enum _opcodes { OP_SUCCESS, OP_FAILURE, // The rest have an argument + OP_SOFT_KEYWORD, OP_TOKEN, OP_RULE, OP_RETURN, @@ -31,6 +32,7 @@ static char *opcode_names[] = { "OP_SUCCESS", "OP_FAILURE", // The rest have an argument + "OP_SOFT_KEYWORD", "OP_TOKEN", "OP_RULE", "OP_RETURN", diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 27299a9acd0d5e..f3886fcbc88ad1 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -8,6 +8,14 @@ static KeywordToken *reserved_keywords[] = { }, }; +enum { + SK___PEG_PARSER__, +}; + +static const char *soft_keywords[] = { + "__peg_parser__", +}; + enum { R_START, R_STMT, @@ -33,6 +41,7 @@ enum { A_FACTOR_1, A_FACTOR_2, A_FACTOR_3, + A_FACTOR_4, A__GATHER_2_0, A__GATHER_2_1, }; @@ -78,12 +87,13 @@ static Rule all_rules[] = { }, {"factor", R_FACTOR, - {0, 8, 16, 19, -1}, + {0, 8, 16, 19, 23, -1}, { OP_TOKEN, 7, OP_RULE, R_EXPR, OP_TOKEN, 8, OP_RETURN, A_FACTOR_0, OP_TOKEN, 9, OP_RULE, R__GATHER_2, OP_TOKEN, 10, OP_RETURN, A_FACTOR_1, OP_NUMBER, OP_RETURN, A_FACTOR_2, - OP_NAME, OP_RETURN, A_FACTOR_3, + OP_SOFT_KEYWORD, SK___PEG_PARSER__, OP_RETURN, A_FACTOR_3, + OP_NAME, OP_RETURN, A_FACTOR_4, }, }, {"root", @@ -132,7 +142,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_EXPR_1: case A_TERM_1: case A_FACTOR_2: - case A_FACTOR_3: + case A_FACTOR_4: case A__GATHER_2_0: case A__GATHER_2_1: return _f->vals[0]; @@ -146,6 +156,8 @@ call_action(Parser *p, Frame *_f, int _iaction) return _f->vals[1]; case A_FACTOR_1: return _Py_List ( _f->vals[1] , Load , EXTRA ); + case A_FACTOR_3: + return RAISE_SYNTAX_ERROR("You found it!"); default: assert(0); } diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index dec41289e64dfb..9218934d04b9c1 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -14,4 +14,5 @@ factor: | '(' a=expr ')' { a } | '[' a=','.expr+ ']' { _Py_List(a, Load, EXTRA) } | NUMBER + | "__peg_parser__" { RAISE_SYNTAX_ERROR("You found it!") } | NAME diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index c0a6a52b809af2..4c30355d30d8a6 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -68,17 +68,27 @@ def __init__( self.gen = parser_generator self.cache: Dict[Any, Any] = {} self.keyword_cache: Dict[str, int] = {} + self.soft_keyword_cache: List[str] = [] - def keyword_helper(self, keyword: str) -> int: + def keyword_helper(self, keyword: str) -> Tuple[str, int]: if keyword not in self.keyword_cache: self.keyword_cache[keyword] = self.gen.keyword_type() - return self.keyword_cache[keyword] + return "OP_TOKEN", self.keyword_cache[keyword] - def visit_StringLeaf(self, node: StringLeaf) -> int: + def soft_keyword_helper(self, keyword: str) -> Tuple[str, str]: + if keyword not in self.soft_keyword_cache: + self.soft_keyword_cache.append(keyword) + return "OP_SOFT_KEYWORD", f"SK_{keyword.upper()}" + + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, Union[str, int]]: val = ast.literal_eval(node.value) if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword - return self.keyword_helper(val) - return token.EXACT_TOKEN_TYPES[val] # type: ignore [attr-defined] + if node.value.endswith("'"): + return self.keyword_helper(val) + else: + return self.soft_keyword_helper(val) + tok_num = token.EXACT_TOKEN_TYPES[val] + return "OP_TOKEN", token.tok_name[tok_num] def visit_Repeat0(self, node: Repeat0) -> None: if node in self.cache: @@ -133,6 +143,7 @@ def generate(self, filename: str) -> None: self.collect_todo() self.gather_actions() self._setup_keywords() + self._setup_soft_keywords() self.print("enum {") with self.indent(): @@ -194,6 +205,24 @@ def _setup_keywords(self) -> None: self.print("};") self.print() + def _setup_soft_keywords(self) -> None: + soft_keywords = self.callmakervisitor.soft_keyword_cache + if not soft_keywords: + return + + self.print("enum {") + with self.indent(): + for soft_keyword in soft_keywords: + self.print(f"SK_{soft_keyword.upper()},") + self.print("};") + self.print() + self.print("static const char *soft_keywords[] = {") + with self.indent(): + for soft_keyword in soft_keywords: + self.print(f'"{soft_keyword}",') + self.print("};") + self.print() + def print_action_cases(self) -> None: unique_actions: Dict[str, List[str]] = defaultdict(list) for actionname, action in self.actions.items(): @@ -310,8 +339,8 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: self.add_opcode("OP_RULE", self._get_rule_opcode(name)) def visit_StringLeaf(self, node: StringLeaf) -> None: - token_type = self.callmakervisitor.visit(node) - self.add_opcode("OP_TOKEN", token_type) + op_pair = self.callmakervisitor.visit(node) + self.add_opcode(*op_pair) def handle_loop_rhs( self, node: Rhs, opcodes_by_alt: Dict[int, List[str]], collect_opcode: str, From b13169b7f7c378eb0b7bd5163dca84eb9a1ee407 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Jun 2020 12:21:50 -0700 Subject: [PATCH 46/67] Update generated vmparse.h --- Parser/pegen/vmparse.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index f3886fcbc88ad1..f76629be1b33b5 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -66,14 +66,14 @@ static Rule all_rules[] = { R_IF_STMT, {0, -1}, { - OP_TOKEN, 500, OP_NAME, OP_TOKEN, 11, OP_RULE, R_EXPR, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_EXPR, OP_RETURN, A_IF_STMT_0, }, }, {"expr", R_EXPR, {0, 8, -1}, { - OP_RULE, R_TERM, OP_TOKEN, 14, OP_RULE, R_EXPR, OP_RETURN, A_EXPR_0, + OP_RULE, R_TERM, OP_TOKEN, PLUS, OP_RULE, R_EXPR, OP_RETURN, A_EXPR_0, OP_RULE, R_TERM, OP_RETURN, A_EXPR_1, }, }, @@ -81,7 +81,7 @@ static Rule all_rules[] = { R_TERM, {0, 8, -1}, { - OP_RULE, R_FACTOR, OP_TOKEN, 16, OP_RULE, R_TERM, OP_RETURN, A_TERM_0, + OP_RULE, R_FACTOR, OP_TOKEN, STAR, OP_RULE, R_TERM, OP_RETURN, A_TERM_0, OP_RULE, R_FACTOR, OP_RETURN, A_TERM_1, }, }, @@ -89,8 +89,8 @@ static Rule all_rules[] = { R_FACTOR, {0, 8, 16, 19, 23, -1}, { - OP_TOKEN, 7, OP_RULE, R_EXPR, OP_TOKEN, 8, OP_RETURN, A_FACTOR_0, - OP_TOKEN, 9, OP_RULE, R__GATHER_2, OP_TOKEN, 10, OP_RETURN, A_FACTOR_1, + OP_TOKEN, LPAR, OP_RULE, R_EXPR, OP_TOKEN, RPAR, OP_RETURN, A_FACTOR_0, + OP_TOKEN, LSQB, OP_RULE, R__GATHER_2, OP_TOKEN, RSQB, OP_RETURN, A_FACTOR_1, OP_NUMBER, OP_RETURN, A_FACTOR_2, OP_SOFT_KEYWORD, SK___PEG_PARSER__, OP_RETURN, A_FACTOR_3, OP_NAME, OP_RETURN, A_FACTOR_4, @@ -116,7 +116,7 @@ static Rule all_rules[] = { R__GATHER_2, {0, 5, -1}, { - OP_RULE, R_EXPR, OP_TOKEN, 12, OP_LOOP_ITERATE, + OP_RULE, R_EXPR, OP_TOKEN, COMMA, OP_LOOP_ITERATE, OP_RULE, R_EXPR, OP_LOOP_COLLECT_DELIMITED, }, }, @@ -157,7 +157,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_FACTOR_1: return _Py_List ( _f->vals[1] , Load , EXTRA ); case A_FACTOR_3: - return RAISE_SYNTAX_ERROR("You found it!"); + return RAISE_SYNTAX_ERROR ( "You found it!" ); default: assert(0); } From c2a7cf66e70638c60e31b1e54db0efb3375878fa Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Jun 2020 12:59:28 -0700 Subject: [PATCH 47/67] Fix code generation for if_stmt --- Parser/pegen/vmparse.h | 6 +++--- Tools/peg_generator/data/simple.gram | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index f76629be1b33b5..dc7e5a5b18a5f2 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -59,14 +59,14 @@ static Rule all_rules[] = { {0, 6, -1}, { OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, - OP_RULE, R_IF_STMT, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_1, + OP_RULE, R_IF_STMT, OP_RETURN, A_STMT_1, }, }, {"if_stmt", R_IF_STMT, {0, -1}, { - OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_EXPR, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, }, }, {"expr", @@ -147,7 +147,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A__GATHER_2_1: return _f->vals[0]; case A_IF_STMT_0: - return _Py_If ( _f->vals[1] , _f->vals[3] , NULL , EXTRA ); + return _Py_If ( _f->vals[1] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[3] ) ) , NULL , EXTRA ); case A_EXPR_0: return _Py_BinOp ( _f->vals[0] , Add , _f->vals[2] , EXTRA ); case A_TERM_0: diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 9218934d04b9c1..50b41226b368c2 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -1,9 +1,9 @@ start: a=stmt+ ENDMARKER { _PyPegen_make_module(p, a) } stmt: | a=expr NEWLINE { _Py_Expr(a, EXTRA) } - | a=if_stmt NEWLINE { a } + | a=if_stmt if_stmt: - | 'if' a=NAME ':' b=expr { _Py_If(a, b, NULL, EXTRA) } + | 'if' a=NAME ':' b=stmt { _Py_If(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } expr: | a=term '+' b=expr { _Py_BinOp(a, Add, b, EXTRA) } | term From a862a69f8f248c99c6072f909d46dd69a437389f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Jun 2020 14:16:47 -0700 Subject: [PATCH 48/67] Implement lookahead ops --- Parser/pegen/vm.c | 23 ++++++- Parser/pegen/vm.h | 9 ++- Parser/pegen/vmparse.h | 8 +-- Parser/pegen/vmreadme.md | 126 +++++++++++++++++++-------------------- 4 files changed, 96 insertions(+), 70 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 34710c9574f3ea..8cb5b02a17e54d 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -106,6 +106,7 @@ run_vm(Parser *p, Rule rules[], int root) goto top; case OP_OPTIONAL: goto top; + case OP_LOOP_ITERATE: f->mark = p->mark; assert(f->ival == 1 || f->ival == 2); @@ -149,6 +150,17 @@ run_vm(Parser *p, Rule rules[], int root) return PyErr_NoMemory(); } break; + + case OP_SAVE_MARK: + f->savemark = p->mark; + goto top; + case OP_POS_LOOKAHEAD: + p->mark = f->savemark; + goto top; + case OP_NEG_LOOKAHEAD: + v = NULL; + break; + case OP_TOKEN: oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); @@ -199,13 +211,20 @@ run_vm(Parser *p, Rule rules[], int root) } fail: - if (f->rule->opcodes[f->iop] == OP_OPTIONAL) { - D(printf(" GA!\n")); + opc = f->rule->opcodes[f->iop]; + if (opc == OP_OPTIONAL) { + D(printf(" OP_OPTIONAL\n")); assert(f->ival < MAXVALS); f->vals[f->ival++] = NULL; f->iop++; // Skip over the OP_OPTIONAL opcode goto top; } + if (opc == OP_NEG_LOOKAHEAD) { + D(printf(" OP_NEG_LOOKAHEAD\n")); + p->mark = f->savemark; + f->iop++; // Skip over the OP_NEG_LOOKAHEAD opcode + goto top; + } D(printf(" fail\n")); p->mark = f->mark; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 692d9ce2cd012b..09499f1dfb4a3a 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -9,6 +9,9 @@ typedef enum _opcodes { OP_LOOP_COLLECT, OP_LOOP_COLLECT_NONEMPTY, OP_LOOP_COLLECT_DELIMITED, + OP_SAVE_MARK, + OP_POS_LOOKAHEAD, + OP_NEG_LOOKAHEAD, OP_SUCCESS, OP_FAILURE, // The rest have an argument @@ -29,6 +32,9 @@ static char *opcode_names[] = { "OP_LOOP_COLLECT", "OP_LOOP_COLLECT_NONEMPTY", "OP_LOOP_COLLECT_DELIMITED", + "OP_SAVE_MARK", + "OP_POS_LOOKAHEAD", + "OP_NEG_LOOKAHEAD", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument @@ -53,12 +59,13 @@ typedef struct _rule { typedef struct _frame { Rule *rule; int mark; + int savemark; int ialt; int iop; int cut; int ncollected; int capacity; - void **collection; int ival; + void **collection; void *vals[MAXVALS]; } Frame; diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index dc7e5a5b18a5f2..4acacd7d1abce7 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -56,9 +56,9 @@ static Rule all_rules[] = { }, {"stmt", R_STMT, - {0, 6, -1}, + {0, 10, -1}, { - OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, + OP_SAVE_MARK, OP_TOKEN, 500, OP_NEG_LOOKAHEAD, OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, OP_RULE, R_IF_STMT, OP_RETURN, A_STMT_1, }, }, @@ -66,7 +66,7 @@ static Rule all_rules[] = { R_IF_STMT, {0, -1}, { - OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, + OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, }, }, {"expr", @@ -147,7 +147,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A__GATHER_2_1: return _f->vals[0]; case A_IF_STMT_0: - return _Py_If ( _f->vals[1] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[3] ) ) , NULL , EXTRA ); + return _Py_If ( _f->vals[2] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[4] ) ) , NULL , EXTRA ); case A_EXPR_0: return _Py_BinOp ( _f->vals[0] , Add , _f->vals[2] , EXTRA ); case A_TERM_0: diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 8b99f8be3273d6..77965fc9706e8e 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -61,6 +61,7 @@ The `opcodes` array is a sequence of operation codes and arguments. Some opcodes (e.g., `OP_TOKEN`) are followed by an argument; others (e.g., `OP_NAME`) are not. Both are representable as integers. + Operations ---------- @@ -82,6 +83,7 @@ Some operations manipulate other frame fields. Calls into the support runtime can produce *errors* -- when an error is detected, the VM exits immediately, returning `NULL`. + ### General operations The following opcodes take no argument. @@ -115,6 +117,7 @@ The following operations are followed by a single integer argument. newly revealed by that pop operation) as if the previous operation succeeded or failed with the return value of the action. + ### Operations for root rules only A grammar must have one or more *root rules*. A root rule is a @@ -133,6 +136,7 @@ second alternative must be the single operation `OP_FAILURE`. - `OP_FAILURE` -- report a syntax error and exit the VM with a NULL result. + ### Operations for loop rules only For a loop such as `a*` or `a+`, a synthetic rule must be created with @@ -188,6 +192,44 @@ The new operation is: - `OP_LOOP_COLLECT_DELIMITED` -- Add the first value from the values array to the collection and then do everything that `OP_LOOP_COLLECT does. + +### Operations for lookaheads + +Positive lookaheads use the following pattern: + +``` +OP_SAVE_MARK + +OP_POS_LOOKAHEAD +``` + +The operations work as follows: + +- `OP_SAVE_MARK` -- saves the current input position in a dedicated + field of the frame, `savemark`. + +- `OP_POS_LOOKAHEAD` -- restores the current input position from the + frame's `savemark` field. (It does not reset the values array; + values produced by positive lookaheads are ignored by the actions. + +Negative lookaheads use the following pattern: + +``` +OP_SAVE_MARK + +OP_NEG_LOOKAHEAD +``` + +The new operation works as follows: + +- `OP_NEG_LOOKAHEAD` -- fails the current alternative. + +In addition, the standard code for success/failure processing checks +whether the next operation is `OP_NEG_LOOKAHEAD`. If so, it treats +`NULL` as a success and restores the current input position from the +frame's `savemark` field. + + More about `OP_OPTIONAL` ------------------------ @@ -202,6 +244,10 @@ to the next operation. When the operation preceding `OP_OPTIONAL` succeeds, `OP_OPTIONAL` is executed as a regular operation, and always succeed. +The `OP_NEG_LOOKAHEAD` works similar (but it also restores the input +position). + + Constraints on operation order ------------------------------ @@ -218,6 +264,7 @@ These operations must be last in their alternative: `OP_RETURN`, This operation must be first in its alternative: `OP_FAILURE`. + Grammar for lists of operations ------------------------------- @@ -245,78 +292,31 @@ delimited_collect_alt: OP_LOOP_COLLECT_DELIMITED alt: any_op+ return_op -any_op: regular_op [OP_OPTIONAL] | special_op +any_op: regular_op [OP_OPTIONAL] | lookahead_block | special_op regular_op: short_op | long_op short_op: OP_NAME | OP_NUMBER | OP_STRING long_op: OP_TOKEN | OP_RULE special_op: OP_NOOP | OP_CUT return_op: OP_RETURN + +lookahead_block: OP_SAVE_MARK regular_op lookahead_op +lookahead_op: OP_POS_LOOKAHEAD | OP_NEG_LOOKAHEAD ``` Ideas ----- -### "Special" loops - -We still need opcodes for `b.a+` (e.g. `','.expr+` is a -comma-separated list of expressions). We could generate code like this: - -``` -# first alternative: - - -OP_LOOP_ITERATE - -# second alternative: - -OP_LOOP_COLLECT_NONEMPTY -``` - -- The `OP_LOOP_ITERATE` opcode would ignore the value collected for - the delimiter. - -- The `OP_LOOP_COLLECT_NONEMPTY` opcode would add the final collected - value to the collection, if one was collected. (Alternatively, we - could use a new opcode, e.g. `OP_LOOP_COLLECT_DELIMITED`.) - -### Lookaheads - -We also need to extend the VM to support lookaheads. - -- Positive lookahead (`&a`) -- add a new opcode to save the mark in a - new frame field, and another opcode that restores the mark from - there. - -- Negative lookahead (`!a`) -- add another opcode that catches failure - and turns it into success, restoring the saved mark. - ### Left-recursion -Hmm, tricky... Will think about this later. - -Lookahead opcodes ------------------ - -``` -case OP_SAVE_MARK: - f->savemark = p->mark; - goto top; - -case OP_POS_LOOKAHEAD: - p->mark - f->savemark; - goto top; - -case OP_NEG_LOOKAHEAD: - goto fail; - -/* Later, when v == NULL */ -if (f->rule->opcodes[f->iop] == OP_NEG_LOOKAHEAD) { - p->mark - f->savemark; - goto top; -} -/* Also at the other fail check, under pop */ -``` - -If we initialize savemark to the same value as mark, we can avoid a -`SAVE_MARK` opcode at the start of alternatives -- this is a common -pattern. (OTOH, it costs an extra store for each frame.) +- First opcode of first alt is `OP_SETUP_LEFT_REC`. This initializes + the memo cache for f->rule->type to `NULL`. + +- All `OP_RETURN` opcodes in a left-rec rule are replaced with + `OP_RETURN_LEFT_REC`. This compares the current position with the + most recently cached position for this rule at this point in the + input. If the new match is *further*, it updates the memo cache, + resets `f->mark`, and resets `f->iop` and `f->ialt` to the start of + the rule. It then goes back to the top (possibly skipping the setup + op). If the new match is not further, it is discarded and the most + recent match from the memo cache is returned as the result (also + updating the end position). From ab863df02e62e5bcf92f605a24e912555b5b5778 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Jun 2020 15:22:55 -0700 Subject: [PATCH 49/67] Generate code for lookaheads (only one token supported!) This is not complete, but I want to get to left-recursion before I fix this, and I don't actually understand the code generator well enough to know how to make it work for `&('foo'|'bar')`. --- Parser/pegen/vm.c | 2 ++ Parser/pegen/vmparse.h | 6 +++--- Tools/peg_generator/data/simple.gram | 4 ++-- Tools/peg_generator/pegen/vm_generator.py | 11 ++++++++++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 8cb5b02a17e54d..8334aaf61dab83 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -155,6 +155,8 @@ run_vm(Parser *p, Rule rules[], int root) f->savemark = p->mark; goto top; case OP_POS_LOOKAHEAD: + assert(f->ival > 0); + f->ival--; /* Back out last added value */ p->mark = f->savemark; goto top; case OP_NEG_LOOKAHEAD: diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 4acacd7d1abce7..7c5f3e175c52dd 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -59,14 +59,14 @@ static Rule all_rules[] = { {0, 10, -1}, { OP_SAVE_MARK, OP_TOKEN, 500, OP_NEG_LOOKAHEAD, OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, - OP_RULE, R_IF_STMT, OP_RETURN, A_STMT_1, + OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_RULE, R_IF_STMT, OP_RETURN, A_STMT_1, }, }, {"if_stmt", R_IF_STMT, {0, -1}, { - OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, }, }, {"expr", @@ -147,7 +147,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A__GATHER_2_1: return _f->vals[0]; case A_IF_STMT_0: - return _Py_If ( _f->vals[2] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[4] ) ) , NULL , EXTRA ); + return _Py_If ( _f->vals[1] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[3] ) ) , NULL , EXTRA ); case A_EXPR_0: return _Py_BinOp ( _f->vals[0] , Add , _f->vals[2] , EXTRA ); case A_TERM_0: diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 50b41226b368c2..016ad4c51012d2 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -1,7 +1,7 @@ start: a=stmt+ ENDMARKER { _PyPegen_make_module(p, a) } stmt: - | a=expr NEWLINE { _Py_Expr(a, EXTRA) } - | a=if_stmt + | !'if' a=expr NEWLINE { _Py_Expr(a, EXTRA) } + | &'if' a=if_stmt if_stmt: | 'if' a=NAME ':' b=stmt { _Py_If(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } expr: diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 4c30355d30d8a6..2866397a047b81 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -326,6 +326,16 @@ def visit_Rule(self, node: Rule) -> None: def visit_NamedItem(self, node: NamedItem) -> None: self.visit(node.item) + def visit_PositiveLookahead(self, node: PositiveLookahead) -> None: + self.add_opcode("OP_SAVE_MARK") + self.visit(node.node) + self.add_opcode("OP_POS_LOOKAHEAD") + + def visit_NegativeLookahead(self, node: PositiveLookahead) -> None: + self.add_opcode("OP_SAVE_MARK") + self.visit(node.node) + self.add_opcode("OP_NEG_LOOKAHEAD") + def _get_rule_opcode(self, name: str) -> str: return f"R_{name.upper()}" @@ -337,7 +347,6 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: self.add_opcode("OP_TOKEN", name) else: self.add_opcode("OP_RULE", self._get_rule_opcode(name)) - def visit_StringLeaf(self, node: StringLeaf) -> None: op_pair = self.callmakervisitor.visit(node) self.add_opcode(*op_pair) From 1be4b62b3df9338256c61d80795c6b1bce86d307 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 1 Jun 2020 21:47:00 -0700 Subject: [PATCH 50/67] Implement left-recursion (with hand-coded vmparse.h) It definitely makes parsing xxl.py 5-10% slower. :-( --- Parser/pegen/vm.c | 81 ++++++++++++++++++++++++++++++++-------- Parser/pegen/vm.h | 4 ++ Parser/pegen/vmparse.h | 14 ++++--- Parser/pegen/vmreadme.md | 25 +++++++++++++ 4 files changed, 102 insertions(+), 22 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 8334aaf61dab83..95a87c155821fd 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -51,6 +51,7 @@ pop_frame(Stack *stack, void *v) if (f->collection) { PyMem_Free(f->collection); } + // TODO: Selective memoization _PyPegen_insert_memo(stack->p, f->mark, f->rule->type + 1000, v); f = &stack->frames[stack->top - 1]; // New top of stack D(printf(" pop %s\n", f->rule->name)); @@ -95,17 +96,21 @@ run_vm(Parser *p, Rule rules[], int root) switch (opc) { case OP_NOOP: goto top; + case OP_CUT: + f->cut = 1; + goto top; + case OP_OPTIONAL: + goto top; + case OP_NAME: v = _PyPegen_name_token(p); break; case OP_NUMBER: v = _PyPegen_number_token(p); break; - case OP_CUT: - f->cut = 1; - goto top; - case OP_OPTIONAL: - goto top; + case OP_STRING: + v = _PyPegen_string_token(p); + break; case OP_LOOP_ITERATE: f->mark = p->mark; @@ -163,6 +168,20 @@ run_vm(Parser *p, Rule rules[], int root) v = NULL; break; + case OP_SETUP_LEFT_REC: + assert(p->mark == f->mark); + assert (_PyPegen_is_memoized(p, f->rule->type + 1000, &v) == 0); + if (_PyPegen_insert_memo(p, f->mark, f->rule->type + 1000, NULL) == -1) { + return NULL; + } + goto top; + + case OP_SUCCESS: + v = f->vals[0]; + return v; + case OP_FAILURE: + return RAISE_SYNTAX_ERROR("A syntax error"); + case OP_TOKEN: oparg = f->rule->opcodes[f->iop++]; v = _PyPegen_expect_token(p, oparg); @@ -174,14 +193,12 @@ run_vm(Parser *p, Rule rules[], int root) case OP_RULE: oparg = f->rule->opcodes[f->iop++]; Rule *rule = &rules[oparg]; + // TODO: Selective memoization + v = NULL; // In case is_memoized ran into an error int memo = _PyPegen_is_memoized(p, rule->type + 1000, &v); if (memo) { - if (memo < 0) { - return NULL; - } - else { - break; // The result is v - } + // The result is v; if v != NULL, p->mark has been updated + break; } f = push_frame(&stack, rule); goto top; @@ -190,11 +207,43 @@ run_vm(Parser *p, Rule rules[], int root) v = call_action(p, f, oparg); f = pop_frame(&stack, v); break; - case OP_SUCCESS: - v = f->vals[0]; - return v; - case OP_FAILURE: - return RAISE_SYNTAX_ERROR("A syntax error"); + + case OP_RETURN_LEFT_REC: + oparg = f->rule->opcodes[f->iop++]; + v = call_action(p, f, oparg); + if (v) { + // It's a little tricky to recover the cached position: + // we must call is_memoized() for the start position, and then + // is_memoized() updates p->mark to the cached end position + int newmark = p->mark; + p->mark = f->mark; + void *w; + if (_PyPegen_is_memoized(p, f->rule->type + 1000, &w) == -1) { + return NULL; + } + if (newmark > p->mark) { + D(printf(" newmark wins\n")); + // The new result is better than the cached value; update memo + p->mark = newmark; + int ok = _PyPegen_update_memo(p, f->mark, f->rule->type + 1000, v); + if (ok == -1) { + return NULL; + } + // Restart parsing this rule from the top + p->mark = f->mark; + f->ival = 0; + f->ialt = 0; + f->iop = f->rule->alts[0]; + assert(f->rule->opcodes[f->iop] == OP_SETUP_LEFT_REC); + f->iop++; // Skip over OP_SETUP_LEFT_REC + goto top; + } + // End the recursion loop, popping the frame + v = w; + f = pop_frame(&stack, v); + } + break; + default: printf("opc=%d\n", opc); assert(0); diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 09499f1dfb4a3a..1340863f08df48 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -12,6 +12,7 @@ typedef enum _opcodes { OP_SAVE_MARK, OP_POS_LOOKAHEAD, OP_NEG_LOOKAHEAD, + OP_SETUP_LEFT_REC, OP_SUCCESS, OP_FAILURE, // The rest have an argument @@ -19,6 +20,7 @@ typedef enum _opcodes { OP_TOKEN, OP_RULE, OP_RETURN, + OP_RETURN_LEFT_REC, } Opcode; static char *opcode_names[] = { @@ -35,6 +37,7 @@ static char *opcode_names[] = { "OP_SAVE_MARK", "OP_POS_LOOKAHEAD", "OP_NEG_LOOKAHEAD", + "OP_SETUP_LEFT_REC", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument @@ -42,6 +45,7 @@ static char *opcode_names[] = { "OP_TOKEN", "OP_RULE", "OP_RETURN", + "OP_RETURN_LEFT_REC", }; #define MAXALTS 10 diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 7c5f3e175c52dd..87b316a71e32ce 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -71,18 +71,20 @@ static Rule all_rules[] = { }, {"expr", R_EXPR, - {0, 8, -1}, + {0, 9, -1}, { - OP_RULE, R_TERM, OP_TOKEN, PLUS, OP_RULE, R_EXPR, OP_RETURN, A_EXPR_0, - OP_RULE, R_TERM, OP_RETURN, A_EXPR_1, + OP_SETUP_LEFT_REC, + OP_RULE, R_EXPR, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_0, + OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_1, }, }, {"term", R_TERM, - {0, 8, -1}, + {0, 9, -1}, { - OP_RULE, R_FACTOR, OP_TOKEN, STAR, OP_RULE, R_TERM, OP_RETURN, A_TERM_0, - OP_RULE, R_FACTOR, OP_RETURN, A_TERM_1, + OP_SETUP_LEFT_REC, + OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, + OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, }, }, {"factor", diff --git a/Parser/pegen/vmreadme.md b/Parser/pegen/vmreadme.md index 77965fc9706e8e..c82b83b3464d42 100644 --- a/Parser/pegen/vmreadme.md +++ b/Parser/pegen/vmreadme.md @@ -320,3 +320,28 @@ Ideas op). If the new match is not further, it is discarded and the most recent match from the memo cache is returned as the result (also updating the end position). + +### Selective memoization + +- We could have a flag in the rule that prevents memo lookups and + inserts. Or we could have separate opcodes, e.g. `OP_RULE_NOMEMO` + and `OP_RETURN_NOMEMO`. + +### Performance tuning + +- To make frames smaller, we could have a separate values stack; the + frame would have a `void** vals` instead of `void *vals[]`. Most + frames won't need 20 values, but we could ensure there are always at + least that many on the stack. + +- Is it faster to check for flags in the rule object (e.g. leftrec) or + is it faster to have dedicated opcodes? My current hunch is that + dedicated opcodes are faster, but I really don't know. Maybe having + fewer opcodes is more important than having smaller Rule objects. + +- Random other C tricks, esp. tricks that might increase the hit rate + in the CPU's L1 cache. (Remember in modern CPUs memory access is + 10-100x slower than cache access.) Who can we tap that knows this + stuff? + +- If we know `x >= -1`, Which is faster: `if (x < 0)` or `if (x == -1)`? From b53c2a4824e46543d49cea1efe428f66da2a54c2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 09:58:55 -0700 Subject: [PATCH 51/67] Code generation for left-recursive rules This was super easy. Benchmark time for xxl.py is now around 2.030 seconds. (But this is with a super simple grammar. We'll have to see what is will be with the real grammar.) --- Parser/pegen/vmparse.h | 6 ++---- Tools/peg_generator/data/simple.gram | 4 ++-- Tools/peg_generator/pegen/vm_generator.py | 16 +++++++++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 87b316a71e32ce..661188538c48de 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -73,8 +73,7 @@ static Rule all_rules[] = { R_EXPR, {0, 9, -1}, { - OP_SETUP_LEFT_REC, - OP_RULE, R_EXPR, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_0, + OP_SETUP_LEFT_REC, OP_RULE, R_EXPR, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_0, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_1, }, }, @@ -82,8 +81,7 @@ static Rule all_rules[] = { R_TERM, {0, 9, -1}, { - OP_SETUP_LEFT_REC, - OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, + OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, }, }, diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 016ad4c51012d2..ed281691bf76cc 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -5,10 +5,10 @@ stmt: if_stmt: | 'if' a=NAME ':' b=stmt { _Py_If(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } expr: - | a=term '+' b=expr { _Py_BinOp(a, Add, b, EXTRA) } + | a=expr '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } | term term: - | a=factor '*' b=term { _Py_BinOp(a, Mult, b, EXTRA) } + | a=term '*' b=factor { _Py_BinOp(a, Mult, b, EXTRA) } | factor factor: | '(' a=expr ')' { a } diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 2866397a047b81..f00ddfe16a3e4b 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -315,12 +315,10 @@ def visit_Rule(self, node: Rule) -> None: is_loop1 = node.name.startswith("_loop1") is_gather = node.is_gather() rhs = node.flatten() - if node.left_recursive: - raise ValueError("No left recursiveness") self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") self.current_rule = node # TODO: make this a context manager - self.visit(rhs, is_loop=is_loop, is_loop1=is_loop1, is_gather=is_gather) + self.visit(rhs, is_loop=is_loop, is_loop1=is_loop1, is_gather=is_gather, is_leftrec=node.left_recursive) self.print("},") def visit_NamedItem(self, node: NamedItem) -> None: @@ -365,9 +363,12 @@ def handle_default_rhs( is_loop: bool = False, is_loop1: bool = False, is_gather: bool = False, + is_leftrec: bool = False, ) -> None: for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[index]): + if is_leftrec and index == 0: + self.add_opcode("OP_SETUP_LEFT_REC") self.visit(alt, is_loop=False, is_loop1=False, is_gather=False) assert not (alt.action and (is_loop or is_gather)) # A loop rule can't have actions if is_loop or is_gather: @@ -376,10 +377,14 @@ def handle_default_rhs( else: self.add_opcode("OP_LOOP_COLLECT_DELIMITED") else: - self.add_opcode("OP_RETURN", f"A_{self.current_rule.name.upper()}_{index}") + opcode = "OP_RETURN" + if is_leftrec: + opcode += "_LEFT_REC" + self.add_opcode(opcode, f"A_{self.current_rule.name.upper()}_{index}") def visit_Rhs( self, node: Rhs, is_loop: bool = False, is_loop1: bool = False, is_gather: bool = False, + is_leftrec: bool = False, ) -> None: opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) if is_loop: @@ -388,7 +393,8 @@ def visit_Rhs( opcode += "_NONEMPTY" self.handle_loop_rhs(node, opcodes_by_alt, opcode) else: - self.handle_default_rhs(node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather) + self.handle_default_rhs(node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather, + is_leftrec=is_leftrec) *indexes, _ = accumulate(map(len, opcodes_by_alt.values())) indexes = [0, *indexes, -1] self.print(f" {{{', '.join(map(str, indexes))}}},") From 7a59aa0942b770efe186d3b46f7b5c28d3f5ba33 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 10:09:13 -0700 Subject: [PATCH 52/67] Allow specifying different grammars --- Tools/peg_generator/pegen/vm_generator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index f00ddfe16a3e4b..bf54e9b20e3b07 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -6,6 +6,7 @@ import tokenize from collections import defaultdict from itertools import accumulate +import sys from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from pegen import grammar @@ -423,7 +424,10 @@ def visit_Gather(self, node: Gather) -> None: def main() -> None: - grammar, parser, tokenizer = build_parser("./data/simple.gram", False, False) + filename = "./data/simple.gram" + if sys.argv[1:]: + filename = sys.argv[1] + grammar, parser, tokenizer = build_parser(filename, False, False) p = VMParserGenerator(grammar) p.generate("") From cac4149a4718b5a579bb2c108c36a6f1879fb61d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 15:39:24 -0700 Subject: [PATCH 53/67] Generate code for 'cut' Also silence compiler warning about default case in call_action(). --- Parser/pegen/vmparse.h | 4 +++- Tools/peg_generator/data/simple.gram | 2 +- Tools/peg_generator/pegen/vm_generator.py | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 661188538c48de..25caea5b7b11d7 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -66,7 +66,7 @@ static Rule all_rules[] = { R_IF_STMT, {0, -1}, { - OP_TOKEN, 500, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 500, OP_CUT, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, }, }, {"expr", @@ -160,5 +160,7 @@ call_action(Parser *p, Frame *_f, int _iaction) return RAISE_SYNTAX_ERROR ( "You found it!" ); default: assert(0); + RAISE_SYNTAX_ERROR("invalid opcode"); + return NULL; } } diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index ed281691bf76cc..1387225e322bbc 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -3,7 +3,7 @@ stmt: | !'if' a=expr NEWLINE { _Py_Expr(a, EXTRA) } | &'if' a=if_stmt if_stmt: - | 'if' a=NAME ':' b=stmt { _Py_If(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } + | 'if' ~ a=NAME ':' b=stmt { _Py_If(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } expr: | a=expr '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } | term diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index bf54e9b20e3b07..96da56fa6859ba 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -52,6 +52,8 @@ CALL_ACTION_TAIL = """\ default: assert(0); + RAISE_SYNTAX_ERROR("invalid opcode"); + return NULL; } } """ @@ -346,10 +348,14 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: self.add_opcode("OP_TOKEN", name) else: self.add_opcode("OP_RULE", self._get_rule_opcode(name)) + def visit_StringLeaf(self, node: StringLeaf) -> None: op_pair = self.callmakervisitor.visit(node) self.add_opcode(*op_pair) + def visit_Cut(self, node: Cut) -> None: + self.add_opcode("OP_CUT") + def handle_loop_rhs( self, node: Rhs, opcodes_by_alt: Dict[int, List[str]], collect_opcode: str, ) -> None: From 180dfffd7f56df8a027115f443780e1b9e0e4e3b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 16:46:54 -0700 Subject: [PATCH 54/67] Support groups and optional in code generator Also cleaned up the type of the second arg to add_opcode() -- it now must always be a string, the one call that didn't pass a string now calls str(). --- Parser/pegen/vmparse.h | 4 +- Tools/peg_generator/data/simple.gram | 2 +- Tools/peg_generator/pegen/vm_generator.py | 62 +++++++++++++++++++---- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 25caea5b7b11d7..556a8395b62710 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -79,9 +79,9 @@ static Rule all_rules[] = { }, {"term", R_TERM, - {0, 9, -1}, + {0, 10, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, + OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_OPTIONAL, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, }, }, diff --git a/Tools/peg_generator/data/simple.gram b/Tools/peg_generator/data/simple.gram index 1387225e322bbc..c1261e752e8d58 100644 --- a/Tools/peg_generator/data/simple.gram +++ b/Tools/peg_generator/data/simple.gram @@ -8,7 +8,7 @@ expr: | a=expr '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } | term term: - | a=term '*' b=factor { _Py_BinOp(a, Mult, b, EXTRA) } + | a=term ['*'] b=factor { _Py_BinOp(a, Mult, b, EXTRA) } | factor factor: | '(' a=expr ')' { a } diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 96da56fa6859ba..7c088cac92075b 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -7,7 +7,7 @@ from collections import defaultdict from itertools import accumulate import sys -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import Any, Dict, Iterator, List, Optional, Tuple from pegen import grammar from pegen.build import build_parser @@ -73,43 +73,67 @@ def __init__( self.keyword_cache: Dict[str, int] = {} self.soft_keyword_cache: List[str] = [] - def keyword_helper(self, keyword: str) -> Tuple[str, int]: + def keyword_helper(self, keyword: str) -> Tuple[str, str]: if keyword not in self.keyword_cache: self.keyword_cache[keyword] = self.gen.keyword_type() - return "OP_TOKEN", self.keyword_cache[keyword] + return "OP_TOKEN", str(self.keyword_cache[keyword]) def soft_keyword_helper(self, keyword: str) -> Tuple[str, str]: if keyword not in self.soft_keyword_cache: self.soft_keyword_cache.append(keyword) return "OP_SOFT_KEYWORD", f"SK_{keyword.upper()}" - def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, Union[str, int]]: + def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: val = ast.literal_eval(node.value) if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword if node.value.endswith("'"): return self.keyword_helper(val) else: return self.soft_keyword_helper(val) - tok_num = token.EXACT_TOKEN_TYPES[val] + tok_num: int = token.EXACT_TOKEN_TYPES[val] # type: ignore [attr-defined] return "OP_TOKEN", token.tok_name[tok_num] - def visit_Repeat0(self, node: Repeat0) -> None: + def visit_Repeat0(self, node: Repeat0) -> str: if node in self.cache: return self.cache[node] name = self.gen.name_loop(node.node, False) self.cache[node] = name + return name - def visit_Repeat1(self, node: Repeat1) -> None: + def visit_Repeat1(self, node: Repeat1) -> str: if node in self.cache: return self.cache[node] name = self.gen.name_loop(node.node, True) self.cache[node] = name + return name - def visit_Gather(self, node: Gather) -> None: + def visit_Gather(self, node: Gather) -> str: if node in self.cache: return self.cache[node] name = self.gen.name_gather(node) self.cache[node] = name + return name + + def visit_Group(self, node: Group) -> str: + return self.visit(node.rhs) + + def visit_Rhs(self, node: Rhs) -> Optional[str]: + if node in self.cache: + return self.cache[node] + if can_we_inline(node): + return None + name = self.gen.name_node(node) + self.cache[node] = name + return name + + +def can_we_inline(node: Rhs) -> int: + if len(node.alts) != 1 or len(node.alts[0].items) != 1: + return False + # If the alternative has an action we cannot inline + if getattr(node.alts[0], "action", None) is not None: + return False + return True class VMParserGenerator(ParserGenerator, GrammarVisitor): @@ -127,11 +151,12 @@ def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: yield self.opcode_buffer = None - def add_opcode(self, opcode: str, oparg: Optional[Union[int, str]] = None) -> None: + def add_opcode(self, opcode: str, oparg: Optional[str] = None) -> None: + assert not isinstance(oparg, int) assert self.opcode_buffer is not None self.opcode_buffer.append(opcode) if oparg is not None: - self.opcode_buffer.append(str(oparg)) + self.opcode_buffer.append(oparg) def name_gather(self, node: Gather) -> str: self.counter += 1 @@ -416,6 +441,23 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_loop1: bool, is_gather: bool) - for item in node.items: self.visit(item) + def group_helper(self, node: Rhs) -> None: + name = self.callmakervisitor.visit(node) + if name: + self.add_opcode("OP_RULE", self._get_rule_opcode(name)) + else: + self.visit(node.alts[0].items[0]) + + def visit_Opt(self, node: Opt) -> None: + if isinstance(node.node, Rhs): + self.group_helper(node.node) + else: + self.visit(node.node) + self.add_opcode("OP_OPTIONAL") + + def visit_Group(self, node: Group) -> None: + self.group_helper(node.rhs) + def visit_Repeat0(self, node: Repeat0) -> None: name = self.callmakervisitor.visit(node) self.add_opcode("OP_RULE", self._get_rule_opcode(name)) From e01e643920e50153b649446b70691fda143e3281 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 16:58:28 -0700 Subject: [PATCH 55/67] There's no need to special-case -> in actions --- Tools/peg_generator/pegen/vm_generator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 7c088cac92075b..3f57c7d342d42a 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -290,11 +290,8 @@ def translate_action(self, alt: Alt) -> str: i = name_to_index.get(s) if i is not None: s = f"_f->vals[{i}]" - if prevs == "-" and s == ">": - prevs = "->" - else: - prevs = s res.append(s) + prevs = s return " ".join(res).strip() def map_alt_names_to_vals_index(self, alt: Alt) -> Tuple[Dict[str, int], int]: From 13c3bbbc59e9d21888f76337b88f4b42d0ee1af8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 17:04:45 -0700 Subject: [PATCH 56/67] Treat TYPE_COMMENT as a token (since it is) --- Tools/peg_generator/pegen/vm_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 3f57c7d342d42a..7dc41a0e900177 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -366,7 +366,7 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: name = node.value if name in ("NAME", "NUMBER", "STRING"): self.add_opcode(f"OP_{name}") - elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT"): + elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT", "TYPE_COMMENT"): self.add_opcode("OP_TOKEN", name) else: self.add_opcode("OP_RULE", self._get_rule_opcode(name)) From 65304e610a330c4903eee39ba635599005070a2c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 2 Jun 2020 17:50:55 -0700 Subject: [PATCH 57/67] Generate code for Grammar/parser.gram (The bad news: it's currently twice as slow as the 'new' parser.) --- Parser/pegen/vm.c | 4 +- Parser/pegen/vm.h | 2 +- Parser/pegen/vmparse.h | 3869 ++++++++++++++++++++- Tools/peg_generator/pegen/vm_generator.py | 43 +- 4 files changed, 3804 insertions(+), 114 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 95a87c155821fd..c6fb772c79241e 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -17,7 +17,7 @@ #define D(x) #endif -#define MAXFRAMES 100 +#define MAXFRAMES 1000 typedef struct _stack { Parser *p; @@ -142,7 +142,7 @@ run_vm(Parser *p, Rule rules[], int root) // Fallthrough! case OP_LOOP_COLLECT_NONEMPTY: if (!f->ncollected) { - printf("Nothing collected for %s\n", f->rule->name); + D(printf("Nothing collected for %s\n", f->rule->name)); v = NULL; f = pop_frame(&stack, v); break; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 1340863f08df48..1f54c9f6e75e2b 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -48,7 +48,7 @@ static char *opcode_names[] = { "OP_RETURN_LEFT_REC", }; -#define MAXALTS 10 +#define MAXALTS 15 #define MAXOPCODES 100 typedef struct _rule { diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 556a8395b62710..8e724ae309f82b 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -1,163 +1,3838 @@ -static const int n_keyword_lists = 3; +static const int n_keyword_lists = 15; static KeywordToken *reserved_keywords[] = { NULL, NULL, (KeywordToken[]) { - {"if", 500}, + {"if", 510}, + {"in", 518}, + {"is", 526}, + {"as", 531}, + {"or", 532}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"del", 503}, + {"try", 511}, + {"for", 517}, + {"def", 522}, + {"not", 525}, + {"and", 533}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"pass", 502}, + {"from", 514}, + {"elif", 515}, + {"else", 516}, + {"with", 519}, + {"True", 527}, + {"None", 529}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"raise", 501}, + {"yield", 504}, + {"break", 506}, + {"while", 512}, + {"class", 523}, + {"False", 528}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"return", 500}, + {"assert", 505}, + {"global", 508}, + {"import", 513}, + {"except", 520}, + {"lambda", 524}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"finally", 521}, + {NULL, -1}, + }, + (KeywordToken[]) { + {"continue", 507}, + {"nonlocal", 509}, + {NULL, -1}, + }, + NULL, + NULL, + NULL, + NULL, + NULL, + (KeywordToken[]) { + {"__new_parser__", 530}, {NULL, -1}, }, -}; - -enum { - SK___PEG_PARSER__, }; static const char *soft_keywords[] = { - "__peg_parser__", }; enum { - R_START, - R_STMT, + R_FILE, + R_INTERACTIVE, + R_EVAL, + R_FUNC_TYPE, + R_FSTRING, + R_TYPE_EXPRESSIONS, + R_STATEMENTS, + R_STATEMENT, + R_STATEMENT_NEWLINE, + R_SIMPLE_STMT, + R_SMALL_STMT, + R_COMPOUND_STMT, + R_ASSIGNMENT, + R_AUGASSIGN, + R_GLOBAL_STMT, + R_NONLOCAL_STMT, + R_YIELD_STMT, + R_ASSERT_STMT, + R_DEL_STMT, + R_IMPORT_STMT, + R_IMPORT_NAME, + R_IMPORT_FROM, + R_IMPORT_FROM_TARGETS, + R_IMPORT_FROM_AS_NAMES, + R_IMPORT_FROM_AS_NAME, + R_DOTTED_AS_NAMES, + R_DOTTED_AS_NAME, + R_DOTTED_NAME, R_IF_STMT, - R_EXPR, + R_ELIF_STMT, + R_ELSE_BLOCK, + R_WHILE_STMT, + R_FOR_STMT, + R_WITH_STMT, + R_WITH_ITEM, + R_TRY_STMT, + R_EXCEPT_BLOCK, + R_FINALLY_BLOCK, + R_RETURN_STMT, + R_RAISE_STMT, + R_FUNCTION_DEF, + R_FUNCTION_DEF_RAW, + R_FUNC_TYPE_COMMENT, + R_PARAMS, + R_PARAMETERS, + R_SLASH_NO_DEFAULT, + R_SLASH_WITH_DEFAULT, + R_STAR_ETC, + R_KWDS, + R_PARAM_NO_DEFAULT, + R_PARAM_WITH_DEFAULT, + R_PARAM_MAYBE_DEFAULT, + R_PARAM, + R_ANNOTATION, + R_DEFAULT, + R_DECORATORS, + R_CLASS_DEF, + R_CLASS_DEF_RAW, + R_BLOCK, + R_EXPRESSIONS_LIST, + R_STAR_EXPRESSIONS, + R_STAR_EXPRESSION, + R_STAR_NAMED_EXPRESSIONS, + R_STAR_NAMED_EXPRESSION, + R_NAMED_EXPRESSION, + R_ANNOTATED_RHS, + R_EXPRESSIONS, + R_EXPRESSION, + R_LAMBDEF, + R_LAMBDA_PARAMETERS, + R_LAMBDA_SLASH_NO_DEFAULT, + R_LAMBDA_SLASH_WITH_DEFAULT, + R_LAMBDA_STAR_ETC, + R_LAMBDA_KWDS, + R_LAMBDA_PARAM_NO_DEFAULT, + R_LAMBDA_PARAM_WITH_DEFAULT, + R_LAMBDA_PARAM_MAYBE_DEFAULT, + R_LAMBDA_PARAM, + R_DISJUNCTION, + R_CONJUNCTION, + R_INVERSION, + R_COMPARISON, + R_COMPARE_OP_BITWISE_OR_PAIR, + R_EQ_BITWISE_OR, + R_NOTEQ_BITWISE_OR, + R_LTE_BITWISE_OR, + R_LT_BITWISE_OR, + R_GTE_BITWISE_OR, + R_GT_BITWISE_OR, + R_NOTIN_BITWISE_OR, + R_IN_BITWISE_OR, + R_ISNOT_BITWISE_OR, + R_IS_BITWISE_OR, + R_BITWISE_OR, + R_BITWISE_XOR, + R_BITWISE_AND, + R_SHIFT_EXPR, + R_SUM, R_TERM, R_FACTOR, + R_POWER, + R_AWAIT_PRIMARY, + R_PRIMARY, + R_SLICES, + R_SLICE, + R_ATOM, + R_STRINGS, + R_LIST, + R_LISTCOMP, + R_TUPLE, + R_GROUP, + R_GENEXP, + R_SET, + R_SETCOMP, + R_DICT, + R_DICTCOMP, + R_DOUBLE_STARRED_KVPAIRS, + R_DOUBLE_STARRED_KVPAIR, + R_KVPAIR, + R_FOR_IF_CLAUSES, + R_FOR_IF_CLAUSE, + R_YIELD_EXPR, + R_ARGUMENTS, + R_ARGS, + R_KWARGS, + R_STARRED_EXPRESSION, + R_KWARG_OR_STARRED, + R_KWARG_OR_DOUBLE_STARRED, + R_STAR_TARGETS, + R_STAR_TARGETS_SEQ, + R_STAR_TARGET, + R_STAR_ATOM, + R_SINGLE_TARGET, + R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, + R_DEL_TARGETS, + R_DEL_TARGET, + R_DEL_T_ATOM, + R_DEL_TARGET_END, + R_TARGETS, + R_TARGET, + R_T_PRIMARY, + R_T_LOOKAHEAD, + R_T_ATOM, + R_INCORRECT_ARGUMENTS, + R_INVALID_KWARG, + R_INVALID_NAMED_EXPRESSION, + R_INVALID_ASSIGNMENT, + R_INVALID_BLOCK, + R_INVALID_COMPREHENSION, + R_INVALID_DICT_COMPREHENSION, + R_INVALID_PARAMETERS, + R_INVALID_STAR_ETC, + R_INVALID_LAMBDA_STAR_ETC, + R_INVALID_DOUBLE_TYPE_COMMENTS, + R_INVALID_DEL_TARGET, + R_INVALID_IMPORT_FROM_TARGETS, R_ROOT, - R__LOOP1_1, - R__GATHER_2, + R__LOOP0_1, + R__LOOP0_2, + R__GATHER_3, + R__GATHER_4, + R__GATHER_5, + R__GATHER_6, + R__LOOP1_7, + R__GATHER_8, + R__TMP_9, + R__TMP_10, + R__TMP_11, + R__TMP_12, + R__TMP_13, + R__TMP_14, + R__TMP_15, + R__TMP_16, + R__LOOP1_17, + R__TMP_18, + R__TMP_19, + R__GATHER_20, + R__GATHER_21, + R__TMP_22, + R__LOOP0_23, + R__LOOP1_24, + R__GATHER_25, + R__TMP_26, + R__GATHER_27, + R__TMP_28, + R__GATHER_29, + R__GATHER_30, + R__GATHER_31, + R__GATHER_32, + R__TMP_33, + R__LOOP1_34, + R__TMP_35, + R__TMP_36, + R__TMP_37, + R__TMP_38, + R__TMP_39, + R__LOOP0_40, + R__LOOP0_41, + R__LOOP0_42, + R__LOOP1_43, + R__LOOP0_44, + R__LOOP1_45, + R__LOOP1_46, + R__LOOP1_47, + R__LOOP0_48, + R__LOOP1_49, + R__LOOP0_50, + R__LOOP1_51, + R__LOOP0_52, + R__LOOP1_53, + R__LOOP1_54, + R__TMP_55, + R__GATHER_56, + R__LOOP1_57, + R__GATHER_58, + R__LOOP1_59, + R__LOOP0_60, + R__LOOP0_61, + R__LOOP0_62, + R__LOOP1_63, + R__LOOP0_64, + R__LOOP1_65, + R__LOOP1_66, + R__LOOP1_67, + R__LOOP0_68, + R__LOOP1_69, + R__LOOP0_70, + R__LOOP1_71, + R__LOOP0_72, + R__LOOP1_73, + R__LOOP1_74, + R__LOOP1_75, + R__LOOP1_76, + R__TMP_77, + R__GATHER_78, + R__TMP_79, + R__TMP_80, + R__TMP_81, + R__TMP_82, + R__LOOP1_83, + R__TMP_84, + R__TMP_85, + R__GATHER_86, + R__LOOP1_87, + R__LOOP0_88, + R__LOOP0_89, + R__TMP_90, + R__TMP_91, + R__GATHER_92, + R__GATHER_93, + R__GATHER_94, + R__GATHER_95, + R__LOOP0_96, + R__GATHER_97, + R__TMP_98, + R__GATHER_99, + R__GATHER_100, + R__TMP_101, + R__LOOP0_102, + R__TMP_103, + R__TMP_104, + R__TMP_105, + R__TMP_106, + R__LOOP0_107, + R__TMP_108, + R__TMP_109, + R__TMP_110, + R__TMP_111, + R__TMP_112, + R__TMP_113, + R__TMP_114, + R__TMP_115, + R__TMP_116, + R__TMP_117, + R__TMP_118, + R__TMP_119, + R__TMP_120, + R__TMP_121, + R__LOOP1_122, + R__TMP_123, + R__TMP_124, }; enum { - A_START_0, - A_STMT_0, - A_STMT_1, + A_FILE_0, + A_INTERACTIVE_0, + A_EVAL_0, + A_FUNC_TYPE_0, + A_FSTRING_0, + A_TYPE_EXPRESSIONS_0, + A_TYPE_EXPRESSIONS_1, + A_TYPE_EXPRESSIONS_2, + A_TYPE_EXPRESSIONS_3, + A_TYPE_EXPRESSIONS_4, + A_TYPE_EXPRESSIONS_5, + A_TYPE_EXPRESSIONS_6, + A_STATEMENTS_0, + A_STATEMENT_0, + A_STATEMENT_1, + A_STATEMENT_NEWLINE_0, + A_STATEMENT_NEWLINE_1, + A_STATEMENT_NEWLINE_2, + A_STATEMENT_NEWLINE_3, + A_SIMPLE_STMT_0, + A_SIMPLE_STMT_1, + A_SMALL_STMT_0, + A_SMALL_STMT_1, + A_SMALL_STMT_2, + A_SMALL_STMT_3, + A_SMALL_STMT_4, + A_SMALL_STMT_5, + A_SMALL_STMT_6, + A_SMALL_STMT_7, + A_SMALL_STMT_8, + A_SMALL_STMT_9, + A_SMALL_STMT_10, + A_SMALL_STMT_11, + A_SMALL_STMT_12, + A_COMPOUND_STMT_0, + A_COMPOUND_STMT_1, + A_COMPOUND_STMT_2, + A_COMPOUND_STMT_3, + A_COMPOUND_STMT_4, + A_COMPOUND_STMT_5, + A_COMPOUND_STMT_6, + A_ASSIGNMENT_0, + A_ASSIGNMENT_1, + A_ASSIGNMENT_2, + A_ASSIGNMENT_3, + A_ASSIGNMENT_4, + A_AUGASSIGN_0, + A_AUGASSIGN_1, + A_AUGASSIGN_2, + A_AUGASSIGN_3, + A_AUGASSIGN_4, + A_AUGASSIGN_5, + A_AUGASSIGN_6, + A_AUGASSIGN_7, + A_AUGASSIGN_8, + A_AUGASSIGN_9, + A_AUGASSIGN_10, + A_AUGASSIGN_11, + A_AUGASSIGN_12, + A_GLOBAL_STMT_0, + A_NONLOCAL_STMT_0, + A_YIELD_STMT_0, + A_ASSERT_STMT_0, + A_DEL_STMT_0, + A_IMPORT_STMT_0, + A_IMPORT_STMT_1, + A_IMPORT_NAME_0, + A_IMPORT_FROM_0, + A_IMPORT_FROM_1, + A_IMPORT_FROM_TARGETS_0, + A_IMPORT_FROM_TARGETS_1, + A_IMPORT_FROM_TARGETS_2, + A_IMPORT_FROM_TARGETS_3, + A_IMPORT_FROM_AS_NAMES_0, + A_IMPORT_FROM_AS_NAME_0, + A_DOTTED_AS_NAMES_0, + A_DOTTED_AS_NAME_0, + A_DOTTED_NAME_0, + A_DOTTED_NAME_1, A_IF_STMT_0, - A_EXPR_0, - A_EXPR_1, + A_IF_STMT_1, + A_ELIF_STMT_0, + A_ELIF_STMT_1, + A_ELSE_BLOCK_0, + A_WHILE_STMT_0, + A_FOR_STMT_0, + A_FOR_STMT_1, + A_WITH_STMT_0, + A_WITH_STMT_1, + A_WITH_STMT_2, + A_WITH_STMT_3, + A_WITH_ITEM_0, + A_TRY_STMT_0, + A_TRY_STMT_1, + A_EXCEPT_BLOCK_0, + A_EXCEPT_BLOCK_1, + A_FINALLY_BLOCK_0, + A_RETURN_STMT_0, + A_RAISE_STMT_0, + A_RAISE_STMT_1, + A_FUNCTION_DEF_0, + A_FUNCTION_DEF_1, + A_FUNCTION_DEF_RAW_0, + A_FUNCTION_DEF_RAW_1, + A_FUNC_TYPE_COMMENT_0, + A_FUNC_TYPE_COMMENT_1, + A_FUNC_TYPE_COMMENT_2, + A_PARAMS_0, + A_PARAMS_1, + A_PARAMETERS_0, + A_PARAMETERS_1, + A_PARAMETERS_2, + A_PARAMETERS_3, + A_PARAMETERS_4, + A_SLASH_NO_DEFAULT_0, + A_SLASH_NO_DEFAULT_1, + A_SLASH_WITH_DEFAULT_0, + A_SLASH_WITH_DEFAULT_1, + A_STAR_ETC_0, + A_STAR_ETC_1, + A_STAR_ETC_2, + A_STAR_ETC_3, + A_KWDS_0, + A_PARAM_NO_DEFAULT_0, + A_PARAM_NO_DEFAULT_1, + A_PARAM_WITH_DEFAULT_0, + A_PARAM_WITH_DEFAULT_1, + A_PARAM_MAYBE_DEFAULT_0, + A_PARAM_MAYBE_DEFAULT_1, + A_PARAM_0, + A_ANNOTATION_0, + A_DEFAULT_0, + A_DECORATORS_0, + A_CLASS_DEF_0, + A_CLASS_DEF_1, + A_CLASS_DEF_RAW_0, + A_BLOCK_0, + A_BLOCK_1, + A_BLOCK_2, + A_EXPRESSIONS_LIST_0, + A_STAR_EXPRESSIONS_0, + A_STAR_EXPRESSIONS_1, + A_STAR_EXPRESSIONS_2, + A_STAR_EXPRESSION_0, + A_STAR_EXPRESSION_1, + A_STAR_NAMED_EXPRESSIONS_0, + A_STAR_NAMED_EXPRESSION_0, + A_STAR_NAMED_EXPRESSION_1, + A_NAMED_EXPRESSION_0, + A_NAMED_EXPRESSION_1, + A_NAMED_EXPRESSION_2, + A_ANNOTATED_RHS_0, + A_ANNOTATED_RHS_1, + A_EXPRESSIONS_0, + A_EXPRESSIONS_1, + A_EXPRESSIONS_2, + A_EXPRESSION_0, + A_EXPRESSION_1, + A_EXPRESSION_2, + A_LAMBDEF_0, + A_LAMBDA_PARAMETERS_0, + A_LAMBDA_PARAMETERS_1, + A_LAMBDA_PARAMETERS_2, + A_LAMBDA_PARAMETERS_3, + A_LAMBDA_PARAMETERS_4, + A_LAMBDA_SLASH_NO_DEFAULT_0, + A_LAMBDA_SLASH_NO_DEFAULT_1, + A_LAMBDA_SLASH_WITH_DEFAULT_0, + A_LAMBDA_SLASH_WITH_DEFAULT_1, + A_LAMBDA_STAR_ETC_0, + A_LAMBDA_STAR_ETC_1, + A_LAMBDA_STAR_ETC_2, + A_LAMBDA_STAR_ETC_3, + A_LAMBDA_KWDS_0, + A_LAMBDA_PARAM_NO_DEFAULT_0, + A_LAMBDA_PARAM_NO_DEFAULT_1, + A_LAMBDA_PARAM_WITH_DEFAULT_0, + A_LAMBDA_PARAM_WITH_DEFAULT_1, + A_LAMBDA_PARAM_MAYBE_DEFAULT_0, + A_LAMBDA_PARAM_MAYBE_DEFAULT_1, + A_LAMBDA_PARAM_0, + A_DISJUNCTION_0, + A_DISJUNCTION_1, + A_CONJUNCTION_0, + A_CONJUNCTION_1, + A_INVERSION_0, + A_INVERSION_1, + A_COMPARISON_0, + A_COMPARISON_1, + A_COMPARE_OP_BITWISE_OR_PAIR_0, + A_COMPARE_OP_BITWISE_OR_PAIR_1, + A_COMPARE_OP_BITWISE_OR_PAIR_2, + A_COMPARE_OP_BITWISE_OR_PAIR_3, + A_COMPARE_OP_BITWISE_OR_PAIR_4, + A_COMPARE_OP_BITWISE_OR_PAIR_5, + A_COMPARE_OP_BITWISE_OR_PAIR_6, + A_COMPARE_OP_BITWISE_OR_PAIR_7, + A_COMPARE_OP_BITWISE_OR_PAIR_8, + A_COMPARE_OP_BITWISE_OR_PAIR_9, + A_EQ_BITWISE_OR_0, + A_NOTEQ_BITWISE_OR_0, + A_LTE_BITWISE_OR_0, + A_LT_BITWISE_OR_0, + A_GTE_BITWISE_OR_0, + A_GT_BITWISE_OR_0, + A_NOTIN_BITWISE_OR_0, + A_IN_BITWISE_OR_0, + A_ISNOT_BITWISE_OR_0, + A_IS_BITWISE_OR_0, + A_BITWISE_OR_0, + A_BITWISE_OR_1, + A_BITWISE_XOR_0, + A_BITWISE_XOR_1, + A_BITWISE_AND_0, + A_BITWISE_AND_1, + A_SHIFT_EXPR_0, + A_SHIFT_EXPR_1, + A_SHIFT_EXPR_2, + A_SUM_0, + A_SUM_1, + A_SUM_2, A_TERM_0, A_TERM_1, + A_TERM_2, + A_TERM_3, + A_TERM_4, + A_TERM_5, A_FACTOR_0, A_FACTOR_1, A_FACTOR_2, A_FACTOR_3, - A_FACTOR_4, - A__GATHER_2_0, - A__GATHER_2_1, + A_POWER_0, + A_POWER_1, + A_AWAIT_PRIMARY_0, + A_AWAIT_PRIMARY_1, + A_PRIMARY_0, + A_PRIMARY_1, + A_PRIMARY_2, + A_PRIMARY_3, + A_PRIMARY_4, + A_SLICES_0, + A_SLICES_1, + A_SLICE_0, + A_SLICE_1, + A_ATOM_0, + A_ATOM_1, + A_ATOM_2, + A_ATOM_3, + A_ATOM_4, + A_ATOM_5, + A_ATOM_6, + A_ATOM_7, + A_ATOM_8, + A_ATOM_9, + A_ATOM_10, + A_STRINGS_0, + A_LIST_0, + A_LISTCOMP_0, + A_LISTCOMP_1, + A_TUPLE_0, + A_GROUP_0, + A_GENEXP_0, + A_GENEXP_1, + A_SET_0, + A_SETCOMP_0, + A_SETCOMP_1, + A_DICT_0, + A_DICTCOMP_0, + A_DICTCOMP_1, + A_DOUBLE_STARRED_KVPAIRS_0, + A_DOUBLE_STARRED_KVPAIR_0, + A_DOUBLE_STARRED_KVPAIR_1, + A_KVPAIR_0, + A_FOR_IF_CLAUSES_0, + A_FOR_IF_CLAUSE_0, + A_FOR_IF_CLAUSE_1, + A_YIELD_EXPR_0, + A_YIELD_EXPR_1, + A_ARGUMENTS_0, + A_ARGUMENTS_1, + A_ARGS_0, + A_ARGS_1, + A_ARGS_2, + A_KWARGS_0, + A_KWARGS_1, + A_KWARGS_2, + A_STARRED_EXPRESSION_0, + A_KWARG_OR_STARRED_0, + A_KWARG_OR_STARRED_1, + A_KWARG_OR_STARRED_2, + A_KWARG_OR_DOUBLE_STARRED_0, + A_KWARG_OR_DOUBLE_STARRED_1, + A_KWARG_OR_DOUBLE_STARRED_2, + A_STAR_TARGETS_0, + A_STAR_TARGETS_1, + A_STAR_TARGETS_SEQ_0, + A_STAR_TARGET_0, + A_STAR_TARGET_1, + A_STAR_TARGET_2, + A_STAR_TARGET_3, + A_STAR_ATOM_0, + A_STAR_ATOM_1, + A_STAR_ATOM_2, + A_STAR_ATOM_3, + A_SINGLE_TARGET_0, + A_SINGLE_TARGET_1, + A_SINGLE_TARGET_2, + A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_0, + A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_1, + A_DEL_TARGETS_0, + A_DEL_TARGET_0, + A_DEL_TARGET_1, + A_DEL_TARGET_2, + A_DEL_T_ATOM_0, + A_DEL_T_ATOM_1, + A_DEL_T_ATOM_2, + A_DEL_T_ATOM_3, + A_DEL_T_ATOM_4, + A_DEL_TARGET_END_0, + A_DEL_TARGET_END_1, + A_DEL_TARGET_END_2, + A_DEL_TARGET_END_3, + A_DEL_TARGET_END_4, + A_TARGETS_0, + A_TARGET_0, + A_TARGET_1, + A_TARGET_2, + A_T_PRIMARY_0, + A_T_PRIMARY_1, + A_T_PRIMARY_2, + A_T_PRIMARY_3, + A_T_PRIMARY_4, + A_T_LOOKAHEAD_0, + A_T_LOOKAHEAD_1, + A_T_LOOKAHEAD_2, + A_T_ATOM_0, + A_T_ATOM_1, + A_T_ATOM_2, + A_T_ATOM_3, + A_INCORRECT_ARGUMENTS_0, + A_INCORRECT_ARGUMENTS_1, + A_INCORRECT_ARGUMENTS_2, + A_INCORRECT_ARGUMENTS_3, + A_INCORRECT_ARGUMENTS_4, + A_INVALID_KWARG_0, + A_INVALID_NAMED_EXPRESSION_0, + A_INVALID_ASSIGNMENT_0, + A_INVALID_ASSIGNMENT_1, + A_INVALID_ASSIGNMENT_2, + A_INVALID_ASSIGNMENT_3, + A_INVALID_ASSIGNMENT_4, + A_INVALID_ASSIGNMENT_5, + A_INVALID_BLOCK_0, + A_INVALID_COMPREHENSION_0, + A_INVALID_DICT_COMPREHENSION_0, + A_INVALID_PARAMETERS_0, + A_INVALID_STAR_ETC_0, + A_INVALID_STAR_ETC_1, + A_INVALID_LAMBDA_STAR_ETC_0, + A_INVALID_DOUBLE_TYPE_COMMENTS_0, + A_INVALID_DEL_TARGET_0, + A_INVALID_IMPORT_FROM_TARGETS_0, + A__GATHER_3_0, + A__GATHER_3_1, + A__GATHER_4_0, + A__GATHER_4_1, + A__GATHER_5_0, + A__GATHER_5_1, + A__GATHER_6_0, + A__GATHER_6_1, + A__GATHER_8_0, + A__GATHER_8_1, + A__TMP_9_0, + A__TMP_9_1, + A__TMP_10_0, + A__TMP_10_1, + A__TMP_10_2, + A__TMP_11_0, + A__TMP_11_1, + A__TMP_12_0, + A__TMP_12_1, + A__TMP_13_0, + A__TMP_13_1, + A__TMP_14_0, + A__TMP_15_0, + A__TMP_15_1, + A__TMP_16_0, + A__TMP_18_0, + A__TMP_18_1, + A__TMP_19_0, + A__TMP_19_1, + A__GATHER_20_0, + A__GATHER_20_1, + A__GATHER_21_0, + A__GATHER_21_1, + A__TMP_22_0, + A__GATHER_25_0, + A__GATHER_25_1, + A__TMP_26_0, + A__GATHER_27_0, + A__GATHER_27_1, + A__TMP_28_0, + A__GATHER_29_0, + A__GATHER_29_1, + A__GATHER_30_0, + A__GATHER_30_1, + A__GATHER_31_0, + A__GATHER_31_1, + A__GATHER_32_0, + A__GATHER_32_1, + A__TMP_33_0, + A__TMP_35_0, + A__TMP_36_0, + A__TMP_37_0, + A__TMP_38_0, + A__TMP_39_0, + A__TMP_55_0, + A__GATHER_56_0, + A__GATHER_56_1, + A__GATHER_58_0, + A__GATHER_58_1, + A__TMP_77_0, + A__GATHER_78_0, + A__GATHER_78_1, + A__TMP_79_0, + A__TMP_80_0, + A__TMP_80_1, + A__TMP_80_2, + A__TMP_81_0, + A__TMP_81_1, + A__TMP_82_0, + A__TMP_82_1, + A__TMP_82_2, + A__TMP_82_3, + A__TMP_84_0, + A__TMP_85_0, + A__TMP_85_1, + A__GATHER_86_0, + A__GATHER_86_1, + A__TMP_90_0, + A__TMP_91_0, + A__GATHER_92_0, + A__GATHER_92_1, + A__GATHER_93_0, + A__GATHER_93_1, + A__GATHER_94_0, + A__GATHER_94_1, + A__GATHER_95_0, + A__GATHER_95_1, + A__GATHER_97_0, + A__GATHER_97_1, + A__TMP_98_0, + A__GATHER_99_0, + A__GATHER_99_1, + A__GATHER_100_0, + A__GATHER_100_1, + A__TMP_101_0, + A__TMP_101_1, + A__TMP_103_0, + A__TMP_104_0, + A__TMP_104_1, + A__TMP_105_0, + A__TMP_105_1, + A__TMP_106_0, + A__TMP_106_1, + A__TMP_106_2, + A__TMP_108_0, + A__TMP_108_1, + A__TMP_109_0, + A__TMP_109_1, + A__TMP_110_0, + A__TMP_110_1, + A__TMP_111_0, + A__TMP_112_0, + A__TMP_112_1, + A__TMP_113_0, + A__TMP_113_1, + A__TMP_114_0, + A__TMP_115_0, + A__TMP_116_0, + A__TMP_117_0, + A__TMP_118_0, + A__TMP_119_0, + A__TMP_120_0, + A__TMP_121_0, + A__TMP_123_0, + A__TMP_123_1, + A__TMP_124_0, + A__TMP_124_1, }; static Rule all_rules[] = { - {"start", - R_START, + {"file", + R_FILE, + {0, -1}, + { + OP_RULE, R_STATEMENTS, OP_OPTIONAL, OP_TOKEN, ENDMARKER, OP_RETURN, A_FILE_0, + }, + }, + {"interactive", + R_INTERACTIVE, + {0, -1}, + { + OP_RULE, R_STATEMENT_NEWLINE, OP_RETURN, A_INTERACTIVE_0, + }, + }, + {"eval", + R_EVAL, + {0, -1}, + { + OP_RULE, R_EXPRESSIONS, OP_RULE, R__LOOP0_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_EVAL_0, + }, + }, + {"func_type", + R_FUNC_TYPE, + {0, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R_TYPE_EXPRESSIONS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RULE, R__LOOP0_2, OP_TOKEN, ENDMARKER, OP_RETURN, A_FUNC_TYPE_0, + }, + }, + {"fstring", + R_FSTRING, + {0, -1}, + { + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_FSTRING_0, + }, + }, + {"type_expressions", + R_TYPE_EXPRESSIONS, + {0, 16, 26, 36, 48, 54, 60, -1}, + { + OP_RULE, R__GATHER_3, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_0, + OP_RULE, R__GATHER_4, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_1, + OP_RULE, R__GATHER_5, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_2, + OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_3, + OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_4, + OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_5, + OP_RULE, R__GATHER_6, OP_RETURN, A_TYPE_EXPRESSIONS_6, + }, + }, + {"statements", + R_STATEMENTS, {0, -1}, { - OP_RULE, R__LOOP1_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_START_0, + OP_RULE, R__LOOP1_7, OP_RETURN, A_STATEMENTS_0, + }, + }, + {"statement", + R_STATEMENT, + {0, 4, -1}, + { + OP_RULE, R_COMPOUND_STMT, OP_RETURN, A_STATEMENT_0, + OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_STATEMENT_1, + }, + }, + {"statement_newline", + R_STATEMENT_NEWLINE, + {0, 6, 10, 14, -1}, + { + OP_RULE, R_COMPOUND_STMT, OP_TOKEN, NEWLINE, OP_RETURN, A_STATEMENT_NEWLINE_0, + OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_STATEMENT_NEWLINE_1, + OP_TOKEN, NEWLINE, OP_RETURN, A_STATEMENT_NEWLINE_2, + OP_TOKEN, ENDMARKER, OP_RETURN, A_STATEMENT_NEWLINE_3, }, }, - {"stmt", - R_STMT, + {"simple_stmt", + R_SIMPLE_STMT, {0, 10, -1}, { - OP_SAVE_MARK, OP_TOKEN, 500, OP_NEG_LOOKAHEAD, OP_RULE, R_EXPR, OP_TOKEN, NEWLINE, OP_RETURN, A_STMT_0, - OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_RULE, R_IF_STMT, OP_RETURN, A_STMT_1, + OP_RULE, R_SMALL_STMT, OP_SAVE_MARK, OP_TOKEN, SEMI, OP_NEG_LOOKAHEAD, OP_TOKEN, NEWLINE, OP_RETURN, A_SIMPLE_STMT_0, + OP_RULE, R__GATHER_8, OP_TOKEN, SEMI, OP_OPTIONAL, OP_TOKEN, NEWLINE, OP_RETURN, A_SIMPLE_STMT_1, + }, + }, + {"small_stmt", + R_SMALL_STMT, + {0, 4, 8, 16, 24, 32, 36, 44, 52, 60, 64, 68, 76, -1}, + { + OP_RULE, R_ASSIGNMENT, OP_RETURN, A_SMALL_STMT_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_SMALL_STMT_1, + OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_RULE, R_RETURN_STMT, OP_RETURN, A_SMALL_STMT_2, + OP_SAVE_MARK, OP_RULE, R__TMP_9, OP_POS_LOOKAHEAD, OP_RULE, R_IMPORT_STMT, OP_RETURN, A_SMALL_STMT_3, + OP_SAVE_MARK, OP_TOKEN, 501, OP_POS_LOOKAHEAD, OP_RULE, R_RAISE_STMT, OP_RETURN, A_SMALL_STMT_4, + OP_TOKEN, 502, OP_RETURN, A_SMALL_STMT_5, + OP_SAVE_MARK, OP_TOKEN, 503, OP_POS_LOOKAHEAD, OP_RULE, R_DEL_STMT, OP_RETURN, A_SMALL_STMT_6, + OP_SAVE_MARK, OP_TOKEN, 504, OP_POS_LOOKAHEAD, OP_RULE, R_YIELD_STMT, OP_RETURN, A_SMALL_STMT_7, + OP_SAVE_MARK, OP_TOKEN, 505, OP_POS_LOOKAHEAD, OP_RULE, R_ASSERT_STMT, OP_RETURN, A_SMALL_STMT_8, + OP_TOKEN, 506, OP_RETURN, A_SMALL_STMT_9, + OP_TOKEN, 507, OP_RETURN, A_SMALL_STMT_10, + OP_SAVE_MARK, OP_TOKEN, 508, OP_POS_LOOKAHEAD, OP_RULE, R_GLOBAL_STMT, OP_RETURN, A_SMALL_STMT_11, + OP_SAVE_MARK, OP_TOKEN, 509, OP_POS_LOOKAHEAD, OP_RULE, R_NONLOCAL_STMT, OP_RETURN, A_SMALL_STMT_12, + }, + }, + {"compound_stmt", + R_COMPOUND_STMT, + {0, 8, 16, 24, 32, 40, 48, -1}, + { + OP_SAVE_MARK, OP_RULE, R__TMP_10, OP_POS_LOOKAHEAD, OP_RULE, R_FUNCTION_DEF, OP_RETURN, A_COMPOUND_STMT_0, + OP_SAVE_MARK, OP_TOKEN, 510, OP_POS_LOOKAHEAD, OP_RULE, R_IF_STMT, OP_RETURN, A_COMPOUND_STMT_1, + OP_SAVE_MARK, OP_RULE, R__TMP_11, OP_POS_LOOKAHEAD, OP_RULE, R_CLASS_DEF, OP_RETURN, A_COMPOUND_STMT_2, + OP_SAVE_MARK, OP_RULE, R__TMP_12, OP_POS_LOOKAHEAD, OP_RULE, R_WITH_STMT, OP_RETURN, A_COMPOUND_STMT_3, + OP_SAVE_MARK, OP_RULE, R__TMP_13, OP_POS_LOOKAHEAD, OP_RULE, R_FOR_STMT, OP_RETURN, A_COMPOUND_STMT_4, + OP_SAVE_MARK, OP_TOKEN, 511, OP_POS_LOOKAHEAD, OP_RULE, R_TRY_STMT, OP_RETURN, A_COMPOUND_STMT_5, + OP_SAVE_MARK, OP_TOKEN, 512, OP_POS_LOOKAHEAD, OP_RULE, R_WHILE_STMT, OP_RETURN, A_COMPOUND_STMT_6, + }, + }, + {"assignment", + R_ASSIGNMENT, + {0, 10, 21, 30, 38, -1}, + { + OP_NAME, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_14, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_0, + OP_RULE, R__TMP_15, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_16, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_1, + OP_RULE, R__LOOP1_17, OP_RULE, R__TMP_18, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_2, + OP_RULE, R_SINGLE_TARGET, OP_RULE, R_AUGASSIGN, OP_RULE, R__TMP_19, OP_RETURN, A_ASSIGNMENT_3, + OP_RULE, R_INVALID_ASSIGNMENT, OP_RETURN, A_ASSIGNMENT_4, + }, + }, + {"augassign", + R_AUGASSIGN, + {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, -1}, + { + OP_TOKEN, PLUSEQUAL, OP_RETURN, A_AUGASSIGN_0, + OP_TOKEN, MINEQUAL, OP_RETURN, A_AUGASSIGN_1, + OP_TOKEN, STAREQUAL, OP_RETURN, A_AUGASSIGN_2, + OP_TOKEN, ATEQUAL, OP_RETURN, A_AUGASSIGN_3, + OP_TOKEN, SLASHEQUAL, OP_RETURN, A_AUGASSIGN_4, + OP_TOKEN, PERCENTEQUAL, OP_RETURN, A_AUGASSIGN_5, + OP_TOKEN, AMPEREQUAL, OP_RETURN, A_AUGASSIGN_6, + OP_TOKEN, VBAREQUAL, OP_RETURN, A_AUGASSIGN_7, + OP_TOKEN, CIRCUMFLEXEQUAL, OP_RETURN, A_AUGASSIGN_8, + OP_TOKEN, LEFTSHIFTEQUAL, OP_RETURN, A_AUGASSIGN_9, + OP_TOKEN, RIGHTSHIFTEQUAL, OP_RETURN, A_AUGASSIGN_10, + OP_TOKEN, DOUBLESTAREQUAL, OP_RETURN, A_AUGASSIGN_11, + OP_TOKEN, DOUBLESLASHEQUAL, OP_RETURN, A_AUGASSIGN_12, + }, + }, + {"global_stmt", + R_GLOBAL_STMT, + {0, -1}, + { + OP_TOKEN, 508, OP_RULE, R__GATHER_20, OP_RETURN, A_GLOBAL_STMT_0, + }, + }, + {"nonlocal_stmt", + R_NONLOCAL_STMT, + {0, -1}, + { + OP_TOKEN, 509, OP_RULE, R__GATHER_21, OP_RETURN, A_NONLOCAL_STMT_0, + }, + }, + {"yield_stmt", + R_YIELD_STMT, + {0, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A_YIELD_STMT_0, + }, + }, + {"assert_stmt", + R_ASSERT_STMT, + {0, -1}, + { + OP_TOKEN, 505, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_22, OP_OPTIONAL, OP_RETURN, A_ASSERT_STMT_0, + }, + }, + {"del_stmt", + R_DEL_STMT, + {0, -1}, + { + OP_TOKEN, 503, OP_RULE, R_DEL_TARGETS, OP_RETURN, A_DEL_STMT_0, + }, + }, + {"import_stmt", + R_IMPORT_STMT, + {0, 4, -1}, + { + OP_RULE, R_IMPORT_NAME, OP_RETURN, A_IMPORT_STMT_0, + OP_RULE, R_IMPORT_FROM, OP_RETURN, A_IMPORT_STMT_1, + }, + }, + {"import_name", + R_IMPORT_NAME, + {0, -1}, + { + OP_TOKEN, 513, OP_RULE, R_DOTTED_AS_NAMES, OP_RETURN, A_IMPORT_NAME_0, + }, + }, + {"import_from", + R_IMPORT_FROM, + {0, 12, -1}, + { + OP_TOKEN, 514, OP_RULE, R__LOOP0_23, OP_RULE, R_DOTTED_NAME, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_0, + OP_TOKEN, 514, OP_RULE, R__LOOP1_24, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_1, + }, + }, + {"import_from_targets", + R_IMPORT_FROM_TARGETS, + {0, 11, 19, 23, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_IMPORT_FROM_TARGETS_0, + OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_IMPORT_FROM_TARGETS_1, + OP_TOKEN, STAR, OP_RETURN, A_IMPORT_FROM_TARGETS_2, + OP_RULE, R_INVALID_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_TARGETS_3, + }, + }, + {"import_from_as_names", + R_IMPORT_FROM_AS_NAMES, + {0, -1}, + { + OP_RULE, R__GATHER_25, OP_RETURN, A_IMPORT_FROM_AS_NAMES_0, + }, + }, + {"import_from_as_name", + R_IMPORT_FROM_AS_NAME, + {0, -1}, + { + OP_NAME, OP_RULE, R__TMP_26, OP_OPTIONAL, OP_RETURN, A_IMPORT_FROM_AS_NAME_0, + }, + }, + {"dotted_as_names", + R_DOTTED_AS_NAMES, + {0, -1}, + { + OP_RULE, R__GATHER_27, OP_RETURN, A_DOTTED_AS_NAMES_0, + }, + }, + {"dotted_as_name", + R_DOTTED_AS_NAME, + {0, -1}, + { + OP_RULE, R_DOTTED_NAME, OP_RULE, R__TMP_28, OP_OPTIONAL, OP_RETURN, A_DOTTED_AS_NAME_0, + }, + }, + {"dotted_name", + R_DOTTED_NAME, + {0, 8, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_DOTTED_NAME, OP_TOKEN, DOT, OP_NAME, OP_RETURN_LEFT_REC, A_DOTTED_NAME_0, + OP_NAME, OP_RETURN_LEFT_REC, A_DOTTED_NAME_1, }, }, {"if_stmt", R_IF_STMT, + {0, 12, -1}, + { + OP_TOKEN, 510, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELIF_STMT, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 510, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_IF_STMT_1, + }, + }, + {"elif_stmt", + R_ELIF_STMT, + {0, 12, -1}, + { + OP_TOKEN, 515, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELIF_STMT, OP_RETURN, A_ELIF_STMT_0, + OP_TOKEN, 515, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_ELIF_STMT_1, + }, + }, + {"else_block", + R_ELSE_BLOCK, {0, -1}, { - OP_TOKEN, 500, OP_CUT, OP_NAME, OP_TOKEN, COLON, OP_RULE, R_STMT, OP_RETURN, A_IF_STMT_0, + OP_TOKEN, 516, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_ELSE_BLOCK_0, }, }, - {"expr", - R_EXPR, + {"while_stmt", + R_WHILE_STMT, + {0, -1}, + { + OP_TOKEN, 512, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_WHILE_STMT_0, + }, + }, + {"for_stmt", + R_FOR_STMT, + {0, 20, -1}, + { + OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_FOR_STMT_0, + OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_FOR_STMT_1, + }, + }, + {"with_stmt", + R_WITH_STMT, + {0, 17, 30, 49, -1}, + { + OP_TOKEN, 519, OP_TOKEN, LPAR, OP_RULE, R__GATHER_29, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_0, + OP_TOKEN, 519, OP_RULE, R__GATHER_30, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_1, + OP_TOKEN, ASYNC, OP_TOKEN, 519, OP_TOKEN, LPAR, OP_RULE, R__GATHER_31, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_2, + OP_TOKEN, ASYNC, OP_TOKEN, 519, OP_RULE, R__GATHER_32, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_3, + }, + }, + {"with_item", + R_WITH_ITEM, + {0, -1}, + { + OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_33, OP_OPTIONAL, OP_RETURN, A_WITH_ITEM_0, + }, + }, + {"try_stmt", + R_TRY_STMT, + {0, 10, -1}, + { + OP_TOKEN, 511, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_FINALLY_BLOCK, OP_RETURN, A_TRY_STMT_0, + OP_TOKEN, 511, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R__LOOP1_34, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RULE, R_FINALLY_BLOCK, OP_OPTIONAL, OP_RETURN, A_TRY_STMT_1, + }, + }, + {"except_block", + R_EXCEPT_BLOCK, + {0, 13, -1}, + { + OP_TOKEN, 520, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_35, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_0, + OP_TOKEN, 520, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_1, + }, + }, + {"finally_block", + R_FINALLY_BLOCK, + {0, -1}, + { + OP_TOKEN, 521, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_FINALLY_BLOCK_0, + }, + }, + {"return_stmt", + R_RETURN_STMT, + {0, -1}, + { + OP_TOKEN, 500, OP_RULE, R_STAR_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A_RETURN_STMT_0, + }, + }, + {"raise_stmt", + R_RAISE_STMT, {0, 9, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_EXPR, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_0, - OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_EXPR_1, + OP_TOKEN, 501, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_36, OP_OPTIONAL, OP_RETURN, A_RAISE_STMT_0, + OP_TOKEN, 501, OP_RETURN, A_RAISE_STMT_1, }, }, - {"term", - R_TERM, + {"function_def", + R_FUNCTION_DEF, + {0, 6, -1}, + { + OP_RULE, R_DECORATORS, OP_RULE, R_FUNCTION_DEF_RAW, OP_RETURN, A_FUNCTION_DEF_0, + OP_RULE, R_FUNCTION_DEF_RAW, OP_RETURN, A_FUNCTION_DEF_1, + }, + }, + {"function_def_raw", + R_FUNCTION_DEF_RAW, + {0, 22, -1}, + { + OP_TOKEN, 522, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RULE, R__TMP_37, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_FUNCTION_DEF_RAW_0, + OP_TOKEN, ASYNC, OP_TOKEN, 522, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RULE, R__TMP_38, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_FUNCTION_DEF_RAW_1, + }, + }, + {"func_type_comment", + R_FUNC_TYPE_COMMENT, + {0, 10, 14, -1}, + { + OP_TOKEN, NEWLINE, OP_TOKEN, TYPE_COMMENT, OP_SAVE_MARK, OP_RULE, R__TMP_39, OP_POS_LOOKAHEAD, OP_RETURN, A_FUNC_TYPE_COMMENT_0, + OP_RULE, R_INVALID_DOUBLE_TYPE_COMMENTS, OP_RETURN, A_FUNC_TYPE_COMMENT_1, + OP_TOKEN, TYPE_COMMENT, OP_RETURN, A_FUNC_TYPE_COMMENT_2, + }, + }, + {"params", + R_PARAMS, + {0, 4, -1}, + { + OP_RULE, R_INVALID_PARAMETERS, OP_RETURN, A_PARAMS_0, + OP_RULE, R_PARAMETERS, OP_RETURN, A_PARAMS_1, + }, + }, + {"parameters", + R_PARAMETERS, + {0, 11, 20, 29, 36, -1}, + { + OP_RULE, R_SLASH_NO_DEFAULT, OP_RULE, R__LOOP0_40, OP_RULE, R__LOOP0_41, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_0, + OP_RULE, R_SLASH_WITH_DEFAULT, OP_RULE, R__LOOP0_42, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_1, + OP_RULE, R__LOOP1_43, OP_RULE, R__LOOP0_44, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_2, + OP_RULE, R__LOOP1_45, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_3, + OP_RULE, R_STAR_ETC, OP_RETURN, A_PARAMETERS_4, + }, + }, + {"slash_no_default", + R_SLASH_NO_DEFAULT, + {0, 8, -1}, + { + OP_RULE, R__LOOP1_46, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_NO_DEFAULT_0, + OP_RULE, R__LOOP1_47, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_SLASH_NO_DEFAULT_1, + }, + }, + {"slash_with_default", + R_SLASH_WITH_DEFAULT, {0, 10, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_OPTIONAL, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, - OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, + OP_RULE, R__LOOP0_48, OP_RULE, R__LOOP1_49, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_WITH_DEFAULT_0, + OP_RULE, R__LOOP0_50, OP_RULE, R__LOOP1_51, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_SLASH_WITH_DEFAULT_1, }, }, - {"factor", - R_FACTOR, - {0, 8, 16, 19, 23, -1}, + {"star_etc", + R_STAR_ETC, + {0, 11, 22, 26, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_EXPR, OP_TOKEN, RPAR, OP_RETURN, A_FACTOR_0, - OP_TOKEN, LSQB, OP_RULE, R__GATHER_2, OP_TOKEN, RSQB, OP_RETURN, A_FACTOR_1, - OP_NUMBER, OP_RETURN, A_FACTOR_2, - OP_SOFT_KEYWORD, SK___PEG_PARSER__, OP_RETURN, A_FACTOR_3, - OP_NAME, OP_RETURN, A_FACTOR_4, + OP_TOKEN, STAR, OP_RULE, R_PARAM_NO_DEFAULT, OP_RULE, R__LOOP0_52, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_0, + OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_RULE, R__LOOP1_53, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_1, + OP_RULE, R_KWDS, OP_RETURN, A_STAR_ETC_2, + OP_RULE, R_INVALID_STAR_ETC, OP_RETURN, A_STAR_ETC_3, }, }, - {"root", - R_ROOT, - {0, 3, -1}, + {"kwds", + R_KWDS, + {0, -1}, { - OP_RULE, R_START, OP_SUCCESS, - OP_FAILURE, + OP_TOKEN, DOUBLESTAR, OP_RULE, R_PARAM_NO_DEFAULT, OP_RETURN, A_KWDS_0, }, }, - {"_loop1_1", - R__LOOP1_1, - {0, 3, -1}, + {"param_no_default", + R_PARAM_NO_DEFAULT, + {0, 9, -1}, { - OP_RULE, R_STMT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + OP_RULE, R_PARAM, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_NO_DEFAULT_0, + OP_RULE, R_PARAM, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_NO_DEFAULT_1, }, }, - {"_gather_2", - R__GATHER_2, - {0, 5, -1}, + {"param_with_default", + R_PARAM_WITH_DEFAULT, + {0, 11, -1}, { - OP_RULE, R_EXPR, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_EXPR, OP_LOOP_COLLECT_DELIMITED, + OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_WITH_DEFAULT_0, + OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_WITH_DEFAULT_1, }, }, -}; - -static void * -call_action(Parser *p, Frame *_f, int _iaction) -{ - assert(p->mark > 0); - Token *_t = p->tokens[_f->mark]; - int _start_lineno = _t->lineno; - int _start_col_offset = _t->col_offset; - _t = p->tokens[p->mark - 1]; - int _end_lineno = _t->end_lineno; - int _end_col_offset = _t->end_col_offset; - - switch (_iaction) { - case A_START_0: - return _PyPegen_make_module ( p , _f->vals[0] ); - case A_STMT_0: - return _Py_Expr ( _f->vals[0] , EXTRA ); - case A_STMT_1: - case A_EXPR_1: - case A_TERM_1: - case A_FACTOR_2: - case A_FACTOR_4: - case A__GATHER_2_0: - case A__GATHER_2_1: - return _f->vals[0]; - case A_IF_STMT_0: - return _Py_If ( _f->vals[1] , CHECK ( _PyPegen_singleton_seq ( p , _f->vals[3] ) ) , NULL , EXTRA ); - case A_EXPR_0: - return _Py_BinOp ( _f->vals[0] , Add , _f->vals[2] , EXTRA ); - case A_TERM_0: - return _Py_BinOp ( _f->vals[0] , Mult , _f->vals[2] , EXTRA ); - case A_FACTOR_0: - return _f->vals[1]; - case A_FACTOR_1: - return _Py_List ( _f->vals[1] , Load , EXTRA ); - case A_FACTOR_3: - return RAISE_SYNTAX_ERROR ( "You found it!" ); + {"param_maybe_default", + R_PARAM_MAYBE_DEFAULT, + {0, 12, -1}, + { + OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_MAYBE_DEFAULT_0, + OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_MAYBE_DEFAULT_1, + }, + }, + {"param", + R_PARAM, + {0, -1}, + { + OP_NAME, OP_RULE, R_ANNOTATION, OP_OPTIONAL, OP_RETURN, A_PARAM_0, + }, + }, + {"annotation", + R_ANNOTATION, + {0, -1}, + { + OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_ANNOTATION_0, + }, + }, + {"default", + R_DEFAULT, + {0, -1}, + { + OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_DEFAULT_0, + }, + }, + {"decorators", + R_DECORATORS, + {0, -1}, + { + OP_RULE, R__LOOP1_54, OP_RETURN, A_DECORATORS_0, + }, + }, + {"class_def", + R_CLASS_DEF, + {0, 6, -1}, + { + OP_RULE, R_DECORATORS, OP_RULE, R_CLASS_DEF_RAW, OP_RETURN, A_CLASS_DEF_0, + OP_RULE, R_CLASS_DEF_RAW, OP_RETURN, A_CLASS_DEF_1, + }, + }, + {"class_def_raw", + R_CLASS_DEF_RAW, + {0, -1}, + { + OP_TOKEN, 523, OP_NAME, OP_RULE, R__TMP_55, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_CLASS_DEF_RAW_0, + }, + }, + {"block", + R_BLOCK, + {0, 10, 14, -1}, + { + OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RULE, R_STATEMENTS, OP_TOKEN, DEDENT, OP_RETURN, A_BLOCK_0, + OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_BLOCK_1, + OP_RULE, R_INVALID_BLOCK, OP_RETURN, A_BLOCK_2, + }, + }, + {"expressions_list", + R_EXPRESSIONS_LIST, + {0, -1}, + { + OP_RULE, R__GATHER_56, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_LIST_0, + }, + }, + {"star_expressions", + R_STAR_EXPRESSIONS, + {0, 9, 15, -1}, + { + OP_RULE, R_STAR_EXPRESSION, OP_RULE, R__LOOP1_57, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_EXPRESSIONS_0, + OP_RULE, R_STAR_EXPRESSION, OP_TOKEN, COMMA, OP_RETURN, A_STAR_EXPRESSIONS_1, + OP_RULE, R_STAR_EXPRESSION, OP_RETURN, A_STAR_EXPRESSIONS_2, + }, + }, + {"star_expression", + R_STAR_EXPRESSION, + {0, 6, -1}, + { + OP_TOKEN, STAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_STAR_EXPRESSION_0, + OP_RULE, R_EXPRESSION, OP_RETURN, A_STAR_EXPRESSION_1, + }, + }, + {"star_named_expressions", + R_STAR_NAMED_EXPRESSIONS, + {0, -1}, + { + OP_RULE, R__GATHER_58, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_NAMED_EXPRESSIONS_0, + }, + }, + {"star_named_expression", + R_STAR_NAMED_EXPRESSION, + {0, 6, -1}, + { + OP_TOKEN, STAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_STAR_NAMED_EXPRESSION_0, + OP_RULE, R_NAMED_EXPRESSION, OP_RETURN, A_STAR_NAMED_EXPRESSION_1, + }, + }, + {"named_expression", + R_NAMED_EXPRESSION, + {0, 7, 15, -1}, + { + OP_NAME, OP_TOKEN, COLONEQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_NAMED_EXPRESSION_0, + OP_RULE, R_EXPRESSION, OP_SAVE_MARK, OP_TOKEN, COLONEQUAL, OP_NEG_LOOKAHEAD, OP_RETURN, A_NAMED_EXPRESSION_1, + OP_RULE, R_INVALID_NAMED_EXPRESSION, OP_RETURN, A_NAMED_EXPRESSION_2, + }, + }, + {"annotated_rhs", + R_ANNOTATED_RHS, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A_ANNOTATED_RHS_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_ANNOTATED_RHS_1, + }, + }, + {"expressions", + R_EXPRESSIONS, + {0, 9, 15, -1}, + { + OP_RULE, R_EXPRESSION, OP_RULE, R__LOOP1_59, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_0, + OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_RETURN, A_EXPRESSIONS_1, + OP_RULE, R_EXPRESSION, OP_RETURN, A_EXPRESSIONS_2, + }, + }, + {"expression", + R_EXPRESSION, + {0, 12, 16, -1}, + { + OP_RULE, R_DISJUNCTION, OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_TOKEN, 516, OP_RULE, R_EXPRESSION, OP_RETURN, A_EXPRESSION_0, + OP_RULE, R_DISJUNCTION, OP_RETURN, A_EXPRESSION_1, + OP_RULE, R_LAMBDEF, OP_RETURN, A_EXPRESSION_2, + }, + }, + {"lambdef", + R_LAMBDEF, + {0, -1}, + { + OP_TOKEN, 524, OP_RULE, R_LAMBDA_PARAMETERS, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_LAMBDEF_0, + }, + }, + {"lambda_parameters", + R_LAMBDA_PARAMETERS, + {0, 11, 20, 29, 36, -1}, + { + OP_RULE, R_LAMBDA_SLASH_NO_DEFAULT, OP_RULE, R__LOOP0_60, OP_RULE, R__LOOP0_61, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_0, + OP_RULE, R_LAMBDA_SLASH_WITH_DEFAULT, OP_RULE, R__LOOP0_62, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_1, + OP_RULE, R__LOOP1_63, OP_RULE, R__LOOP0_64, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_2, + OP_RULE, R__LOOP1_65, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_3, + OP_RULE, R_LAMBDA_STAR_ETC, OP_RETURN, A_LAMBDA_PARAMETERS_4, + }, + }, + {"lambda_slash_no_default", + R_LAMBDA_SLASH_NO_DEFAULT, + {0, 8, -1}, + { + OP_RULE, R__LOOP1_66, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_0, + OP_RULE, R__LOOP1_67, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_1, + }, + }, + {"lambda_slash_with_default", + R_LAMBDA_SLASH_WITH_DEFAULT, + {0, 10, -1}, + { + OP_RULE, R__LOOP0_68, OP_RULE, R__LOOP1_69, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_0, + OP_RULE, R__LOOP0_70, OP_RULE, R__LOOP1_71, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_1, + }, + }, + {"lambda_star_etc", + R_LAMBDA_STAR_ETC, + {0, 11, 22, 26, -1}, + { + OP_TOKEN, STAR, OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_RULE, R__LOOP0_72, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_0, + OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_RULE, R__LOOP1_73, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_1, + OP_RULE, R_LAMBDA_KWDS, OP_RETURN, A_LAMBDA_STAR_ETC_2, + OP_RULE, R_INVALID_LAMBDA_STAR_ETC, OP_RETURN, A_LAMBDA_STAR_ETC_3, + }, + }, + {"lambda_kwds", + R_LAMBDA_KWDS, + {0, -1}, + { + OP_TOKEN, DOUBLESTAR, OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_RETURN, A_LAMBDA_KWDS_0, + }, + }, + {"lambda_param_no_default", + R_LAMBDA_PARAM_NO_DEFAULT, + {0, 6, -1}, + { + OP_RULE, R_LAMBDA_PARAM, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_0, + OP_RULE, R_LAMBDA_PARAM, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_1, + }, + }, + {"lambda_param_with_default", + R_LAMBDA_PARAM_WITH_DEFAULT, + {0, 8, -1}, + { + OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_0, + OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_1, + }, + }, + {"lambda_param_maybe_default", + R_LAMBDA_PARAM_MAYBE_DEFAULT, + {0, 9, -1}, + { + OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_0, + OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_1, + }, + }, + {"lambda_param", + R_LAMBDA_PARAM, + {0, -1}, + { + OP_NAME, OP_RETURN, A_LAMBDA_PARAM_0, + }, + }, + {"disjunction", + R_DISJUNCTION, + {0, 6, -1}, + { + OP_RULE, R_CONJUNCTION, OP_RULE, R__LOOP1_74, OP_RETURN, A_DISJUNCTION_0, + OP_RULE, R_CONJUNCTION, OP_RETURN, A_DISJUNCTION_1, + }, + }, + {"conjunction", + R_CONJUNCTION, + {0, 6, -1}, + { + OP_RULE, R_INVERSION, OP_RULE, R__LOOP1_75, OP_RETURN, A_CONJUNCTION_0, + OP_RULE, R_INVERSION, OP_RETURN, A_CONJUNCTION_1, + }, + }, + {"inversion", + R_INVERSION, + {0, 6, -1}, + { + OP_TOKEN, 525, OP_RULE, R_INVERSION, OP_RETURN, A_INVERSION_0, + OP_RULE, R_COMPARISON, OP_RETURN, A_INVERSION_1, + }, + }, + {"comparison", + R_COMPARISON, + {0, 6, -1}, + { + OP_RULE, R_BITWISE_OR, OP_RULE, R__LOOP1_76, OP_RETURN, A_COMPARISON_0, + OP_RULE, R_BITWISE_OR, OP_RETURN, A_COMPARISON_1, + }, + }, + {"compare_op_bitwise_or_pair", + R_COMPARE_OP_BITWISE_OR_PAIR, + {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, -1}, + { + OP_RULE, R_EQ_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_0, + OP_RULE, R_NOTEQ_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_1, + OP_RULE, R_LTE_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_2, + OP_RULE, R_LT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_3, + OP_RULE, R_GTE_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_4, + OP_RULE, R_GT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_5, + OP_RULE, R_NOTIN_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_6, + OP_RULE, R_IN_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_7, + OP_RULE, R_ISNOT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_8, + OP_RULE, R_IS_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_9, + }, + }, + {"eq_bitwise_or", + R_EQ_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, EQEQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_EQ_BITWISE_OR_0, + }, + }, + {"noteq_bitwise_or", + R_NOTEQ_BITWISE_OR, + {0, -1}, + { + OP_RULE, R__TMP_77, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTEQ_BITWISE_OR_0, + }, + }, + {"lte_bitwise_or", + R_LTE_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, LESSEQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_LTE_BITWISE_OR_0, + }, + }, + {"lt_bitwise_or", + R_LT_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, LESS, OP_RULE, R_BITWISE_OR, OP_RETURN, A_LT_BITWISE_OR_0, + }, + }, + {"gte_bitwise_or", + R_GTE_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, GREATEREQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_GTE_BITWISE_OR_0, + }, + }, + {"gt_bitwise_or", + R_GT_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, GREATER, OP_RULE, R_BITWISE_OR, OP_RETURN, A_GT_BITWISE_OR_0, + }, + }, + {"notin_bitwise_or", + R_NOTIN_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, 525, OP_TOKEN, 518, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTIN_BITWISE_OR_0, + }, + }, + {"in_bitwise_or", + R_IN_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, 518, OP_RULE, R_BITWISE_OR, OP_RETURN, A_IN_BITWISE_OR_0, + }, + }, + {"isnot_bitwise_or", + R_ISNOT_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, 526, OP_TOKEN, 525, OP_RULE, R_BITWISE_OR, OP_RETURN, A_ISNOT_BITWISE_OR_0, + }, + }, + {"is_bitwise_or", + R_IS_BITWISE_OR, + {0, -1}, + { + OP_TOKEN, 526, OP_RULE, R_BITWISE_OR, OP_RETURN, A_IS_BITWISE_OR_0, + }, + }, + {"bitwise_or", + R_BITWISE_OR, + {0, 9, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_OR, OP_TOKEN, VBAR, OP_RULE, R_BITWISE_XOR, OP_RETURN_LEFT_REC, A_BITWISE_OR_0, + OP_RULE, R_BITWISE_XOR, OP_RETURN_LEFT_REC, A_BITWISE_OR_1, + }, + }, + {"bitwise_xor", + R_BITWISE_XOR, + {0, 9, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_XOR, OP_TOKEN, CIRCUMFLEX, OP_RULE, R_BITWISE_AND, OP_RETURN_LEFT_REC, A_BITWISE_XOR_0, + OP_RULE, R_BITWISE_AND, OP_RETURN_LEFT_REC, A_BITWISE_XOR_1, + }, + }, + {"bitwise_and", + R_BITWISE_AND, + {0, 9, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_AND, OP_TOKEN, AMPER, OP_RULE, R_SHIFT_EXPR, OP_RETURN_LEFT_REC, A_BITWISE_AND_0, + OP_RULE, R_SHIFT_EXPR, OP_RETURN_LEFT_REC, A_BITWISE_AND_1, + }, + }, + {"shift_expr", + R_SHIFT_EXPR, + {0, 9, 17, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_SHIFT_EXPR, OP_TOKEN, LEFTSHIFT, OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_0, + OP_RULE, R_SHIFT_EXPR, OP_TOKEN, RIGHTSHIFT, OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_1, + OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_2, + }, + }, + {"sum", + R_SUM, + {0, 9, 17, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_SUM, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_0, + OP_RULE, R_SUM, OP_TOKEN, MINUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_1, + OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_2, + }, + }, + {"term", + R_TERM, + {0, 9, 17, 25, 33, 41, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, + OP_RULE, R_TERM, OP_TOKEN, SLASH, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, + OP_RULE, R_TERM, OP_TOKEN, DOUBLESLASH, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_2, + OP_RULE, R_TERM, OP_TOKEN, PERCENT, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_3, + OP_RULE, R_TERM, OP_TOKEN, AT, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_4, + OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_5, + }, + }, + {"factor", + R_FACTOR, + {0, 6, 12, 18, -1}, + { + OP_TOKEN, PLUS, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_0, + OP_TOKEN, MINUS, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_1, + OP_TOKEN, TILDE, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_2, + OP_RULE, R_POWER, OP_RETURN, A_FACTOR_3, + }, + }, + {"power", + R_POWER, + {0, 8, -1}, + { + OP_RULE, R_AWAIT_PRIMARY, OP_TOKEN, DOUBLESTAR, OP_RULE, R_FACTOR, OP_RETURN, A_POWER_0, + OP_RULE, R_AWAIT_PRIMARY, OP_RETURN, A_POWER_1, + }, + }, + {"await_primary", + R_AWAIT_PRIMARY, + {0, 6, -1}, + { + OP_TOKEN, AWAIT, OP_RULE, R_PRIMARY, OP_RETURN, A_AWAIT_PRIMARY_0, + OP_RULE, R_PRIMARY, OP_RETURN, A_AWAIT_PRIMARY_1, + }, + }, + {"primary", + R_PRIMARY, + {0, 8, 14, 25, 35, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_RETURN_LEFT_REC, A_PRIMARY_0, + OP_RULE, R_PRIMARY, OP_RULE, R_GENEXP, OP_RETURN_LEFT_REC, A_PRIMARY_1, + OP_RULE, R_PRIMARY, OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN_LEFT_REC, A_PRIMARY_2, + OP_RULE, R_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_RETURN_LEFT_REC, A_PRIMARY_3, + OP_RULE, R_ATOM, OP_RETURN_LEFT_REC, A_PRIMARY_4, + }, + }, + {"slices", + R_SLICES, + {0, 8, -1}, + { + OP_RULE, R_SLICE, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_SLICES_0, + OP_RULE, R__GATHER_78, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_SLICES_1, + }, + }, + {"slice", + R_SLICE, + {0, 13, -1}, + { + OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_RULE, R__TMP_79, OP_OPTIONAL, OP_RETURN, A_SLICE_0, + OP_RULE, R_EXPRESSION, OP_RETURN, A_SLICE_1, + }, + }, + {"atom", + R_ATOM, + {0, 3, 7, 11, 15, 19, 26, 29, 37, 45, 53, -1}, + { + OP_NAME, OP_RETURN, A_ATOM_0, + OP_TOKEN, 527, OP_RETURN, A_ATOM_1, + OP_TOKEN, 528, OP_RETURN, A_ATOM_2, + OP_TOKEN, 529, OP_RETURN, A_ATOM_3, + OP_TOKEN, 530, OP_RETURN, A_ATOM_4, + OP_SAVE_MARK, OP_STRING, OP_POS_LOOKAHEAD, OP_RULE, R_STRINGS, OP_RETURN, A_ATOM_5, + OP_NUMBER, OP_RETURN, A_ATOM_6, + OP_SAVE_MARK, OP_TOKEN, LPAR, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_80, OP_RETURN, A_ATOM_7, + OP_SAVE_MARK, OP_TOKEN, LSQB, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_81, OP_RETURN, A_ATOM_8, + OP_SAVE_MARK, OP_TOKEN, LBRACE, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_82, OP_RETURN, A_ATOM_9, + OP_TOKEN, ELLIPSIS, OP_RETURN, A_ATOM_10, + }, + }, + {"strings", + R_STRINGS, + {0, -1}, + { + OP_RULE, R__LOOP1_83, OP_RETURN, A_STRINGS_0, + }, + }, + {"list", + R_LIST, + {0, -1}, + { + OP_TOKEN, LSQB, OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_LIST_0, + }, + }, + {"listcomp", + R_LISTCOMP, + {0, 10, -1}, + { + OP_TOKEN, LSQB, OP_RULE, R_NAMED_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RSQB, OP_RETURN, A_LISTCOMP_0, + OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_LISTCOMP_1, + }, + }, + {"tuple", + R_TUPLE, + {0, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R__TMP_84, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_TUPLE_0, + }, + }, + {"group", + R_GROUP, + {0, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R__TMP_85, OP_TOKEN, RPAR, OP_RETURN, A_GROUP_0, + }, + }, + {"genexp", + R_GENEXP, + {0, 10, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RPAR, OP_RETURN, A_GENEXP_0, + OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_GENEXP_1, + }, + }, + {"set", + R_SET, + {0, -1}, + { + OP_TOKEN, LBRACE, OP_RULE, R_EXPRESSIONS_LIST, OP_TOKEN, RBRACE, OP_RETURN, A_SET_0, + }, + }, + {"setcomp", + R_SETCOMP, + {0, 10, -1}, + { + OP_TOKEN, LBRACE, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_SETCOMP_0, + OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_SETCOMP_1, + }, + }, + {"dict", + R_DICT, + {0, -1}, + { + OP_TOKEN, LBRACE, OP_RULE, R_DOUBLE_STARRED_KVPAIRS, OP_OPTIONAL, OP_TOKEN, RBRACE, OP_RETURN, A_DICT_0, + }, + }, + {"dictcomp", + R_DICTCOMP, + {0, 10, -1}, + { + OP_TOKEN, LBRACE, OP_RULE, R_KVPAIR, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_DICTCOMP_0, + OP_RULE, R_INVALID_DICT_COMPREHENSION, OP_RETURN, A_DICTCOMP_1, + }, + }, + {"double_starred_kvpairs", + R_DOUBLE_STARRED_KVPAIRS, + {0, -1}, + { + OP_RULE, R__GATHER_86, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DOUBLE_STARRED_KVPAIRS_0, + }, + }, + {"double_starred_kvpair", + R_DOUBLE_STARRED_KVPAIR, + {0, 6, -1}, + { + OP_TOKEN, DOUBLESTAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_DOUBLE_STARRED_KVPAIR_0, + OP_RULE, R_KVPAIR, OP_RETURN, A_DOUBLE_STARRED_KVPAIR_1, + }, + }, + {"kvpair", + R_KVPAIR, + {0, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_KVPAIR_0, + }, + }, + {"for_if_clauses", + R_FOR_IF_CLAUSES, + {0, -1}, + { + OP_RULE, R__LOOP1_87, OP_RETURN, A_FOR_IF_CLAUSES_0, + }, + }, + {"for_if_clause", + R_FOR_IF_CLAUSE, + {0, 14, -1}, + { + OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_DISJUNCTION, OP_RULE, R__LOOP0_88, OP_RETURN, A_FOR_IF_CLAUSE_0, + OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_DISJUNCTION, OP_RULE, R__LOOP0_89, OP_RETURN, A_FOR_IF_CLAUSE_1, + }, + }, + {"yield_expr", + R_YIELD_EXPR, + {0, 8, -1}, + { + OP_TOKEN, 504, OP_TOKEN, 514, OP_RULE, R_EXPRESSION, OP_RETURN, A_YIELD_EXPR_0, + OP_TOKEN, 504, OP_RULE, R_STAR_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A_YIELD_EXPR_1, + }, + }, + {"arguments", + R_ARGUMENTS, + {0, 11, -1}, + { + OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_ARGUMENTS_0, + OP_RULE, R_INCORRECT_ARGUMENTS, OP_RETURN, A_ARGUMENTS_1, + }, + }, + {"args", + R_ARGS, + {0, 7, 11, -1}, + { + OP_RULE, R_STARRED_EXPRESSION, OP_RULE, R__TMP_90, OP_OPTIONAL, OP_RETURN, A_ARGS_0, + OP_RULE, R_KWARGS, OP_RETURN, A_ARGS_1, + OP_RULE, R_NAMED_EXPRESSION, OP_RULE, R__TMP_91, OP_OPTIONAL, OP_RETURN, A_ARGS_2, + }, + }, + {"kwargs", + R_KWARGS, + {0, 8, 12, -1}, + { + OP_RULE, R__GATHER_92, OP_TOKEN, COMMA, OP_RULE, R__GATHER_93, OP_RETURN, A_KWARGS_0, + OP_RULE, R__GATHER_94, OP_RETURN, A_KWARGS_1, + OP_RULE, R__GATHER_95, OP_RETURN, A_KWARGS_2, + }, + }, + {"starred_expression", + R_STARRED_EXPRESSION, + {0, -1}, + { + OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_STARRED_EXPRESSION_0, + }, + }, + {"kwarg_or_starred", + R_KWARG_OR_STARRED, + {0, 7, 11, -1}, + { + OP_NAME, OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_STARRED_0, + OP_RULE, R_STARRED_EXPRESSION, OP_RETURN, A_KWARG_OR_STARRED_1, + OP_RULE, R_INVALID_KWARG, OP_RETURN, A_KWARG_OR_STARRED_2, + }, + }, + {"kwarg_or_double_starred", + R_KWARG_OR_DOUBLE_STARRED, + {0, 7, 13, -1}, + { + OP_NAME, OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_0, + OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_1, + OP_RULE, R_INVALID_KWARG, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_2, + }, + }, + {"star_targets", + R_STAR_TARGETS, + {0, 8, -1}, + { + OP_RULE, R_STAR_TARGET, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGETS_0, + OP_RULE, R_STAR_TARGET, OP_RULE, R__LOOP0_96, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_1, + }, + }, + {"star_targets_seq", + R_STAR_TARGETS_SEQ, + {0, -1}, + { + OP_RULE, R__GATHER_97, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_SEQ_0, + }, + }, + {"star_target", + R_STAR_TARGET, + {0, 6, 17, 31, -1}, + { + OP_TOKEN, STAR, OP_RULE, R__TMP_98, OP_RETURN, A_STAR_TARGET_0, + OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGET_1, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGET_2, + OP_RULE, R_STAR_ATOM, OP_RETURN, A_STAR_TARGET_3, + }, + }, + {"star_atom", + R_STAR_ATOM, + {0, 3, 11, 20, -1}, + { + OP_NAME, OP_RETURN, A_STAR_ATOM_0, + OP_TOKEN, LPAR, OP_RULE, R_STAR_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_STAR_ATOM_1, + OP_TOKEN, LPAR, OP_RULE, R_STAR_TARGETS_SEQ, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_STAR_ATOM_2, + OP_TOKEN, LSQB, OP_RULE, R_STAR_TARGETS_SEQ, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_STAR_ATOM_3, + }, + }, + {"single_target", + R_SINGLE_TARGET, + {0, 4, 7, -1}, + { + OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, OP_RETURN, A_SINGLE_TARGET_0, + OP_NAME, OP_RETURN, A_SINGLE_TARGET_1, + OP_TOKEN, LPAR, OP_RULE, R_SINGLE_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_SINGLE_TARGET_2, + }, + }, + {"single_subscript_attribute_target", + R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, + {0, 11, -1}, + { + OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_0, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_1, + }, + }, + {"del_targets", + R_DEL_TARGETS, + {0, -1}, + { + OP_RULE, R__GATHER_99, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DEL_TARGETS_0, + }, + }, + {"del_target", + R_DEL_TARGET, + {0, 11, 25, -1}, + { + OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_0, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_1, + OP_RULE, R_DEL_T_ATOM, OP_RETURN, A_DEL_TARGET_2, + }, + }, + {"del_t_atom", + R_DEL_T_ATOM, + {0, 7, 15, 24, 33, -1}, + { + OP_NAME, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_T_ATOM_0, + OP_TOKEN, LPAR, OP_RULE, R_DEL_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_DEL_T_ATOM_1, + OP_TOKEN, LPAR, OP_RULE, R_DEL_TARGETS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_DEL_T_ATOM_2, + OP_TOKEN, LSQB, OP_RULE, R_DEL_TARGETS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_DEL_T_ATOM_3, + OP_RULE, R_INVALID_DEL_TARGET, OP_RETURN, A_DEL_T_ATOM_4, + }, + }, + {"del_target_end", + R_DEL_TARGET_END, + {0, 4, 8, 12, 16, -1}, + { + OP_TOKEN, RPAR, OP_RETURN, A_DEL_TARGET_END_0, + OP_TOKEN, RSQB, OP_RETURN, A_DEL_TARGET_END_1, + OP_TOKEN, COMMA, OP_RETURN, A_DEL_TARGET_END_2, + OP_TOKEN, SEMI, OP_RETURN, A_DEL_TARGET_END_3, + OP_TOKEN, NEWLINE, OP_RETURN, A_DEL_TARGET_END_4, + }, + }, + {"targets", + R_TARGETS, + {0, -1}, + { + OP_RULE, R__GATHER_100, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_TARGETS_0, + }, + }, + {"target", + R_TARGET, + {0, 11, 25, -1}, + { + OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_TARGET_0, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_TARGET_1, + OP_RULE, R_T_ATOM, OP_RETURN, A_TARGET_2, + }, + }, + {"t_primary", + R_T_PRIMARY, + {0, 12, 26, 36, 51, -1}, + { + OP_SETUP_LEFT_REC, OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_0, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_1, + OP_RULE, R_T_PRIMARY, OP_RULE, R_GENEXP, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_2, + OP_RULE, R_T_PRIMARY, OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_3, + OP_RULE, R_ATOM, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_4, + }, + }, + {"t_lookahead", + R_T_LOOKAHEAD, + {0, 4, 8, -1}, + { + OP_TOKEN, LPAR, OP_RETURN, A_T_LOOKAHEAD_0, + OP_TOKEN, LSQB, OP_RETURN, A_T_LOOKAHEAD_1, + OP_TOKEN, DOT, OP_RETURN, A_T_LOOKAHEAD_2, + }, + }, + {"t_atom", + R_T_ATOM, + {0, 3, 11, 20, -1}, + { + OP_NAME, OP_RETURN, A_T_ATOM_0, + OP_TOKEN, LPAR, OP_RULE, R_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_T_ATOM_1, + OP_TOKEN, LPAR, OP_RULE, R_TARGETS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_T_ATOM_2, + OP_TOKEN, LSQB, OP_RULE, R_TARGETS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_T_ATOM_3, + }, + }, + {"incorrect_arguments", + R_INCORRECT_ARGUMENTS, + {0, 8, 19, 25, 35, -1}, + { + OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RETURN, A_INCORRECT_ARGUMENTS_0, + OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, COMMA, OP_RULE, R__TMP_101, OP_OPTIONAL, OP_RETURN, A_INCORRECT_ARGUMENTS_1, + OP_RULE, R_ARGS, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INCORRECT_ARGUMENTS_2, + OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INCORRECT_ARGUMENTS_3, + OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A_INCORRECT_ARGUMENTS_4, + }, + }, + {"invalid_kwarg", + R_INVALID_KWARG, + {0, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, EQUAL, OP_RETURN, A_INVALID_KWARG_0, + }, + }, + {"invalid_named_expression", + R_INVALID_NAMED_EXPRESSION, + {0, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COLONEQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_INVALID_NAMED_EXPRESSION_0, + }, + }, + {"invalid_assignment", + R_INVALID_ASSIGNMENT, + {0, 6, 12, 22, 33, 41, -1}, + { + OP_RULE, R_LIST, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_0, + OP_RULE, R_TUPLE, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_1, + OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_RULE, R__LOOP0_102, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_2, + OP_RULE, R_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_103, OP_OPTIONAL, OP_RETURN, A_INVALID_ASSIGNMENT_3, + OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, EQUAL, OP_RULE, R__TMP_104, OP_RETURN, A_INVALID_ASSIGNMENT_4, + OP_RULE, R_STAR_EXPRESSIONS, OP_RULE, R_AUGASSIGN, OP_RULE, R__TMP_105, OP_RETURN, A_INVALID_ASSIGNMENT_5, + }, + }, + {"invalid_block", + R_INVALID_BLOCK, + {0, -1}, + { + OP_TOKEN, NEWLINE, OP_SAVE_MARK, OP_TOKEN, INDENT, OP_NEG_LOOKAHEAD, OP_RETURN, A_INVALID_BLOCK_0, + }, + }, + {"invalid_comprehension", + R_INVALID_COMPREHENSION, + {0, -1}, + { + OP_RULE, R__TMP_106, OP_RULE, R_STARRED_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INVALID_COMPREHENSION_0, + }, + }, + {"invalid_dict_comprehension", + R_INVALID_DICT_COMPREHENSION, + {0, -1}, + { + OP_TOKEN, LBRACE, OP_TOKEN, DOUBLESTAR, OP_RULE, R_BITWISE_OR, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_INVALID_DICT_COMPREHENSION_0, + }, + }, + {"invalid_parameters", + R_INVALID_PARAMETERS, + {0, -1}, + { + OP_RULE, R__LOOP0_107, OP_RULE, R__TMP_108, OP_RULE, R_PARAM_NO_DEFAULT, OP_RETURN, A_INVALID_PARAMETERS_0, + }, + }, + {"invalid_star_etc", + R_INVALID_STAR_ETC, + {0, 6, -1}, + { + OP_TOKEN, STAR, OP_RULE, R__TMP_109, OP_RETURN, A_INVALID_STAR_ETC_0, + OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_RETURN, A_INVALID_STAR_ETC_1, + }, + }, + {"invalid_lambda_star_etc", + R_INVALID_LAMBDA_STAR_ETC, + {0, -1}, + { + OP_TOKEN, STAR, OP_RULE, R__TMP_110, OP_RETURN, A_INVALID_LAMBDA_STAR_ETC_0, + }, + }, + {"invalid_double_type_comments", + R_INVALID_DOUBLE_TYPE_COMMENTS, + {0, -1}, + { + OP_TOKEN, TYPE_COMMENT, OP_TOKEN, NEWLINE, OP_TOKEN, TYPE_COMMENT, OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RETURN, A_INVALID_DOUBLE_TYPE_COMMENTS_0, + }, + }, + {"invalid_del_target", + R_INVALID_DEL_TARGET, + {0, -1}, + { + OP_RULE, R_STAR_EXPRESSION, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_INVALID_DEL_TARGET_0, + }, + }, + {"invalid_import_from_targets", + R_INVALID_IMPORT_FROM_TARGETS, + {0, -1}, + { + OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_TOKEN, COMMA, OP_RETURN, A_INVALID_IMPORT_FROM_TARGETS_0, + }, + }, + {"root", + R_ROOT, + {0, 3, -1}, + { + OP_RULE, R_FILE, OP_SUCCESS, + OP_FAILURE, + }, + }, + {"_loop0_1", + R__LOOP0_1, + {0, 3, -1}, + { + OP_TOKEN, NEWLINE, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_2", + R__LOOP0_2, + {0, 3, -1}, + { + OP_TOKEN, NEWLINE, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_gather_3", + R__GATHER_3, + {0, 5, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_4", + R__GATHER_4, + {0, 5, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_5", + R__GATHER_5, + {0, 5, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_6", + R__GATHER_6, + {0, 5, -1}, + { + OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_loop1_7", + R__LOOP1_7, + {0, 3, -1}, + { + OP_RULE, R_STATEMENT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_gather_8", + R__GATHER_8, + {0, 5, -1}, + { + OP_RULE, R_SMALL_STMT, OP_TOKEN, SEMI, OP_LOOP_ITERATE, + OP_RULE, R_SMALL_STMT, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_9", + R__TMP_9, + {0, 4, -1}, + { + OP_TOKEN, 513, OP_RETURN, A__TMP_9_0, + OP_TOKEN, 514, OP_RETURN, A__TMP_9_1, + }, + }, + {"_tmp_10", + R__TMP_10, + {0, 4, 8, -1}, + { + OP_TOKEN, 522, OP_RETURN, A__TMP_10_0, + OP_TOKEN, AT, OP_RETURN, A__TMP_10_1, + OP_TOKEN, ASYNC, OP_RETURN, A__TMP_10_2, + }, + }, + {"_tmp_11", + R__TMP_11, + {0, 4, -1}, + { + OP_TOKEN, 523, OP_RETURN, A__TMP_11_0, + OP_TOKEN, AT, OP_RETURN, A__TMP_11_1, + }, + }, + {"_tmp_12", + R__TMP_12, + {0, 4, -1}, + { + OP_TOKEN, 519, OP_RETURN, A__TMP_12_0, + OP_TOKEN, ASYNC, OP_RETURN, A__TMP_12_1, + }, + }, + {"_tmp_13", + R__TMP_13, + {0, 4, -1}, + { + OP_TOKEN, 517, OP_RETURN, A__TMP_13_0, + OP_TOKEN, ASYNC, OP_RETURN, A__TMP_13_1, + }, + }, + {"_tmp_14", + R__TMP_14, + {0, -1}, + { + OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_14_0, + }, + }, + {"_tmp_15", + R__TMP_15, + {0, 8, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R_SINGLE_TARGET, OP_TOKEN, RPAR, OP_RETURN, A__TMP_15_0, + OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, OP_RETURN, A__TMP_15_1, + }, + }, + {"_tmp_16", + R__TMP_16, + {0, -1}, + { + OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_16_0, + }, + }, + {"_loop1_17", + R__LOOP1_17, + {0, 3, -1}, + { + OP_RULE, R__TMP_111, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_18", + R__TMP_18, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_18_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_18_1, + }, + }, + {"_tmp_19", + R__TMP_19, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_19_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_19_1, + }, + }, + {"_gather_20", + R__GATHER_20, + {0, 4, -1}, + { + OP_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_NAME, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_21", + R__GATHER_21, + {0, 4, -1}, + { + OP_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_NAME, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_22", + R__TMP_22, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_22_0, + }, + }, + {"_loop0_23", + R__LOOP0_23, + {0, 3, -1}, + { + OP_RULE, R__TMP_112, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_24", + R__LOOP1_24, + {0, 3, -1}, + { + OP_RULE, R__TMP_113, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_gather_25", + R__GATHER_25, + {0, 5, -1}, + { + OP_RULE, R_IMPORT_FROM_AS_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_IMPORT_FROM_AS_NAME, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_26", + R__TMP_26, + {0, -1}, + { + OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_26_0, + }, + }, + {"_gather_27", + R__GATHER_27, + {0, 5, -1}, + { + OP_RULE, R_DOTTED_AS_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_DOTTED_AS_NAME, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_28", + R__TMP_28, + {0, -1}, + { + OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_28_0, + }, + }, + {"_gather_29", + R__GATHER_29, + {0, 5, -1}, + { + OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_30", + R__GATHER_30, + {0, 5, -1}, + { + OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_31", + R__GATHER_31, + {0, 5, -1}, + { + OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_32", + R__GATHER_32, + {0, 5, -1}, + { + OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_33", + R__TMP_33, + {0, -1}, + { + OP_TOKEN, 531, OP_RULE, R_TARGET, OP_RETURN, A__TMP_33_0, + }, + }, + {"_loop1_34", + R__LOOP1_34, + {0, 3, -1}, + { + OP_RULE, R_EXCEPT_BLOCK, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_35", + R__TMP_35, + {0, -1}, + { + OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_35_0, + }, + }, + {"_tmp_36", + R__TMP_36, + {0, -1}, + { + OP_TOKEN, 514, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_36_0, + }, + }, + {"_tmp_37", + R__TMP_37, + {0, -1}, + { + OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_37_0, + }, + }, + {"_tmp_38", + R__TMP_38, + {0, -1}, + { + OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_38_0, + }, + }, + {"_tmp_39", + R__TMP_39, + {0, -1}, + { + OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RETURN, A__TMP_39_0, + }, + }, + {"_loop0_40", + R__LOOP0_40, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_41", + R__LOOP0_41, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_42", + R__LOOP0_42, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_43", + R__LOOP1_43, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_44", + R__LOOP0_44, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_45", + R__LOOP1_45, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_46", + R__LOOP1_46, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_47", + R__LOOP1_47, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_48", + R__LOOP0_48, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_49", + R__LOOP1_49, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_50", + R__LOOP0_50, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_51", + R__LOOP1_51, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_52", + R__LOOP0_52, + {0, 3, -1}, + { + OP_RULE, R_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_53", + R__LOOP1_53, + {0, 3, -1}, + { + OP_RULE, R_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_54", + R__LOOP1_54, + {0, 3, -1}, + { + OP_RULE, R__TMP_114, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_55", + R__TMP_55, + {0, -1}, + { + OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A__TMP_55_0, + }, + }, + {"_gather_56", + R__GATHER_56, + {0, 5, -1}, + { + OP_RULE, R_STAR_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_STAR_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_loop1_57", + R__LOOP1_57, + {0, 3, -1}, + { + OP_RULE, R__TMP_115, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_gather_58", + R__GATHER_58, + {0, 5, -1}, + { + OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_STAR_NAMED_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_loop1_59", + R__LOOP1_59, + {0, 3, -1}, + { + OP_RULE, R__TMP_116, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_60", + R__LOOP0_60, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_61", + R__LOOP0_61, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_62", + R__LOOP0_62, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_63", + R__LOOP1_63, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_64", + R__LOOP0_64, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_65", + R__LOOP1_65, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_66", + R__LOOP1_66, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_67", + R__LOOP1_67, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_68", + R__LOOP0_68, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_69", + R__LOOP1_69, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_70", + R__LOOP0_70, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_71", + R__LOOP1_71, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_72", + R__LOOP0_72, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop1_73", + R__LOOP1_73, + {0, 3, -1}, + { + OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_74", + R__LOOP1_74, + {0, 3, -1}, + { + OP_RULE, R__TMP_117, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_75", + R__LOOP1_75, + {0, 3, -1}, + { + OP_RULE, R__TMP_118, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop1_76", + R__LOOP1_76, + {0, 3, -1}, + { + OP_RULE, R_COMPARE_OP_BITWISE_OR_PAIR, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_77", + R__TMP_77, + {0, -1}, + { + OP_TOKEN, NOTEQUAL, OP_RETURN, A__TMP_77_0, + }, + }, + {"_gather_78", + R__GATHER_78, + {0, 5, -1}, + { + OP_RULE, R_SLICE, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_SLICE, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_79", + R__TMP_79, + {0, -1}, + { + OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_RETURN, A__TMP_79_0, + }, + }, + {"_tmp_80", + R__TMP_80, + {0, 4, 8, -1}, + { + OP_RULE, R_TUPLE, OP_RETURN, A__TMP_80_0, + OP_RULE, R_GROUP, OP_RETURN, A__TMP_80_1, + OP_RULE, R_GENEXP, OP_RETURN, A__TMP_80_2, + }, + }, + {"_tmp_81", + R__TMP_81, + {0, 4, -1}, + { + OP_RULE, R_LIST, OP_RETURN, A__TMP_81_0, + OP_RULE, R_LISTCOMP, OP_RETURN, A__TMP_81_1, + }, + }, + {"_tmp_82", + R__TMP_82, + {0, 4, 8, 12, -1}, + { + OP_RULE, R_DICT, OP_RETURN, A__TMP_82_0, + OP_RULE, R_SET, OP_RETURN, A__TMP_82_1, + OP_RULE, R_DICTCOMP, OP_RETURN, A__TMP_82_2, + OP_RULE, R_SETCOMP, OP_RETURN, A__TMP_82_3, + }, + }, + {"_loop1_83", + R__LOOP1_83, + {0, 2, -1}, + { + OP_STRING, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_84", + R__TMP_84, + {0, -1}, + { + OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A__TMP_84_0, + }, + }, + {"_tmp_85", + R__TMP_85, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_85_0, + OP_RULE, R_NAMED_EXPRESSION, OP_RETURN, A__TMP_85_1, + }, + }, + {"_gather_86", + R__GATHER_86, + {0, 5, -1}, + { + OP_RULE, R_DOUBLE_STARRED_KVPAIR, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_DOUBLE_STARRED_KVPAIR, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_loop1_87", + R__LOOP1_87, + {0, 3, -1}, + { + OP_RULE, R_FOR_IF_CLAUSE, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_loop0_88", + R__LOOP0_88, + {0, 3, -1}, + { + OP_RULE, R__TMP_119, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_loop0_89", + R__LOOP0_89, + {0, 3, -1}, + { + OP_RULE, R__TMP_120, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_tmp_90", + R__TMP_90, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A__TMP_90_0, + }, + }, + {"_tmp_91", + R__TMP_91, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A__TMP_91_0, + }, + }, + {"_gather_92", + R__GATHER_92, + {0, 5, -1}, + { + OP_RULE, R_KWARG_OR_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_KWARG_OR_STARRED, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_93", + R__GATHER_93, + {0, 5, -1}, + { + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_94", + R__GATHER_94, + {0, 5, -1}, + { + OP_RULE, R_KWARG_OR_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_KWARG_OR_STARRED, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_95", + R__GATHER_95, + {0, 5, -1}, + { + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_loop0_96", + R__LOOP0_96, + {0, 3, -1}, + { + OP_RULE, R__TMP_121, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_gather_97", + R__GATHER_97, + {0, 5, -1}, + { + OP_RULE, R_STAR_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_STAR_TARGET, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_98", + R__TMP_98, + {0, -1}, + { + OP_SAVE_MARK, OP_TOKEN, STAR, OP_NEG_LOOKAHEAD, OP_RULE, R_STAR_TARGET, OP_RETURN, A__TMP_98_0, + }, + }, + {"_gather_99", + R__GATHER_99, + {0, 5, -1}, + { + OP_RULE, R_DEL_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_DEL_TARGET, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_gather_100", + R__GATHER_100, + {0, 5, -1}, + { + OP_RULE, R_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, + OP_RULE, R_TARGET, OP_LOOP_COLLECT_DELIMITED, + }, + }, + {"_tmp_101", + R__TMP_101, + {0, 4, -1}, + { + OP_RULE, R_ARGS, OP_RETURN, A__TMP_101_0, + OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A__TMP_101_1, + }, + }, + {"_loop0_102", + R__LOOP0_102, + {0, 3, -1}, + { + OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_tmp_103", + R__TMP_103, + {0, -1}, + { + OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_103_0, + }, + }, + {"_tmp_104", + R__TMP_104, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_104_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_104_1, + }, + }, + {"_tmp_105", + R__TMP_105, + {0, 4, -1}, + { + OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_105_0, + OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_105_1, + }, + }, + {"_tmp_106", + R__TMP_106, + {0, 4, 8, -1}, + { + OP_TOKEN, LSQB, OP_RETURN, A__TMP_106_0, + OP_TOKEN, LPAR, OP_RETURN, A__TMP_106_1, + OP_TOKEN, LBRACE, OP_RETURN, A__TMP_106_2, + }, + }, + {"_loop0_107", + R__LOOP0_107, + {0, 3, -1}, + { + OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT, + }, + }, + {"_tmp_108", + R__TMP_108, + {0, 4, -1}, + { + OP_RULE, R_SLASH_WITH_DEFAULT, OP_RETURN, A__TMP_108_0, + OP_RULE, R__LOOP1_122, OP_RETURN, A__TMP_108_1, + }, + }, + {"_tmp_109", + R__TMP_109, + {0, 4, -1}, + { + OP_TOKEN, RPAR, OP_RETURN, A__TMP_109_0, + OP_TOKEN, COMMA, OP_RULE, R__TMP_123, OP_RETURN, A__TMP_109_1, + }, + }, + {"_tmp_110", + R__TMP_110, + {0, 4, -1}, + { + OP_TOKEN, COLON, OP_RETURN, A__TMP_110_0, + OP_TOKEN, COMMA, OP_RULE, R__TMP_124, OP_RETURN, A__TMP_110_1, + }, + }, + {"_tmp_111", + R__TMP_111, + {0, -1}, + { + OP_RULE, R_STAR_TARGETS, OP_TOKEN, EQUAL, OP_RETURN, A__TMP_111_0, + }, + }, + {"_tmp_112", + R__TMP_112, + {0, 4, -1}, + { + OP_TOKEN, DOT, OP_RETURN, A__TMP_112_0, + OP_TOKEN, ELLIPSIS, OP_RETURN, A__TMP_112_1, + }, + }, + {"_tmp_113", + R__TMP_113, + {0, 4, -1}, + { + OP_TOKEN, DOT, OP_RETURN, A__TMP_113_0, + OP_TOKEN, ELLIPSIS, OP_RETURN, A__TMP_113_1, + }, + }, + {"_tmp_114", + R__TMP_114, + {0, -1}, + { + OP_TOKEN, AT, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, NEWLINE, OP_RETURN, A__TMP_114_0, + }, + }, + {"_tmp_115", + R__TMP_115, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_STAR_EXPRESSION, OP_RETURN, A__TMP_115_0, + }, + }, + {"_tmp_116", + R__TMP_116, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_116_0, + }, + }, + {"_tmp_117", + R__TMP_117, + {0, -1}, + { + OP_TOKEN, 532, OP_RULE, R_CONJUNCTION, OP_RETURN, A__TMP_117_0, + }, + }, + {"_tmp_118", + R__TMP_118, + {0, -1}, + { + OP_TOKEN, 533, OP_RULE, R_INVERSION, OP_RETURN, A__TMP_118_0, + }, + }, + {"_tmp_119", + R__TMP_119, + {0, -1}, + { + OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_RETURN, A__TMP_119_0, + }, + }, + {"_tmp_120", + R__TMP_120, + {0, -1}, + { + OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_RETURN, A__TMP_120_0, + }, + }, + {"_tmp_121", + R__TMP_121, + {0, -1}, + { + OP_TOKEN, COMMA, OP_RULE, R_STAR_TARGET, OP_RETURN, A__TMP_121_0, + }, + }, + {"_loop1_122", + R__LOOP1_122, + {0, 3, -1}, + { + OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, + OP_LOOP_COLLECT_NONEMPTY, + }, + }, + {"_tmp_123", + R__TMP_123, + {0, 4, -1}, + { + OP_TOKEN, RPAR, OP_RETURN, A__TMP_123_0, + OP_TOKEN, DOUBLESTAR, OP_RETURN, A__TMP_123_1, + }, + }, + {"_tmp_124", + R__TMP_124, + {0, 4, -1}, + { + OP_TOKEN, COLON, OP_RETURN, A__TMP_124_0, + OP_TOKEN, DOUBLESTAR, OP_RETURN, A__TMP_124_1, + }, + }, +}; + +static void * +call_action(Parser *p, Frame *_f, int _iaction) +{ + assert(p->mark > 0); + Token *_t = p->tokens[_f->mark]; + int _start_lineno = _t->lineno; + int _start_col_offset = _t->col_offset; + _t = p->tokens[p->mark - 1]; + int _end_lineno = _t->end_lineno; + int _end_col_offset = _t->end_col_offset; + + switch (_iaction) { + case A_FILE_0: + return _PyPegen_make_module ( p , _f->vals[0] ); + case A_INTERACTIVE_0: + return Interactive ( ((asdl_seq*)_f->vals[0]) , p -> arena ); + case A_EVAL_0: + return Expression ( ((expr_ty)_f->vals[0]) , p -> arena ); + case A_FUNC_TYPE_0: + return FunctionType ( _f->vals[1] , ((expr_ty)_f->vals[4]) , p -> arena ); + case A_FSTRING_0: + case A_DOTTED_NAME_1: + case A_STAR_EXPRESSIONS_2: + case A_STAR_EXPRESSION_1: + case A_STAR_NAMED_EXPRESSION_1: + case A_NAMED_EXPRESSION_1: + case A_ANNOTATED_RHS_0: + case A_ANNOTATED_RHS_1: + case A_EXPRESSIONS_2: + case A_EXPRESSION_1: + case A_EXPRESSION_2: + case A_DISJUNCTION_1: + case A_CONJUNCTION_1: + case A_INVERSION_1: + case A_COMPARISON_1: + case A_BITWISE_OR_1: + case A_BITWISE_XOR_1: + case A_BITWISE_AND_1: + case A_SHIFT_EXPR_2: + case A_SUM_2: + case A_TERM_5: + case A_FACTOR_3: + case A_POWER_1: + case A_AWAIT_PRIMARY_1: + case A_PRIMARY_4: + case A_SLICES_0: + case A_SLICE_1: + case A_ATOM_0: + case A_ARGUMENTS_0: + case A_STAR_TARGETS_0: + case A_STAR_TARGET_3: + case A_SINGLE_TARGET_0: + case A_DEL_TARGET_2: + case A_TARGET_2: + case A_T_PRIMARY_4: + case A__GATHER_3_0: + case A__GATHER_3_1: + case A__GATHER_4_0: + case A__GATHER_4_1: + case A__GATHER_5_0: + case A__GATHER_5_1: + case A__GATHER_6_0: + case A__GATHER_6_1: + case A__TMP_15_1: + case A__TMP_18_0: + case A__TMP_18_1: + case A__TMP_19_0: + case A__TMP_19_1: + case A__GATHER_20_0: + case A__GATHER_20_1: + case A__GATHER_21_0: + case A__GATHER_21_1: + case A__GATHER_56_0: + case A__GATHER_56_1: + case A__GATHER_58_0: + case A__GATHER_58_1: + case A__GATHER_78_0: + case A__GATHER_78_1: + case A__TMP_80_0: + case A__TMP_80_1: + case A__TMP_80_2: + case A__TMP_81_0: + case A__TMP_81_1: + case A__TMP_82_0: + case A__TMP_82_1: + case A__TMP_82_2: + case A__TMP_82_3: + case A__TMP_85_0: + case A__TMP_85_1: + case A__GATHER_97_0: + case A__GATHER_97_1: + case A__GATHER_99_0: + case A__GATHER_99_1: + case A__GATHER_100_0: + case A__GATHER_100_1: + case A__TMP_101_0: + case A__TMP_101_1: + case A__TMP_104_0: + case A__TMP_104_1: + case A__TMP_105_0: + case A__TMP_105_1: + case A__TMP_111_0: + return ((expr_ty)_f->vals[0]); + case A_TYPE_EXPRESSIONS_0: + return _PyPegen_seq_append_to_end ( p , CHECK ( _PyPegen_seq_append_to_end ( p , _f->vals[0] , ((expr_ty)_f->vals[3]) ) ) , ((expr_ty)_f->vals[6]) ); + case A_TYPE_EXPRESSIONS_1: + case A_TYPE_EXPRESSIONS_2: + return _PyPegen_seq_append_to_end ( p , _f->vals[0] , ((expr_ty)_f->vals[3]) ); + case A_TYPE_EXPRESSIONS_3: + return _PyPegen_seq_append_to_end ( p , CHECK ( _PyPegen_singleton_seq ( p , ((expr_ty)_f->vals[1]) ) ) , ((expr_ty)_f->vals[4]) ); + case A_TYPE_EXPRESSIONS_4: + case A_TYPE_EXPRESSIONS_5: + return _PyPegen_singleton_seq ( p , ((expr_ty)_f->vals[1]) ); + case A_TYPE_EXPRESSIONS_6: + case A_SIMPLE_STMT_1: + case A_SMALL_STMT_2: + case A_SMALL_STMT_3: + case A_SMALL_STMT_4: + case A_SMALL_STMT_6: + case A_SMALL_STMT_7: + case A_SMALL_STMT_8: + case A_SMALL_STMT_11: + case A_SMALL_STMT_12: + case A_COMPOUND_STMT_0: + case A_COMPOUND_STMT_1: + case A_COMPOUND_STMT_2: + case A_COMPOUND_STMT_3: + case A_COMPOUND_STMT_4: + case A_COMPOUND_STMT_5: + case A_COMPOUND_STMT_6: + case A_ASSIGNMENT_4: + case A_IMPORT_FROM_TARGETS_3: + case A_IMPORT_FROM_AS_NAMES_0: + case A_DOTTED_AS_NAMES_0: + case A_FUNC_TYPE_COMMENT_1: + case A_FUNC_TYPE_COMMENT_2: + case A_PARAMS_0: + case A_SLASH_NO_DEFAULT_0: + case A_SLASH_NO_DEFAULT_1: + case A_STAR_ETC_3: + case A_DECORATORS_0: + case A_BLOCK_2: + case A_EXPRESSIONS_LIST_0: + case A_STAR_NAMED_EXPRESSIONS_0: + case A_NAMED_EXPRESSION_2: + case A_LAMBDA_SLASH_NO_DEFAULT_0: + case A_LAMBDA_SLASH_NO_DEFAULT_1: + case A_LAMBDA_STAR_ETC_3: + case A_ATOM_5: + case A_ATOM_6: + case A_ATOM_7: + case A_ATOM_8: + case A_ATOM_9: + case A_LISTCOMP_1: + case A_GENEXP_1: + case A_SETCOMP_1: + case A_DICTCOMP_1: + case A_DOUBLE_STARRED_KVPAIRS_0: + case A_FOR_IF_CLAUSES_0: + case A_ARGUMENTS_1: + case A_KWARGS_1: + case A_KWARGS_2: + case A_KWARG_OR_STARRED_2: + case A_KWARG_OR_DOUBLE_STARRED_2: + case A_STAR_TARGETS_SEQ_0: + case A_DEL_TARGETS_0: + case A_DEL_T_ATOM_4: + case A_DEL_TARGET_END_4: + case A_TARGETS_0: + case A__TMP_10_2: + case A__TMP_12_1: + case A__TMP_13_1: + case A__TMP_39_0: + case A__TMP_98_0: + case A__TMP_108_1: + return _f->vals[0]; + case A_STATEMENTS_0: + return _PyPegen_seq_flatten ( p , _f->vals[0] ); + case A_STATEMENT_0: + case A_STATEMENT_NEWLINE_0: + case A_SIMPLE_STMT_0: + return _PyPegen_singleton_seq ( p , ((stmt_ty)_f->vals[0]) ); + case A_STATEMENT_1: + case A_STATEMENT_NEWLINE_1: + case A_IMPORT_FROM_TARGETS_1: + case A_BLOCK_1: + return ((asdl_seq*)_f->vals[0]); + case A_STATEMENT_NEWLINE_2: + return _PyPegen_singleton_seq ( p , CHECK ( _Py_Pass ( EXTRA ) ) ); + case A_STATEMENT_NEWLINE_3: + return _PyPegen_interactive_exit ( p ); + case A_SMALL_STMT_0: + case A_IMPORT_STMT_0: + case A_IMPORT_STMT_1: + case A_FUNCTION_DEF_1: + case A_CLASS_DEF_1: + case A__GATHER_8_0: + case A__GATHER_8_1: + return ((stmt_ty)_f->vals[0]); + case A_SMALL_STMT_1: + case A_YIELD_STMT_0: + return _Py_Expr ( ((expr_ty)_f->vals[0]) , EXTRA ); + case A_SMALL_STMT_5: + return _Py_Pass ( EXTRA ); + case A_SMALL_STMT_9: + return _Py_Break ( EXTRA ); + case A_SMALL_STMT_10: + return _Py_Continue ( EXTRA ); + case A_ASSIGNMENT_0: + return CHECK_VERSION ( 6 , "Variable annotation syntax is" , _Py_AnnAssign ( CHECK ( _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Store ) ) , ((expr_ty)_f->vals[2]) , _f->vals[3] , 1 , EXTRA ) ); + case A_ASSIGNMENT_1: + return CHECK_VERSION ( 6 , "Variable annotations syntax is" , _Py_AnnAssign ( _f->vals[0] , ((expr_ty)_f->vals[2]) , _f->vals[3] , 0 , EXTRA ) ); + case A_ASSIGNMENT_2: + return _Py_Assign ( _f->vals[0] , _f->vals[1] , NEW_TYPE_COMMENT ( p , _f->vals[2] ) , EXTRA ); + case A_ASSIGNMENT_3: + return _Py_AugAssign ( ((expr_ty)_f->vals[0]) , ((AugOperator*)_f->vals[1]) -> kind , _f->vals[2] , EXTRA ); + case A_AUGASSIGN_0: + return _PyPegen_augoperator ( p , Add ); + case A_AUGASSIGN_1: + return _PyPegen_augoperator ( p , Sub ); + case A_AUGASSIGN_2: + return _PyPegen_augoperator ( p , Mult ); + case A_AUGASSIGN_3: + return CHECK_VERSION ( 5 , "The '@' operator is" , _PyPegen_augoperator ( p , MatMult ) ); + case A_AUGASSIGN_4: + return _PyPegen_augoperator ( p , Div ); + case A_AUGASSIGN_5: + return _PyPegen_augoperator ( p , Mod ); + case A_AUGASSIGN_6: + return _PyPegen_augoperator ( p , BitAnd ); + case A_AUGASSIGN_7: + return _PyPegen_augoperator ( p , BitOr ); + case A_AUGASSIGN_8: + return _PyPegen_augoperator ( p , BitXor ); + case A_AUGASSIGN_9: + return _PyPegen_augoperator ( p , LShift ); + case A_AUGASSIGN_10: + return _PyPegen_augoperator ( p , RShift ); + case A_AUGASSIGN_11: + return _PyPegen_augoperator ( p , Pow ); + case A_AUGASSIGN_12: + return _PyPegen_augoperator ( p , FloorDiv ); + case A_GLOBAL_STMT_0: + return _Py_Global ( CHECK ( _PyPegen_map_names_to_ids ( p , _f->vals[1] ) ) , EXTRA ); + case A_NONLOCAL_STMT_0: + return _Py_Nonlocal ( CHECK ( _PyPegen_map_names_to_ids ( p , _f->vals[1] ) ) , EXTRA ); + case A_ASSERT_STMT_0: + return _Py_Assert ( ((expr_ty)_f->vals[1]) , _f->vals[2] , EXTRA ); + case A_DEL_STMT_0: + return _Py_Delete ( ((asdl_seq*)_f->vals[1]) , EXTRA ); + case A_IMPORT_NAME_0: + return _Py_Import ( ((asdl_seq*)_f->vals[1]) , EXTRA ); + case A_IMPORT_FROM_0: + return _Py_ImportFrom ( ((expr_ty)_f->vals[2]) -> v . Name . id , ((asdl_seq*)_f->vals[4]) , _PyPegen_seq_count_dots ( _f->vals[1] ) , EXTRA ); + case A_IMPORT_FROM_1: + return _Py_ImportFrom ( NULL , ((asdl_seq*)_f->vals[3]) , _PyPegen_seq_count_dots ( _f->vals[1] ) , EXTRA ); + case A_IMPORT_FROM_TARGETS_0: + return ((asdl_seq*)_f->vals[1]); + case A_IMPORT_FROM_TARGETS_2: + return _PyPegen_singleton_seq ( p , CHECK ( _PyPegen_alias_for_star ( p ) ) ); + case A_IMPORT_FROM_AS_NAME_0: + case A_DOTTED_AS_NAME_0: + return _Py_alias ( ((expr_ty)_f->vals[0]) -> v . Name . id , ( _f->vals[1] ) ? ( ( expr_ty ) _f->vals[1] ) -> v . Name . id : NULL , p -> arena ); + case A_DOTTED_NAME_0: + return _PyPegen_join_names_with_dot ( p , ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) ); + case A_IF_STMT_0: + case A_ELIF_STMT_0: + return _Py_If ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[3]) , CHECK ( _PyPegen_singleton_seq ( p , ((stmt_ty)_f->vals[4]) ) ) , EXTRA ); + case A_IF_STMT_1: + case A_ELIF_STMT_1: + return _Py_If ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[3]) , _f->vals[4] , EXTRA ); + case A_ELSE_BLOCK_0: + case A_FINALLY_BLOCK_0: + case A_BLOCK_0: + return ((asdl_seq*)_f->vals[2]); + case A_WHILE_STMT_0: + return _Py_While ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[3]) , _f->vals[4] , EXTRA ); + case A_FOR_STMT_0: + return _Py_For ( ((expr_ty)_f->vals[1]) , ((expr_ty)_f->vals[3]) , ((asdl_seq*)_f->vals[6]) , _f->vals[7] , NEW_TYPE_COMMENT ( p , _f->vals[5] ) , EXTRA ); + case A_FOR_STMT_1: + return CHECK_VERSION ( 5 , "Async for loops are" , _Py_AsyncFor ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[4]) , ((asdl_seq*)_f->vals[7]) , _f->vals[8] , NEW_TYPE_COMMENT ( p , _f->vals[6] ) , EXTRA ) ); + case A_WITH_STMT_0: + return _Py_With ( _f->vals[2] , ((asdl_seq*)_f->vals[6]) , NULL , EXTRA ); + case A_WITH_STMT_1: + return _Py_With ( _f->vals[1] , ((asdl_seq*)_f->vals[4]) , NEW_TYPE_COMMENT ( p , _f->vals[3] ) , EXTRA ); + case A_WITH_STMT_2: + return CHECK_VERSION ( 5 , "Async with statements are" , _Py_AsyncWith ( _f->vals[3] , ((asdl_seq*)_f->vals[7]) , NULL , EXTRA ) ); + case A_WITH_STMT_3: + return CHECK_VERSION ( 5 , "Async with statements are" , _Py_AsyncWith ( _f->vals[2] , ((asdl_seq*)_f->vals[5]) , NEW_TYPE_COMMENT ( p , _f->vals[4] ) , EXTRA ) ); + case A_WITH_ITEM_0: + return _Py_withitem ( ((expr_ty)_f->vals[0]) , _f->vals[1] , p -> arena ); + case A_TRY_STMT_0: + return _Py_Try ( ((asdl_seq*)_f->vals[2]) , NULL , NULL , ((asdl_seq*)_f->vals[3]) , EXTRA ); + case A_TRY_STMT_1: + return _Py_Try ( ((asdl_seq*)_f->vals[2]) , _f->vals[3] , _f->vals[4] , _f->vals[5] , EXTRA ); + case A_EXCEPT_BLOCK_0: + return _Py_ExceptHandler ( ((expr_ty)_f->vals[1]) , ( _f->vals[2] ) ? ( ( expr_ty ) _f->vals[2] ) -> v . Name . id : NULL , ((asdl_seq*)_f->vals[4]) , EXTRA ); + case A_EXCEPT_BLOCK_1: + return _Py_ExceptHandler ( NULL , NULL , ((asdl_seq*)_f->vals[2]) , EXTRA ); + case A_RETURN_STMT_0: + return _Py_Return ( _f->vals[1] , EXTRA ); + case A_RAISE_STMT_0: + return _Py_Raise ( ((expr_ty)_f->vals[1]) , _f->vals[2] , EXTRA ); + case A_RAISE_STMT_1: + return _Py_Raise ( NULL , NULL , EXTRA ); + case A_FUNCTION_DEF_0: + return _PyPegen_function_def_decorators ( p , ((asdl_seq*)_f->vals[0]) , ((stmt_ty)_f->vals[1]) ); + case A_FUNCTION_DEF_RAW_0: + return _Py_FunctionDef ( ((expr_ty)_f->vals[1]) -> v . Name . id , ( _f->vals[3] ) ? _f->vals[3] : CHECK ( _PyPegen_empty_arguments ( p ) ) , ((asdl_seq*)_f->vals[8]) , NULL , _f->vals[5] , NEW_TYPE_COMMENT ( p , _f->vals[7] ) , EXTRA ); + case A_FUNCTION_DEF_RAW_1: + return CHECK_VERSION ( 5 , "Async functions are" , _Py_AsyncFunctionDef ( ((expr_ty)_f->vals[2]) -> v . Name . id , ( _f->vals[4] ) ? _f->vals[4] : CHECK ( _PyPegen_empty_arguments ( p ) ) , ((asdl_seq*)_f->vals[9]) , NULL , _f->vals[6] , NEW_TYPE_COMMENT ( p , _f->vals[8] ) , EXTRA ) ); + case A_FUNC_TYPE_COMMENT_0: + case A_GROUP_0: + case A__TMP_55_0: + case A__TMP_79_0: + return _f->vals[1]; + case A_PARAMS_1: + return ((arguments_ty)_f->vals[0]); + case A_PARAMETERS_0: + case A_LAMBDA_PARAMETERS_0: + return _PyPegen_make_arguments ( p , ((asdl_seq*)_f->vals[0]) , NULL , _f->vals[1] , _f->vals[2] , _f->vals[3] ); + case A_PARAMETERS_1: + case A_LAMBDA_PARAMETERS_1: + return _PyPegen_make_arguments ( p , NULL , ((SlashWithDefault*)_f->vals[0]) , NULL , _f->vals[1] , _f->vals[2] ); + case A_PARAMETERS_2: + case A_LAMBDA_PARAMETERS_2: + return _PyPegen_make_arguments ( p , NULL , NULL , _f->vals[0] , _f->vals[1] , _f->vals[2] ); + case A_PARAMETERS_3: + case A_LAMBDA_PARAMETERS_3: + return _PyPegen_make_arguments ( p , NULL , NULL , NULL , _f->vals[0] , _f->vals[1] ); + case A_PARAMETERS_4: + case A_LAMBDA_PARAMETERS_4: + return _PyPegen_make_arguments ( p , NULL , NULL , NULL , NULL , ((StarEtc*)_f->vals[0]) ); + case A_SLASH_WITH_DEFAULT_0: + case A_SLASH_WITH_DEFAULT_1: + case A_LAMBDA_SLASH_WITH_DEFAULT_0: + case A_LAMBDA_SLASH_WITH_DEFAULT_1: + return _PyPegen_slash_with_default ( p , _f->vals[0] , _f->vals[1] ); + case A_STAR_ETC_0: + case A_LAMBDA_STAR_ETC_0: + return _PyPegen_star_etc ( p , ((arg_ty)_f->vals[1]) , _f->vals[2] , _f->vals[3] ); + case A_STAR_ETC_1: + case A_LAMBDA_STAR_ETC_1: + return _PyPegen_star_etc ( p , NULL , _f->vals[2] , _f->vals[3] ); + case A_STAR_ETC_2: + case A_LAMBDA_STAR_ETC_2: + return _PyPegen_star_etc ( p , NULL , NULL , ((arg_ty)_f->vals[0]) ); + case A_KWDS_0: + case A_LAMBDA_KWDS_0: + return ((arg_ty)_f->vals[1]); + case A_PARAM_NO_DEFAULT_0: + return _PyPegen_add_type_comment_to_arg ( p , ((arg_ty)_f->vals[0]) , _f->vals[2] ); + case A_PARAM_NO_DEFAULT_1: + return _PyPegen_add_type_comment_to_arg ( p , ((arg_ty)_f->vals[0]) , _f->vals[1] ); + case A_PARAM_WITH_DEFAULT_0: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , ((expr_ty)_f->vals[1]) , _f->vals[3] ); + case A_PARAM_WITH_DEFAULT_1: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , ((expr_ty)_f->vals[1]) , _f->vals[2] ); + case A_PARAM_MAYBE_DEFAULT_0: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , _f->vals[1] , _f->vals[3] ); + case A_PARAM_MAYBE_DEFAULT_1: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , _f->vals[1] , _f->vals[2] ); + case A_PARAM_0: + return _Py_arg ( ((expr_ty)_f->vals[0]) -> v . Name . id , _f->vals[1] , NULL , EXTRA ); + case A_ANNOTATION_0: + case A_DEFAULT_0: + case A_SINGLE_TARGET_2: + case A__TMP_14_0: + case A__TMP_15_0: + case A__TMP_16_0: + case A__TMP_22_0: + case A__TMP_26_0: + case A__TMP_28_0: + case A__TMP_33_0: + case A__TMP_35_0: + case A__TMP_36_0: + case A__TMP_37_0: + case A__TMP_38_0: + case A__TMP_90_0: + case A__TMP_91_0: + case A__TMP_114_0: + case A__TMP_115_0: + case A__TMP_116_0: + case A__TMP_117_0: + case A__TMP_118_0: + case A__TMP_119_0: + case A__TMP_120_0: + case A__TMP_121_0: + return ((expr_ty)_f->vals[1]); + case A_CLASS_DEF_0: + return _PyPegen_class_def_decorators ( p , ((asdl_seq*)_f->vals[0]) , ((stmt_ty)_f->vals[1]) ); + case A_CLASS_DEF_RAW_0: + return _Py_ClassDef ( ((expr_ty)_f->vals[1]) -> v . Name . id , ( _f->vals[2] ) ? ( ( expr_ty ) _f->vals[2] ) -> v . Call . args : NULL , ( _f->vals[2] ) ? ( ( expr_ty ) _f->vals[2] ) -> v . Call . keywords : NULL , ((asdl_seq*)_f->vals[4]) , NULL , EXTRA ); + case A_STAR_EXPRESSIONS_0: + case A_EXPRESSIONS_0: + return _Py_Tuple ( CHECK ( _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[1] ) ) , Load , EXTRA ); + case A_STAR_EXPRESSIONS_1: + case A_EXPRESSIONS_1: + return _Py_Tuple ( CHECK ( _PyPegen_singleton_seq ( p , ((expr_ty)_f->vals[0]) ) ) , Load , EXTRA ); + case A_STAR_EXPRESSION_0: + case A_STAR_NAMED_EXPRESSION_0: + case A_STARRED_EXPRESSION_0: + return _Py_Starred ( ((expr_ty)_f->vals[1]) , Load , EXTRA ); + case A_NAMED_EXPRESSION_0: + return _Py_NamedExpr ( CHECK ( _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Store ) ) , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_EXPRESSION_0: + return _Py_IfExp ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[4]) , EXTRA ); + case A_LAMBDEF_0: + return _Py_Lambda ( ( _f->vals[1] ) ? _f->vals[1] : CHECK ( _PyPegen_empty_arguments ( p ) ) , ((expr_ty)_f->vals[3]) , EXTRA ); + case A_LAMBDA_PARAM_NO_DEFAULT_0: + case A_LAMBDA_PARAM_NO_DEFAULT_1: + return ((arg_ty)_f->vals[0]); + case A_LAMBDA_PARAM_WITH_DEFAULT_0: + case A_LAMBDA_PARAM_WITH_DEFAULT_1: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , ((expr_ty)_f->vals[1]) , NULL ); + case A_LAMBDA_PARAM_MAYBE_DEFAULT_0: + case A_LAMBDA_PARAM_MAYBE_DEFAULT_1: + return _PyPegen_name_default_pair ( p , ((arg_ty)_f->vals[0]) , _f->vals[1] , NULL ); + case A_LAMBDA_PARAM_0: + return _Py_arg ( ((expr_ty)_f->vals[0]) -> v . Name . id , NULL , NULL , EXTRA ); + case A_DISJUNCTION_0: + return _Py_BoolOp ( Or , CHECK ( _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[1] ) ) , EXTRA ); + case A_CONJUNCTION_0: + return _Py_BoolOp ( And , CHECK ( _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[1] ) ) , EXTRA ); + case A_INVERSION_0: + return _Py_UnaryOp ( Not , ((expr_ty)_f->vals[1]) , EXTRA ); + case A_COMPARISON_0: + return _Py_Compare ( ((expr_ty)_f->vals[0]) , CHECK ( _PyPegen_get_cmpops ( p , _f->vals[1] ) ) , CHECK ( _PyPegen_get_exprs ( p , _f->vals[1] ) ) , EXTRA ); + case A_COMPARE_OP_BITWISE_OR_PAIR_0: + case A_COMPARE_OP_BITWISE_OR_PAIR_1: + case A_COMPARE_OP_BITWISE_OR_PAIR_2: + case A_COMPARE_OP_BITWISE_OR_PAIR_3: + case A_COMPARE_OP_BITWISE_OR_PAIR_4: + case A_COMPARE_OP_BITWISE_OR_PAIR_5: + case A_COMPARE_OP_BITWISE_OR_PAIR_6: + case A_COMPARE_OP_BITWISE_OR_PAIR_7: + case A_COMPARE_OP_BITWISE_OR_PAIR_8: + case A_COMPARE_OP_BITWISE_OR_PAIR_9: + return ((CmpopExprPair*)_f->vals[0]); + case A_EQ_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , Eq , ((expr_ty)_f->vals[1]) ); + case A_NOTEQ_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , NotEq , ((expr_ty)_f->vals[1]) ); + case A_LTE_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , LtE , ((expr_ty)_f->vals[1]) ); + case A_LT_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , Lt , ((expr_ty)_f->vals[1]) ); + case A_GTE_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , GtE , ((expr_ty)_f->vals[1]) ); + case A_GT_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , Gt , ((expr_ty)_f->vals[1]) ); + case A_NOTIN_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , NotIn , ((expr_ty)_f->vals[2]) ); + case A_IN_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , In , ((expr_ty)_f->vals[1]) ); + case A_ISNOT_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , IsNot , ((expr_ty)_f->vals[2]) ); + case A_IS_BITWISE_OR_0: + return _PyPegen_cmpop_expr_pair ( p , Is , ((expr_ty)_f->vals[1]) ); + case A_BITWISE_OR_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , BitOr , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_BITWISE_XOR_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , BitXor , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_BITWISE_AND_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , BitAnd , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_SHIFT_EXPR_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , LShift , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_SHIFT_EXPR_1: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , RShift , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_SUM_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Add , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_SUM_1: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Sub , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_TERM_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Mult , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_TERM_1: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Div , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_TERM_2: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , FloorDiv , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_TERM_3: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Mod , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_TERM_4: + return CHECK_VERSION ( 5 , "The '@' operator is" , _Py_BinOp ( ((expr_ty)_f->vals[0]) , MatMult , ((expr_ty)_f->vals[2]) , EXTRA ) ); + case A_FACTOR_0: + return _Py_UnaryOp ( UAdd , ((expr_ty)_f->vals[1]) , EXTRA ); + case A_FACTOR_1: + return _Py_UnaryOp ( USub , ((expr_ty)_f->vals[1]) , EXTRA ); + case A_FACTOR_2: + return _Py_UnaryOp ( Invert , ((expr_ty)_f->vals[1]) , EXTRA ); + case A_POWER_0: + return _Py_BinOp ( ((expr_ty)_f->vals[0]) , Pow , ((expr_ty)_f->vals[2]) , EXTRA ); + case A_AWAIT_PRIMARY_0: + return CHECK_VERSION ( 5 , "Await expressions are" , _Py_Await ( ((expr_ty)_f->vals[1]) , EXTRA ) ); + case A_PRIMARY_0: + case A_T_PRIMARY_0: + return _Py_Attribute ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) -> v . Name . id , Load , EXTRA ); + case A_PRIMARY_1: + case A_T_PRIMARY_2: + return _Py_Call ( ((expr_ty)_f->vals[0]) , CHECK ( _PyPegen_singleton_seq ( p , ((expr_ty)_f->vals[1]) ) ) , NULL , EXTRA ); + case A_PRIMARY_2: + case A_T_PRIMARY_3: + return _Py_Call ( ((expr_ty)_f->vals[0]) , ( _f->vals[2] ) ? ( ( expr_ty ) _f->vals[2] ) -> v . Call . args : NULL , ( _f->vals[2] ) ? ( ( expr_ty ) _f->vals[2] ) -> v . Call . keywords : NULL , EXTRA ); + case A_PRIMARY_3: + case A_T_PRIMARY_1: + return _Py_Subscript ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) , Load , EXTRA ); + case A_SLICES_1: + return _Py_Tuple ( _f->vals[0] , Load , EXTRA ); + case A_SLICE_0: + return _Py_Slice ( _f->vals[0] , _f->vals[2] , _f->vals[3] , EXTRA ); + case A_ATOM_1: + return _Py_Constant ( Py_True , NULL , EXTRA ); + case A_ATOM_2: + return _Py_Constant ( Py_False , NULL , EXTRA ); + case A_ATOM_3: + return _Py_Constant ( Py_None , NULL , EXTRA ); + case A_ATOM_4: + return RAISE_SYNTAX_ERROR ( "You found it!" ); + case A_ATOM_10: + return _Py_Constant ( Py_Ellipsis , NULL , EXTRA ); + case A_STRINGS_0: + return _PyPegen_concatenate_strings ( p , _f->vals[0] ); + case A_LIST_0: + return _Py_List ( _f->vals[1] , Load , EXTRA ); + case A_LISTCOMP_0: + return _Py_ListComp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + case A_TUPLE_0: + return _Py_Tuple ( _f->vals[1] , Load , EXTRA ); + case A_GENEXP_0: + return _Py_GeneratorExp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + case A_SET_0: + return _Py_Set ( ((asdl_seq*)_f->vals[1]) , EXTRA ); + case A_SETCOMP_0: + return _Py_SetComp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + case A_DICT_0: + return _Py_Dict ( CHECK ( _PyPegen_get_keys ( p , _f->vals[1] ) ) , CHECK ( _PyPegen_get_values ( p , _f->vals[1] ) ) , EXTRA ); + case A_DICTCOMP_0: + return _Py_DictComp ( ((KeyValuePair*)_f->vals[1]) -> key , ((KeyValuePair*)_f->vals[1]) -> value , ((asdl_seq*)_f->vals[2]) , EXTRA ); + case A_DOUBLE_STARRED_KVPAIR_0: + return _PyPegen_key_value_pair ( p , NULL , ((expr_ty)_f->vals[1]) ); + case A_DOUBLE_STARRED_KVPAIR_1: + case A__GATHER_86_0: + case A__GATHER_86_1: + return ((KeyValuePair*)_f->vals[0]); + case A_KVPAIR_0: + return _PyPegen_key_value_pair ( p , ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) ); + case A_FOR_IF_CLAUSE_0: + return CHECK_VERSION ( 6 , "Async comprehensions are" , _Py_comprehension ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[4]) , _f->vals[5] , 1 , p -> arena ) ); + case A_FOR_IF_CLAUSE_1: + return _Py_comprehension ( ((expr_ty)_f->vals[1]) , ((expr_ty)_f->vals[3]) , _f->vals[4] , 0 , p -> arena ); + case A_YIELD_EXPR_0: + return _Py_YieldFrom ( ((expr_ty)_f->vals[2]) , EXTRA ); + case A_YIELD_EXPR_1: + return _Py_Yield ( _f->vals[1] , EXTRA ); + case A_ARGS_0: + case A_ARGS_2: + return _Py_Call ( _PyPegen_dummy_name ( p ) , ( _f->vals[1] ) ? CHECK ( _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , ( ( expr_ty ) _f->vals[1] ) -> v . Call . args ) ) : CHECK ( _PyPegen_singleton_seq ( p , ((expr_ty)_f->vals[0]) ) ) , ( _f->vals[1] ) ? ( ( expr_ty ) _f->vals[1] ) -> v . Call . keywords : NULL , EXTRA ); + case A_ARGS_1: + return _Py_Call ( _PyPegen_dummy_name ( p ) , CHECK_NULL_ALLOWED ( _PyPegen_seq_extract_starred_exprs ( p , ((asdl_seq*)_f->vals[0]) ) ) , CHECK_NULL_ALLOWED ( _PyPegen_seq_delete_starred_exprs ( p , ((asdl_seq*)_f->vals[0]) ) ) , EXTRA ); + case A_KWARGS_0: + return _PyPegen_join_sequences ( p , _f->vals[0] , _f->vals[2] ); + case A_KWARG_OR_STARRED_0: + case A_KWARG_OR_DOUBLE_STARRED_0: + return _PyPegen_keyword_or_starred ( p , CHECK ( _Py_keyword ( ((expr_ty)_f->vals[0]) -> v . Name . id , ((expr_ty)_f->vals[2]) , EXTRA ) ) , 1 ); + case A_KWARG_OR_STARRED_1: + return _PyPegen_keyword_or_starred ( p , ((expr_ty)_f->vals[0]) , 0 ); + case A_KWARG_OR_DOUBLE_STARRED_1: + return _PyPegen_keyword_or_starred ( p , CHECK ( _Py_keyword ( NULL , ((expr_ty)_f->vals[1]) , EXTRA ) ) , 1 ); + case A_STAR_TARGETS_1: + return _Py_Tuple ( CHECK ( _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[1] ) ) , Store , EXTRA ); + case A_STAR_TARGET_0: + return _Py_Starred ( CHECK ( _PyPegen_set_expr_context ( p , _f->vals[1] , Store ) ) , Store , EXTRA ); + case A_STAR_TARGET_1: + case A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_0: + case A_TARGET_0: + return _Py_Attribute ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) -> v . Name . id , Store , EXTRA ); + case A_STAR_TARGET_2: + case A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_1: + case A_TARGET_1: + return _Py_Subscript ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) , Store , EXTRA ); + case A_STAR_ATOM_0: + case A_SINGLE_TARGET_1: + case A_T_ATOM_0: + return _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Store ); + case A_STAR_ATOM_1: + case A_T_ATOM_1: + return _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[1]) , Store ); + case A_STAR_ATOM_2: + case A_T_ATOM_2: + return _Py_Tuple ( _f->vals[1] , Store , EXTRA ); + case A_STAR_ATOM_3: + case A_T_ATOM_3: + return _Py_List ( _f->vals[1] , Store , EXTRA ); + case A_DEL_TARGET_0: + return _Py_Attribute ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) -> v . Name . id , Del , EXTRA ); + case A_DEL_TARGET_1: + return _Py_Subscript ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) , Del , EXTRA ); + case A_DEL_T_ATOM_0: + return _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Del ); + case A_DEL_T_ATOM_1: + return _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[1]) , Del ); + case A_DEL_T_ATOM_2: + return _Py_Tuple ( _f->vals[1] , Del , EXTRA ); + case A_DEL_T_ATOM_3: + return _Py_List ( _f->vals[1] , Del , EXTRA ); + case A_DEL_TARGET_END_0: + case A_DEL_TARGET_END_1: + case A_DEL_TARGET_END_2: + case A_DEL_TARGET_END_3: + case A_T_LOOKAHEAD_0: + case A_T_LOOKAHEAD_1: + case A_T_LOOKAHEAD_2: + case A__TMP_9_0: + case A__TMP_9_1: + case A__TMP_10_0: + case A__TMP_10_1: + case A__TMP_11_0: + case A__TMP_11_1: + case A__TMP_12_0: + case A__TMP_13_0: + case A__TMP_103_0: + case A__TMP_106_0: + case A__TMP_106_1: + case A__TMP_106_2: + case A__TMP_109_0: + case A__TMP_109_1: + case A__TMP_110_0: + case A__TMP_110_1: + case A__TMP_112_0: + case A__TMP_112_1: + case A__TMP_113_0: + case A__TMP_113_1: + case A__TMP_123_0: + case A__TMP_123_1: + case A__TMP_124_0: + case A__TMP_124_1: + return ((Token *)_f->vals[0]); + case A_INCORRECT_ARGUMENTS_0: + return RAISE_SYNTAX_ERROR ( "iterable argument unpacking follows keyword argument unpacking" ); + case A_INCORRECT_ARGUMENTS_1: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "Generator expression must be parenthesized" ); + case A_INCORRECT_ARGUMENTS_2: + return _PyPegen_nonparen_genexp_in_call ( p , ((expr_ty)_f->vals[0]) ); + case A_INCORRECT_ARGUMENTS_3: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[2]) , "Generator expression must be parenthesized" ); + case A_INCORRECT_ARGUMENTS_4: + return _PyPegen_arguments_parsing_error ( p , ((expr_ty)_f->vals[0]) ); + case A_INVALID_KWARG_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "expression cannot contain assignment, perhaps you meant \"==\"?" ); + case A_INVALID_NAMED_EXPRESSION_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); + case A_INVALID_ASSIGNMENT_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "only single target (not list) can be annotated" ); + case A_INVALID_ASSIGNMENT_1: + case A_INVALID_ASSIGNMENT_2: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "only single target (not tuple) can be annotated" ); + case A_INVALID_ASSIGNMENT_3: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "illegal target for annotation" ); + case A_INVALID_ASSIGNMENT_4: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( _PyPegen_get_invalid_target ( ((expr_ty)_f->vals[0]) ) , "cannot assign to %s" , _PyPegen_get_expr_name ( _PyPegen_get_invalid_target ( ((expr_ty)_f->vals[0]) ) ) ); + case A_INVALID_ASSIGNMENT_5: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "'%s' is an illegal expression for augmented assignment" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); + case A_INVALID_BLOCK_0: + return RAISE_INDENTATION_ERROR ( "expected an indented block" ); + case A_INVALID_COMPREHENSION_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[1]) , "iterable unpacking cannot be used in comprehension" ); + case A_INVALID_DICT_COMPREHENSION_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((Token *)_f->vals[1]) , "dict unpacking cannot be used in dict comprehension" ); + case A_INVALID_PARAMETERS_0: + return RAISE_SYNTAX_ERROR ( "non-default argument follows default argument" ); + case A_INVALID_STAR_ETC_0: + case A_INVALID_LAMBDA_STAR_ETC_0: + return RAISE_SYNTAX_ERROR ( "named arguments must follow bare *" ); + case A_INVALID_STAR_ETC_1: + return RAISE_SYNTAX_ERROR ( "bare * has associated type comment" ); + case A_INVALID_DOUBLE_TYPE_COMMENTS_0: + return RAISE_SYNTAX_ERROR ( "Cannot have two type comments on def" ); + case A_INVALID_DEL_TARGET_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "cannot delete %s" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); + case A_INVALID_IMPORT_FROM_TARGETS_0: + return RAISE_SYNTAX_ERROR ( "trailing comma not allowed without surrounding parentheses" ); + case A__GATHER_25_0: + case A__GATHER_25_1: + case A__GATHER_27_0: + case A__GATHER_27_1: + return ((alias_ty)_f->vals[0]); + case A__GATHER_29_0: + case A__GATHER_29_1: + case A__GATHER_30_0: + case A__GATHER_30_1: + case A__GATHER_31_0: + case A__GATHER_31_1: + case A__GATHER_32_0: + case A__GATHER_32_1: + return ((withitem_ty)_f->vals[0]); + case A__TMP_77_0: + return _PyPegen_check_barry_as_flufl ( p ) ? NULL : ((Token *)_f->vals[0]); + case A__TMP_84_0: + return _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[2] ); + case A__GATHER_92_0: + case A__GATHER_92_1: + case A__GATHER_93_0: + case A__GATHER_93_1: + case A__GATHER_94_0: + case A__GATHER_94_1: + case A__GATHER_95_0: + case A__GATHER_95_1: + return ((KeywordOrStarred*)_f->vals[0]); + case A__TMP_108_0: + return ((SlashWithDefault*)_f->vals[0]); default: assert(0); RAISE_SYNTAX_ERROR("invalid opcode"); diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 7dc41a0e900177..53b16172ed06b6 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -235,15 +235,13 @@ def _setup_keywords(self) -> None: def _setup_soft_keywords(self) -> None: soft_keywords = self.callmakervisitor.soft_keyword_cache - if not soft_keywords: - return - - self.print("enum {") - with self.indent(): - for soft_keyword in soft_keywords: - self.print(f"SK_{soft_keyword.upper()},") - self.print("};") - self.print() + if soft_keywords: + self.print("enum {") + with self.indent(): + for soft_keyword in soft_keywords: + self.print(f"SK_{soft_keyword.upper()},") + self.print("};") + self.print() self.print("static const char *soft_keywords[] = {") with self.indent(): for soft_keyword in soft_keywords: @@ -278,10 +276,9 @@ def translate_action(self, alt: Alt) -> str: if not alt.action: # TODO: Restore the assert, but expect index == 2 in Gather ##assert index == 1, "Alternative with >1 item must have an action" - return "_f->vals[0]" + return self.make_typed_var(alt, 0) # Sadly, the action is given as a string, so tokenize it back. # We must not substitute item names when preceded by '.' or '->'. - # Note that Python tokenizes '->' as two separate tokens. res = [] prevs = "" for stuff in tokenize.generate_tokens(iter([alt.action]).__next__): @@ -289,7 +286,7 @@ def translate_action(self, alt: Alt) -> str: if prevs not in (".", "->"): i = name_to_index.get(s) if i is not None: - s = f"_f->vals[{i}]" + s = self.make_typed_var(alt, i) res.append(s) prevs = s return " ".join(res).strip() @@ -304,10 +301,28 @@ def map_alt_names_to_vals_index(self, alt: Alt) -> Tuple[Dict[str, int], int]: index += 1 return map, index + def make_typed_var(self, alt: Alt, index: int) -> str: + var = f"_f->vals[{index}]" + type = self.get_type_of_indexed_item(alt, index) + if type: + var = f"(({type}){var})" + return var + + def get_type_of_indexed_item(self, alt: Alt, index: int) -> Optional[str]: + item = alt.items[index].item + if isinstance(item, NameLeaf): + if item.value in self.rules: + return self.rules[item.value].type + if item.value == "NAME": + return "expr_ty" + if isinstance(item, StringLeaf): + return "Token *" + return None + def add_root_rules(self) -> None: assert "root" not in self.todo assert "root" not in self.rules - root = RootRule("root", "start") # TODO: determine start rules dynamically + root = RootRule("root", "file") # TODO: determine start rules dynamically self.todo["root"] = root self.rules["root"] = root @@ -469,7 +484,7 @@ def visit_Gather(self, node: Gather) -> None: def main() -> None: - filename = "./data/simple.gram" + filename = "../../Grammar/python.gram" if sys.argv[1:]: filename = sys.argv[1] grammar, parser, tokenizer = build_parser(filename, False, False) From 10f7be11812ab3b45dc137f6585900e4a76d48d9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 3 Jun 2020 16:08:04 +0100 Subject: [PATCH 58/67] Group every opcode with its argument (#131) --- Parser/pegen/vmparse.h | 3673 +++++++++++++++++---- Tools/peg_generator/pegen/vm_generator.py | 111 +- 2 files changed, 3156 insertions(+), 628 deletions(-) diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 8e724ae309f82b..88c9855fca4afa 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -851,2275 +851,4770 @@ static Rule all_rules[] = { R_FILE, {0, -1}, { - OP_RULE, R_STATEMENTS, OP_OPTIONAL, OP_TOKEN, ENDMARKER, OP_RETURN, A_FILE_0, + // statements? $ + OP_RULE, R_STATEMENTS, + OP_OPTIONAL, + OP_TOKEN, ENDMARKER, + OP_RETURN, A_FILE_0, + }, }, {"interactive", R_INTERACTIVE, {0, -1}, { - OP_RULE, R_STATEMENT_NEWLINE, OP_RETURN, A_INTERACTIVE_0, + // statement_newline + OP_RULE, R_STATEMENT_NEWLINE, + OP_RETURN, A_INTERACTIVE_0, + }, }, {"eval", R_EVAL, {0, -1}, { - OP_RULE, R_EXPRESSIONS, OP_RULE, R__LOOP0_1, OP_TOKEN, ENDMARKER, OP_RETURN, A_EVAL_0, + // expressions NEWLINE* $ + OP_RULE, R_EXPRESSIONS, + OP_RULE, R__LOOP0_1, + OP_TOKEN, ENDMARKER, + OP_RETURN, A_EVAL_0, + }, }, {"func_type", R_FUNC_TYPE, {0, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_TYPE_EXPRESSIONS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RULE, R__LOOP0_2, OP_TOKEN, ENDMARKER, OP_RETURN, A_FUNC_TYPE_0, + // '(' type_expressions? ')' '->' expression NEWLINE* $ + OP_TOKEN, LPAR, + OP_RULE, R_TYPE_EXPRESSIONS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_TOKEN, RARROW, + OP_RULE, R_EXPRESSION, + OP_RULE, R__LOOP0_2, + OP_TOKEN, ENDMARKER, + OP_RETURN, A_FUNC_TYPE_0, + }, }, {"fstring", R_FSTRING, {0, -1}, { - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_FSTRING_0, + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A_FSTRING_0, + }, }, {"type_expressions", R_TYPE_EXPRESSIONS, {0, 16, 26, 36, 48, 54, 60, -1}, { - OP_RULE, R__GATHER_3, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_0, - OP_RULE, R__GATHER_4, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_1, - OP_RULE, R__GATHER_5, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_2, - OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_3, - OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_4, - OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_TYPE_EXPRESSIONS_5, - OP_RULE, R__GATHER_6, OP_RETURN, A_TYPE_EXPRESSIONS_6, + // ','.expression+ ',' '*' expression ',' '**' expression + OP_RULE, R__GATHER_3, + OP_TOKEN, COMMA, + OP_TOKEN, STAR, + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_0, + + // ','.expression+ ',' '*' expression + OP_RULE, R__GATHER_4, + OP_TOKEN, COMMA, + OP_TOKEN, STAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_1, + + // ','.expression+ ',' '**' expression + OP_RULE, R__GATHER_5, + OP_TOKEN, COMMA, + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_2, + + // '*' expression ',' '**' expression + OP_TOKEN, STAR, + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_3, + + // '*' expression + OP_TOKEN, STAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_4, + + // '**' expression + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_TYPE_EXPRESSIONS_5, + + // ','.expression+ + OP_RULE, R__GATHER_6, + OP_RETURN, A_TYPE_EXPRESSIONS_6, + }, }, {"statements", R_STATEMENTS, {0, -1}, { - OP_RULE, R__LOOP1_7, OP_RETURN, A_STATEMENTS_0, + // statement+ + OP_RULE, R__LOOP1_7, + OP_RETURN, A_STATEMENTS_0, + }, }, {"statement", R_STATEMENT, {0, 4, -1}, { - OP_RULE, R_COMPOUND_STMT, OP_RETURN, A_STATEMENT_0, - OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_STATEMENT_1, + // compound_stmt + OP_RULE, R_COMPOUND_STMT, + OP_RETURN, A_STATEMENT_0, + + // simple_stmt + OP_RULE, R_SIMPLE_STMT, + OP_RETURN, A_STATEMENT_1, + }, }, {"statement_newline", R_STATEMENT_NEWLINE, {0, 6, 10, 14, -1}, { - OP_RULE, R_COMPOUND_STMT, OP_TOKEN, NEWLINE, OP_RETURN, A_STATEMENT_NEWLINE_0, - OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_STATEMENT_NEWLINE_1, - OP_TOKEN, NEWLINE, OP_RETURN, A_STATEMENT_NEWLINE_2, - OP_TOKEN, ENDMARKER, OP_RETURN, A_STATEMENT_NEWLINE_3, + // compound_stmt NEWLINE + OP_RULE, R_COMPOUND_STMT, + OP_TOKEN, NEWLINE, + OP_RETURN, A_STATEMENT_NEWLINE_0, + + // simple_stmt + OP_RULE, R_SIMPLE_STMT, + OP_RETURN, A_STATEMENT_NEWLINE_1, + + // NEWLINE + OP_TOKEN, NEWLINE, + OP_RETURN, A_STATEMENT_NEWLINE_2, + + // $ + OP_TOKEN, ENDMARKER, + OP_RETURN, A_STATEMENT_NEWLINE_3, + }, }, {"simple_stmt", R_SIMPLE_STMT, {0, 10, -1}, { - OP_RULE, R_SMALL_STMT, OP_SAVE_MARK, OP_TOKEN, SEMI, OP_NEG_LOOKAHEAD, OP_TOKEN, NEWLINE, OP_RETURN, A_SIMPLE_STMT_0, - OP_RULE, R__GATHER_8, OP_TOKEN, SEMI, OP_OPTIONAL, OP_TOKEN, NEWLINE, OP_RETURN, A_SIMPLE_STMT_1, + // small_stmt !';' NEWLINE + OP_RULE, R_SMALL_STMT, + OP_SAVE_MARK, + OP_TOKEN, SEMI, + OP_NEG_LOOKAHEAD, + OP_TOKEN, NEWLINE, + OP_RETURN, A_SIMPLE_STMT_0, + + // ';'.small_stmt+ ';'? NEWLINE + OP_RULE, R__GATHER_8, + OP_TOKEN, SEMI, + OP_OPTIONAL, + OP_TOKEN, NEWLINE, + OP_RETURN, A_SIMPLE_STMT_1, + }, }, {"small_stmt", R_SMALL_STMT, {0, 4, 8, 16, 24, 32, 36, 44, 52, 60, 64, 68, 76, -1}, { - OP_RULE, R_ASSIGNMENT, OP_RETURN, A_SMALL_STMT_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_SMALL_STMT_1, - OP_SAVE_MARK, OP_TOKEN, 500, OP_POS_LOOKAHEAD, OP_RULE, R_RETURN_STMT, OP_RETURN, A_SMALL_STMT_2, - OP_SAVE_MARK, OP_RULE, R__TMP_9, OP_POS_LOOKAHEAD, OP_RULE, R_IMPORT_STMT, OP_RETURN, A_SMALL_STMT_3, - OP_SAVE_MARK, OP_TOKEN, 501, OP_POS_LOOKAHEAD, OP_RULE, R_RAISE_STMT, OP_RETURN, A_SMALL_STMT_4, - OP_TOKEN, 502, OP_RETURN, A_SMALL_STMT_5, - OP_SAVE_MARK, OP_TOKEN, 503, OP_POS_LOOKAHEAD, OP_RULE, R_DEL_STMT, OP_RETURN, A_SMALL_STMT_6, - OP_SAVE_MARK, OP_TOKEN, 504, OP_POS_LOOKAHEAD, OP_RULE, R_YIELD_STMT, OP_RETURN, A_SMALL_STMT_7, - OP_SAVE_MARK, OP_TOKEN, 505, OP_POS_LOOKAHEAD, OP_RULE, R_ASSERT_STMT, OP_RETURN, A_SMALL_STMT_8, - OP_TOKEN, 506, OP_RETURN, A_SMALL_STMT_9, - OP_TOKEN, 507, OP_RETURN, A_SMALL_STMT_10, - OP_SAVE_MARK, OP_TOKEN, 508, OP_POS_LOOKAHEAD, OP_RULE, R_GLOBAL_STMT, OP_RETURN, A_SMALL_STMT_11, - OP_SAVE_MARK, OP_TOKEN, 509, OP_POS_LOOKAHEAD, OP_RULE, R_NONLOCAL_STMT, OP_RETURN, A_SMALL_STMT_12, + // assignment + OP_RULE, R_ASSIGNMENT, + OP_RETURN, A_SMALL_STMT_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A_SMALL_STMT_1, + + // &'return' return_stmt + OP_SAVE_MARK, + OP_TOKEN, 500, + OP_POS_LOOKAHEAD, + OP_RULE, R_RETURN_STMT, + OP_RETURN, A_SMALL_STMT_2, + + // &('import' | 'from') import_stmt + OP_SAVE_MARK, + OP_RULE, R__TMP_9, + OP_POS_LOOKAHEAD, + OP_RULE, R_IMPORT_STMT, + OP_RETURN, A_SMALL_STMT_3, + + // &'raise' raise_stmt + OP_SAVE_MARK, + OP_TOKEN, 501, + OP_POS_LOOKAHEAD, + OP_RULE, R_RAISE_STMT, + OP_RETURN, A_SMALL_STMT_4, + + // 'pass' + OP_TOKEN, 502, + OP_RETURN, A_SMALL_STMT_5, + + // &'del' del_stmt + OP_SAVE_MARK, + OP_TOKEN, 503, + OP_POS_LOOKAHEAD, + OP_RULE, R_DEL_STMT, + OP_RETURN, A_SMALL_STMT_6, + + // &'yield' yield_stmt + OP_SAVE_MARK, + OP_TOKEN, 504, + OP_POS_LOOKAHEAD, + OP_RULE, R_YIELD_STMT, + OP_RETURN, A_SMALL_STMT_7, + + // &'assert' assert_stmt + OP_SAVE_MARK, + OP_TOKEN, 505, + OP_POS_LOOKAHEAD, + OP_RULE, R_ASSERT_STMT, + OP_RETURN, A_SMALL_STMT_8, + + // 'break' + OP_TOKEN, 506, + OP_RETURN, A_SMALL_STMT_9, + + // 'continue' + OP_TOKEN, 507, + OP_RETURN, A_SMALL_STMT_10, + + // &'global' global_stmt + OP_SAVE_MARK, + OP_TOKEN, 508, + OP_POS_LOOKAHEAD, + OP_RULE, R_GLOBAL_STMT, + OP_RETURN, A_SMALL_STMT_11, + + // &'nonlocal' nonlocal_stmt + OP_SAVE_MARK, + OP_TOKEN, 509, + OP_POS_LOOKAHEAD, + OP_RULE, R_NONLOCAL_STMT, + OP_RETURN, A_SMALL_STMT_12, + }, }, {"compound_stmt", R_COMPOUND_STMT, {0, 8, 16, 24, 32, 40, 48, -1}, { - OP_SAVE_MARK, OP_RULE, R__TMP_10, OP_POS_LOOKAHEAD, OP_RULE, R_FUNCTION_DEF, OP_RETURN, A_COMPOUND_STMT_0, - OP_SAVE_MARK, OP_TOKEN, 510, OP_POS_LOOKAHEAD, OP_RULE, R_IF_STMT, OP_RETURN, A_COMPOUND_STMT_1, - OP_SAVE_MARK, OP_RULE, R__TMP_11, OP_POS_LOOKAHEAD, OP_RULE, R_CLASS_DEF, OP_RETURN, A_COMPOUND_STMT_2, - OP_SAVE_MARK, OP_RULE, R__TMP_12, OP_POS_LOOKAHEAD, OP_RULE, R_WITH_STMT, OP_RETURN, A_COMPOUND_STMT_3, - OP_SAVE_MARK, OP_RULE, R__TMP_13, OP_POS_LOOKAHEAD, OP_RULE, R_FOR_STMT, OP_RETURN, A_COMPOUND_STMT_4, - OP_SAVE_MARK, OP_TOKEN, 511, OP_POS_LOOKAHEAD, OP_RULE, R_TRY_STMT, OP_RETURN, A_COMPOUND_STMT_5, - OP_SAVE_MARK, OP_TOKEN, 512, OP_POS_LOOKAHEAD, OP_RULE, R_WHILE_STMT, OP_RETURN, A_COMPOUND_STMT_6, + // &('def' | '@' | ASYNC) function_def + OP_SAVE_MARK, + OP_RULE, R__TMP_10, + OP_POS_LOOKAHEAD, + OP_RULE, R_FUNCTION_DEF, + OP_RETURN, A_COMPOUND_STMT_0, + + // &'if' if_stmt + OP_SAVE_MARK, + OP_TOKEN, 510, + OP_POS_LOOKAHEAD, + OP_RULE, R_IF_STMT, + OP_RETURN, A_COMPOUND_STMT_1, + + // &('class' | '@') class_def + OP_SAVE_MARK, + OP_RULE, R__TMP_11, + OP_POS_LOOKAHEAD, + OP_RULE, R_CLASS_DEF, + OP_RETURN, A_COMPOUND_STMT_2, + + // &('with' | ASYNC) with_stmt + OP_SAVE_MARK, + OP_RULE, R__TMP_12, + OP_POS_LOOKAHEAD, + OP_RULE, R_WITH_STMT, + OP_RETURN, A_COMPOUND_STMT_3, + + // &('for' | ASYNC) for_stmt + OP_SAVE_MARK, + OP_RULE, R__TMP_13, + OP_POS_LOOKAHEAD, + OP_RULE, R_FOR_STMT, + OP_RETURN, A_COMPOUND_STMT_4, + + // &'try' try_stmt + OP_SAVE_MARK, + OP_TOKEN, 511, + OP_POS_LOOKAHEAD, + OP_RULE, R_TRY_STMT, + OP_RETURN, A_COMPOUND_STMT_5, + + // &'while' while_stmt + OP_SAVE_MARK, + OP_TOKEN, 512, + OP_POS_LOOKAHEAD, + OP_RULE, R_WHILE_STMT, + OP_RETURN, A_COMPOUND_STMT_6, + }, }, {"assignment", R_ASSIGNMENT, {0, 10, 21, 30, 38, -1}, { - OP_NAME, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_14, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_0, - OP_RULE, R__TMP_15, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_16, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_1, - OP_RULE, R__LOOP1_17, OP_RULE, R__TMP_18, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_2, - OP_RULE, R_SINGLE_TARGET, OP_RULE, R_AUGASSIGN, OP_RULE, R__TMP_19, OP_RETURN, A_ASSIGNMENT_3, - OP_RULE, R_INVALID_ASSIGNMENT, OP_RETURN, A_ASSIGNMENT_4, + // NAME ':' expression ['=' annotated_rhs] + OP_NAME, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_14, + OP_OPTIONAL, + OP_RETURN, A_ASSIGNMENT_0, + + // ('(' single_target ')' | single_subscript_attribute_target) ':' expression ['=' annotated_rhs] + OP_RULE, R__TMP_15, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_16, + OP_OPTIONAL, + OP_RETURN, A_ASSIGNMENT_1, + + // ((star_targets '='))+ (yield_expr | star_expressions) TYPE_COMMENT? + OP_RULE, R__LOOP1_17, + OP_RULE, R__TMP_18, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RETURN, A_ASSIGNMENT_2, + + // single_target augassign (yield_expr | star_expressions) + OP_RULE, R_SINGLE_TARGET, + OP_RULE, R_AUGASSIGN, + OP_RULE, R__TMP_19, + OP_RETURN, A_ASSIGNMENT_3, + + // invalid_assignment + OP_RULE, R_INVALID_ASSIGNMENT, + OP_RETURN, A_ASSIGNMENT_4, + }, }, {"augassign", R_AUGASSIGN, {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, -1}, { - OP_TOKEN, PLUSEQUAL, OP_RETURN, A_AUGASSIGN_0, - OP_TOKEN, MINEQUAL, OP_RETURN, A_AUGASSIGN_1, - OP_TOKEN, STAREQUAL, OP_RETURN, A_AUGASSIGN_2, - OP_TOKEN, ATEQUAL, OP_RETURN, A_AUGASSIGN_3, - OP_TOKEN, SLASHEQUAL, OP_RETURN, A_AUGASSIGN_4, - OP_TOKEN, PERCENTEQUAL, OP_RETURN, A_AUGASSIGN_5, - OP_TOKEN, AMPEREQUAL, OP_RETURN, A_AUGASSIGN_6, - OP_TOKEN, VBAREQUAL, OP_RETURN, A_AUGASSIGN_7, - OP_TOKEN, CIRCUMFLEXEQUAL, OP_RETURN, A_AUGASSIGN_8, - OP_TOKEN, LEFTSHIFTEQUAL, OP_RETURN, A_AUGASSIGN_9, - OP_TOKEN, RIGHTSHIFTEQUAL, OP_RETURN, A_AUGASSIGN_10, - OP_TOKEN, DOUBLESTAREQUAL, OP_RETURN, A_AUGASSIGN_11, - OP_TOKEN, DOUBLESLASHEQUAL, OP_RETURN, A_AUGASSIGN_12, + // '+=' + OP_TOKEN, PLUSEQUAL, + OP_RETURN, A_AUGASSIGN_0, + + // '-=' + OP_TOKEN, MINEQUAL, + OP_RETURN, A_AUGASSIGN_1, + + // '*=' + OP_TOKEN, STAREQUAL, + OP_RETURN, A_AUGASSIGN_2, + + // '@=' + OP_TOKEN, ATEQUAL, + OP_RETURN, A_AUGASSIGN_3, + + // '/=' + OP_TOKEN, SLASHEQUAL, + OP_RETURN, A_AUGASSIGN_4, + + // '%=' + OP_TOKEN, PERCENTEQUAL, + OP_RETURN, A_AUGASSIGN_5, + + // '&=' + OP_TOKEN, AMPEREQUAL, + OP_RETURN, A_AUGASSIGN_6, + + // '|=' + OP_TOKEN, VBAREQUAL, + OP_RETURN, A_AUGASSIGN_7, + + // '^=' + OP_TOKEN, CIRCUMFLEXEQUAL, + OP_RETURN, A_AUGASSIGN_8, + + // '<<=' + OP_TOKEN, LEFTSHIFTEQUAL, + OP_RETURN, A_AUGASSIGN_9, + + // '>>=' + OP_TOKEN, RIGHTSHIFTEQUAL, + OP_RETURN, A_AUGASSIGN_10, + + // '**=' + OP_TOKEN, DOUBLESTAREQUAL, + OP_RETURN, A_AUGASSIGN_11, + + // '//=' + OP_TOKEN, DOUBLESLASHEQUAL, + OP_RETURN, A_AUGASSIGN_12, + }, }, {"global_stmt", R_GLOBAL_STMT, {0, -1}, { - OP_TOKEN, 508, OP_RULE, R__GATHER_20, OP_RETURN, A_GLOBAL_STMT_0, + // 'global' ','.NAME+ + OP_TOKEN, 508, + OP_RULE, R__GATHER_20, + OP_RETURN, A_GLOBAL_STMT_0, + }, }, {"nonlocal_stmt", R_NONLOCAL_STMT, {0, -1}, { - OP_TOKEN, 509, OP_RULE, R__GATHER_21, OP_RETURN, A_NONLOCAL_STMT_0, + // 'nonlocal' ','.NAME+ + OP_TOKEN, 509, + OP_RULE, R__GATHER_21, + OP_RETURN, A_NONLOCAL_STMT_0, + }, }, {"yield_stmt", R_YIELD_STMT, {0, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A_YIELD_STMT_0, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A_YIELD_STMT_0, + }, }, {"assert_stmt", R_ASSERT_STMT, {0, -1}, { - OP_TOKEN, 505, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_22, OP_OPTIONAL, OP_RETURN, A_ASSERT_STMT_0, + // 'assert' expression [',' expression] + OP_TOKEN, 505, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_22, + OP_OPTIONAL, + OP_RETURN, A_ASSERT_STMT_0, + }, }, {"del_stmt", R_DEL_STMT, {0, -1}, { - OP_TOKEN, 503, OP_RULE, R_DEL_TARGETS, OP_RETURN, A_DEL_STMT_0, + // 'del' del_targets + OP_TOKEN, 503, + OP_RULE, R_DEL_TARGETS, + OP_RETURN, A_DEL_STMT_0, + }, }, {"import_stmt", R_IMPORT_STMT, {0, 4, -1}, { - OP_RULE, R_IMPORT_NAME, OP_RETURN, A_IMPORT_STMT_0, - OP_RULE, R_IMPORT_FROM, OP_RETURN, A_IMPORT_STMT_1, + // import_name + OP_RULE, R_IMPORT_NAME, + OP_RETURN, A_IMPORT_STMT_0, + + // import_from + OP_RULE, R_IMPORT_FROM, + OP_RETURN, A_IMPORT_STMT_1, + }, }, {"import_name", R_IMPORT_NAME, {0, -1}, { - OP_TOKEN, 513, OP_RULE, R_DOTTED_AS_NAMES, OP_RETURN, A_IMPORT_NAME_0, + // 'import' dotted_as_names + OP_TOKEN, 513, + OP_RULE, R_DOTTED_AS_NAMES, + OP_RETURN, A_IMPORT_NAME_0, + }, }, {"import_from", R_IMPORT_FROM, {0, 12, -1}, { - OP_TOKEN, 514, OP_RULE, R__LOOP0_23, OP_RULE, R_DOTTED_NAME, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_0, - OP_TOKEN, 514, OP_RULE, R__LOOP1_24, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_1, + // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets + OP_TOKEN, 514, + OP_RULE, R__LOOP0_23, + OP_RULE, R_DOTTED_NAME, + OP_TOKEN, 513, + OP_RULE, R_IMPORT_FROM_TARGETS, + OP_RETURN, A_IMPORT_FROM_0, + + // 'from' (('.' | '...'))+ 'import' import_from_targets + OP_TOKEN, 514, + OP_RULE, R__LOOP1_24, + OP_TOKEN, 513, + OP_RULE, R_IMPORT_FROM_TARGETS, + OP_RETURN, A_IMPORT_FROM_1, + }, }, {"import_from_targets", R_IMPORT_FROM_TARGETS, {0, 11, 19, 23, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_IMPORT_FROM_TARGETS_0, - OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_IMPORT_FROM_TARGETS_1, - OP_TOKEN, STAR, OP_RETURN, A_IMPORT_FROM_TARGETS_2, - OP_RULE, R_INVALID_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_TARGETS_3, + // '(' import_from_as_names ','? ')' + OP_TOKEN, LPAR, + OP_RULE, R_IMPORT_FROM_AS_NAMES, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A_IMPORT_FROM_TARGETS_0, + + // import_from_as_names !',' + OP_RULE, R_IMPORT_FROM_AS_NAMES, + OP_SAVE_MARK, + OP_TOKEN, COMMA, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_IMPORT_FROM_TARGETS_1, + + // '*' + OP_TOKEN, STAR, + OP_RETURN, A_IMPORT_FROM_TARGETS_2, + + // invalid_import_from_targets + OP_RULE, R_INVALID_IMPORT_FROM_TARGETS, + OP_RETURN, A_IMPORT_FROM_TARGETS_3, + }, }, {"import_from_as_names", R_IMPORT_FROM_AS_NAMES, {0, -1}, { - OP_RULE, R__GATHER_25, OP_RETURN, A_IMPORT_FROM_AS_NAMES_0, + // ','.import_from_as_name+ + OP_RULE, R__GATHER_25, + OP_RETURN, A_IMPORT_FROM_AS_NAMES_0, + }, }, {"import_from_as_name", R_IMPORT_FROM_AS_NAME, {0, -1}, { - OP_NAME, OP_RULE, R__TMP_26, OP_OPTIONAL, OP_RETURN, A_IMPORT_FROM_AS_NAME_0, + // NAME ['as' NAME] + OP_NAME, + OP_RULE, R__TMP_26, + OP_OPTIONAL, + OP_RETURN, A_IMPORT_FROM_AS_NAME_0, + }, }, {"dotted_as_names", R_DOTTED_AS_NAMES, {0, -1}, { - OP_RULE, R__GATHER_27, OP_RETURN, A_DOTTED_AS_NAMES_0, + // ','.dotted_as_name+ + OP_RULE, R__GATHER_27, + OP_RETURN, A_DOTTED_AS_NAMES_0, + }, }, {"dotted_as_name", R_DOTTED_AS_NAME, {0, -1}, { - OP_RULE, R_DOTTED_NAME, OP_RULE, R__TMP_28, OP_OPTIONAL, OP_RETURN, A_DOTTED_AS_NAME_0, + // dotted_name ['as' NAME] + OP_RULE, R_DOTTED_NAME, + OP_RULE, R__TMP_28, + OP_OPTIONAL, + OP_RETURN, A_DOTTED_AS_NAME_0, + }, }, {"dotted_name", R_DOTTED_NAME, {0, 8, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_DOTTED_NAME, OP_TOKEN, DOT, OP_NAME, OP_RETURN_LEFT_REC, A_DOTTED_NAME_0, - OP_NAME, OP_RETURN_LEFT_REC, A_DOTTED_NAME_1, + // dotted_name '.' NAME + OP_SETUP_LEFT_REC, + OP_RULE, R_DOTTED_NAME, + OP_TOKEN, DOT, + OP_NAME, + OP_RETURN_LEFT_REC, A_DOTTED_NAME_0, + + // NAME + OP_NAME, + OP_RETURN_LEFT_REC, A_DOTTED_NAME_1, + }, }, {"if_stmt", R_IF_STMT, {0, 12, -1}, { - OP_TOKEN, 510, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELIF_STMT, OP_RETURN, A_IF_STMT_0, - OP_TOKEN, 510, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_IF_STMT_1, + // 'if' named_expression ':' block elif_stmt + OP_TOKEN, 510, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_ELIF_STMT, + OP_RETURN, A_IF_STMT_0, + + // 'if' named_expression ':' block else_block? + OP_TOKEN, 510, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_IF_STMT_1, + }, }, {"elif_stmt", R_ELIF_STMT, {0, 12, -1}, { - OP_TOKEN, 515, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELIF_STMT, OP_RETURN, A_ELIF_STMT_0, - OP_TOKEN, 515, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_ELIF_STMT_1, + // 'elif' named_expression ':' block elif_stmt + OP_TOKEN, 515, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_ELIF_STMT, + OP_RETURN, A_ELIF_STMT_0, + + // 'elif' named_expression ':' block else_block? + OP_TOKEN, 515, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_ELIF_STMT_1, + }, }, {"else_block", R_ELSE_BLOCK, {0, -1}, { - OP_TOKEN, 516, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_ELSE_BLOCK_0, + // 'else' ':' block + OP_TOKEN, 516, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_ELSE_BLOCK_0, + }, }, {"while_stmt", R_WHILE_STMT, {0, -1}, { - OP_TOKEN, 512, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_WHILE_STMT_0, + // 'while' named_expression ':' block else_block? + OP_TOKEN, 512, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_WHILE_STMT_0, + }, }, {"for_stmt", R_FOR_STMT, {0, 20, -1}, { - OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_FOR_STMT_0, - OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RETURN, A_FOR_STMT_1, + // 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? + OP_TOKEN, 517, + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, 518, + OP_RULE, R_STAR_EXPRESSIONS, + OP_TOKEN, COLON, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_FOR_STMT_0, + + // ASYNC 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? + OP_TOKEN, ASYNC, + OP_TOKEN, 517, + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, 518, + OP_RULE, R_STAR_EXPRESSIONS, + OP_TOKEN, COLON, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_FOR_STMT_1, + }, }, {"with_stmt", R_WITH_STMT, {0, 17, 30, 49, -1}, { - OP_TOKEN, 519, OP_TOKEN, LPAR, OP_RULE, R__GATHER_29, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_0, - OP_TOKEN, 519, OP_RULE, R__GATHER_30, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_1, - OP_TOKEN, ASYNC, OP_TOKEN, 519, OP_TOKEN, LPAR, OP_RULE, R__GATHER_31, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_2, - OP_TOKEN, ASYNC, OP_TOKEN, 519, OP_RULE, R__GATHER_32, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_WITH_STMT_3, + // 'with' '(' ','.with_item+ ','? ')' ':' block + OP_TOKEN, 519, + OP_TOKEN, LPAR, + OP_RULE, R__GATHER_29, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_WITH_STMT_0, + + // 'with' ','.with_item+ ':' TYPE_COMMENT? block + OP_TOKEN, 519, + OP_RULE, R__GATHER_30, + OP_TOKEN, COLON, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RETURN, A_WITH_STMT_1, + + // ASYNC 'with' '(' ','.with_item+ ','? ')' ':' block + OP_TOKEN, ASYNC, + OP_TOKEN, 519, + OP_TOKEN, LPAR, + OP_RULE, R__GATHER_31, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_WITH_STMT_2, + + // ASYNC 'with' ','.with_item+ ':' TYPE_COMMENT? block + OP_TOKEN, ASYNC, + OP_TOKEN, 519, + OP_RULE, R__GATHER_32, + OP_TOKEN, COLON, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RETURN, A_WITH_STMT_3, + }, }, {"with_item", R_WITH_ITEM, {0, -1}, { - OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_33, OP_OPTIONAL, OP_RETURN, A_WITH_ITEM_0, + // expression ['as' target] + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_33, + OP_OPTIONAL, + OP_RETURN, A_WITH_ITEM_0, + }, }, {"try_stmt", R_TRY_STMT, {0, 10, -1}, { - OP_TOKEN, 511, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R_FINALLY_BLOCK, OP_RETURN, A_TRY_STMT_0, - OP_TOKEN, 511, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RULE, R__LOOP1_34, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RULE, R_FINALLY_BLOCK, OP_OPTIONAL, OP_RETURN, A_TRY_STMT_1, + // 'try' ':' block finally_block + OP_TOKEN, 511, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R_FINALLY_BLOCK, + OP_RETURN, A_TRY_STMT_0, + + // 'try' ':' block except_block+ else_block? finally_block? + OP_TOKEN, 511, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RULE, R__LOOP1_34, + OP_RULE, R_ELSE_BLOCK, + OP_OPTIONAL, + OP_RULE, R_FINALLY_BLOCK, + OP_OPTIONAL, + OP_RETURN, A_TRY_STMT_1, + }, }, {"except_block", R_EXCEPT_BLOCK, {0, 13, -1}, { - OP_TOKEN, 520, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_35, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_0, - OP_TOKEN, 520, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_1, + // 'except' expression ['as' NAME] ':' block + OP_TOKEN, 520, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_35, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_EXCEPT_BLOCK_0, + + // 'except' ':' block + OP_TOKEN, 520, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_EXCEPT_BLOCK_1, + }, }, {"finally_block", R_FINALLY_BLOCK, {0, -1}, { - OP_TOKEN, 521, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_FINALLY_BLOCK_0, + // 'finally' ':' block + OP_TOKEN, 521, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_FINALLY_BLOCK_0, + }, }, {"return_stmt", R_RETURN_STMT, {0, -1}, { - OP_TOKEN, 500, OP_RULE, R_STAR_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A_RETURN_STMT_0, + // 'return' star_expressions? + OP_TOKEN, 500, + OP_RULE, R_STAR_EXPRESSIONS, + OP_OPTIONAL, + OP_RETURN, A_RETURN_STMT_0, + }, }, {"raise_stmt", R_RAISE_STMT, {0, 9, -1}, { - OP_TOKEN, 501, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_36, OP_OPTIONAL, OP_RETURN, A_RAISE_STMT_0, - OP_TOKEN, 501, OP_RETURN, A_RAISE_STMT_1, + // 'raise' expression ['from' expression] + OP_TOKEN, 501, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_36, + OP_OPTIONAL, + OP_RETURN, A_RAISE_STMT_0, + + // 'raise' + OP_TOKEN, 501, + OP_RETURN, A_RAISE_STMT_1, + }, }, {"function_def", R_FUNCTION_DEF, {0, 6, -1}, { - OP_RULE, R_DECORATORS, OP_RULE, R_FUNCTION_DEF_RAW, OP_RETURN, A_FUNCTION_DEF_0, - OP_RULE, R_FUNCTION_DEF_RAW, OP_RETURN, A_FUNCTION_DEF_1, + // decorators function_def_raw + OP_RULE, R_DECORATORS, + OP_RULE, R_FUNCTION_DEF_RAW, + OP_RETURN, A_FUNCTION_DEF_0, + + // function_def_raw + OP_RULE, R_FUNCTION_DEF_RAW, + OP_RETURN, A_FUNCTION_DEF_1, + }, }, {"function_def_raw", R_FUNCTION_DEF_RAW, {0, 22, -1}, { - OP_TOKEN, 522, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RULE, R__TMP_37, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_FUNCTION_DEF_RAW_0, - OP_TOKEN, ASYNC, OP_TOKEN, 522, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RULE, R__TMP_38, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, OP_OPTIONAL, OP_RULE, R_BLOCK, OP_RETURN, A_FUNCTION_DEF_RAW_1, + // 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block + OP_TOKEN, 522, + OP_NAME, + OP_TOKEN, LPAR, + OP_RULE, R_PARAMS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RULE, R__TMP_37, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_FUNC_TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RETURN, A_FUNCTION_DEF_RAW_0, + + // ASYNC 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block + OP_TOKEN, ASYNC, + OP_TOKEN, 522, + OP_NAME, + OP_TOKEN, LPAR, + OP_RULE, R_PARAMS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RULE, R__TMP_38, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_FUNC_TYPE_COMMENT, + OP_OPTIONAL, + OP_RULE, R_BLOCK, + OP_RETURN, A_FUNCTION_DEF_RAW_1, + }, }, {"func_type_comment", R_FUNC_TYPE_COMMENT, {0, 10, 14, -1}, { - OP_TOKEN, NEWLINE, OP_TOKEN, TYPE_COMMENT, OP_SAVE_MARK, OP_RULE, R__TMP_39, OP_POS_LOOKAHEAD, OP_RETURN, A_FUNC_TYPE_COMMENT_0, - OP_RULE, R_INVALID_DOUBLE_TYPE_COMMENTS, OP_RETURN, A_FUNC_TYPE_COMMENT_1, - OP_TOKEN, TYPE_COMMENT, OP_RETURN, A_FUNC_TYPE_COMMENT_2, + // NEWLINE TYPE_COMMENT &(NEWLINE INDENT) + OP_TOKEN, NEWLINE, + OP_TOKEN, TYPE_COMMENT, + OP_SAVE_MARK, + OP_RULE, R__TMP_39, + OP_POS_LOOKAHEAD, + OP_RETURN, A_FUNC_TYPE_COMMENT_0, + + // invalid_double_type_comments + OP_RULE, R_INVALID_DOUBLE_TYPE_COMMENTS, + OP_RETURN, A_FUNC_TYPE_COMMENT_1, + + // TYPE_COMMENT + OP_TOKEN, TYPE_COMMENT, + OP_RETURN, A_FUNC_TYPE_COMMENT_2, + }, }, {"params", R_PARAMS, {0, 4, -1}, { - OP_RULE, R_INVALID_PARAMETERS, OP_RETURN, A_PARAMS_0, - OP_RULE, R_PARAMETERS, OP_RETURN, A_PARAMS_1, + // invalid_parameters + OP_RULE, R_INVALID_PARAMETERS, + OP_RETURN, A_PARAMS_0, + + // parameters + OP_RULE, R_PARAMETERS, + OP_RETURN, A_PARAMS_1, + }, }, {"parameters", R_PARAMETERS, {0, 11, 20, 29, 36, -1}, { - OP_RULE, R_SLASH_NO_DEFAULT, OP_RULE, R__LOOP0_40, OP_RULE, R__LOOP0_41, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_0, - OP_RULE, R_SLASH_WITH_DEFAULT, OP_RULE, R__LOOP0_42, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_1, - OP_RULE, R__LOOP1_43, OP_RULE, R__LOOP0_44, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_2, - OP_RULE, R__LOOP1_45, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_3, - OP_RULE, R_STAR_ETC, OP_RETURN, A_PARAMETERS_4, + // slash_no_default param_no_default* param_with_default* star_etc? + OP_RULE, R_SLASH_NO_DEFAULT, + OP_RULE, R__LOOP0_40, + OP_RULE, R__LOOP0_41, + OP_RULE, R_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_PARAMETERS_0, + + // slash_with_default param_with_default* star_etc? + OP_RULE, R_SLASH_WITH_DEFAULT, + OP_RULE, R__LOOP0_42, + OP_RULE, R_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_PARAMETERS_1, + + // param_no_default+ param_with_default* star_etc? + OP_RULE, R__LOOP1_43, + OP_RULE, R__LOOP0_44, + OP_RULE, R_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_PARAMETERS_2, + + // param_with_default+ star_etc? + OP_RULE, R__LOOP1_45, + OP_RULE, R_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_PARAMETERS_3, + + // star_etc + OP_RULE, R_STAR_ETC, + OP_RETURN, A_PARAMETERS_4, + }, }, {"slash_no_default", R_SLASH_NO_DEFAULT, {0, 8, -1}, { - OP_RULE, R__LOOP1_46, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_NO_DEFAULT_0, - OP_RULE, R__LOOP1_47, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_SLASH_NO_DEFAULT_1, + // param_no_default+ '/' ',' + OP_RULE, R__LOOP1_46, + OP_TOKEN, SLASH, + OP_TOKEN, COMMA, + OP_RETURN, A_SLASH_NO_DEFAULT_0, + + // param_no_default+ '/' &')' + OP_RULE, R__LOOP1_47, + OP_TOKEN, SLASH, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_SLASH_NO_DEFAULT_1, + }, }, {"slash_with_default", R_SLASH_WITH_DEFAULT, {0, 10, -1}, { - OP_RULE, R__LOOP0_48, OP_RULE, R__LOOP1_49, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_WITH_DEFAULT_0, - OP_RULE, R__LOOP0_50, OP_RULE, R__LOOP1_51, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_SLASH_WITH_DEFAULT_1, + // param_no_default* param_with_default+ '/' ',' + OP_RULE, R__LOOP0_48, + OP_RULE, R__LOOP1_49, + OP_TOKEN, SLASH, + OP_TOKEN, COMMA, + OP_RETURN, A_SLASH_WITH_DEFAULT_0, + + // param_no_default* param_with_default+ '/' &')' + OP_RULE, R__LOOP0_50, + OP_RULE, R__LOOP1_51, + OP_TOKEN, SLASH, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_SLASH_WITH_DEFAULT_1, + }, }, {"star_etc", R_STAR_ETC, {0, 11, 22, 26, -1}, { - OP_TOKEN, STAR, OP_RULE, R_PARAM_NO_DEFAULT, OP_RULE, R__LOOP0_52, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_0, - OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_RULE, R__LOOP1_53, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_1, - OP_RULE, R_KWDS, OP_RETURN, A_STAR_ETC_2, - OP_RULE, R_INVALID_STAR_ETC, OP_RETURN, A_STAR_ETC_3, + // '*' param_no_default param_maybe_default* kwds? + OP_TOKEN, STAR, + OP_RULE, R_PARAM_NO_DEFAULT, + OP_RULE, R__LOOP0_52, + OP_RULE, R_KWDS, + OP_OPTIONAL, + OP_RETURN, A_STAR_ETC_0, + + // '*' ',' param_maybe_default+ kwds? + OP_TOKEN, STAR, + OP_TOKEN, COMMA, + OP_RULE, R__LOOP1_53, + OP_RULE, R_KWDS, + OP_OPTIONAL, + OP_RETURN, A_STAR_ETC_1, + + // kwds + OP_RULE, R_KWDS, + OP_RETURN, A_STAR_ETC_2, + + // invalid_star_etc + OP_RULE, R_INVALID_STAR_ETC, + OP_RETURN, A_STAR_ETC_3, + }, }, {"kwds", R_KWDS, {0, -1}, { - OP_TOKEN, DOUBLESTAR, OP_RULE, R_PARAM_NO_DEFAULT, OP_RETURN, A_KWDS_0, + // '**' param_no_default + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_PARAM_NO_DEFAULT, + OP_RETURN, A_KWDS_0, + }, }, {"param_no_default", R_PARAM_NO_DEFAULT, {0, 9, -1}, { - OP_RULE, R_PARAM, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_NO_DEFAULT_0, - OP_RULE, R_PARAM, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_NO_DEFAULT_1, + // param ',' TYPE_COMMENT? + OP_RULE, R_PARAM, + OP_TOKEN, COMMA, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RETURN, A_PARAM_NO_DEFAULT_0, + + // param TYPE_COMMENT? &')' + OP_RULE, R_PARAM, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_PARAM_NO_DEFAULT_1, + }, }, {"param_with_default", R_PARAM_WITH_DEFAULT, {0, 11, -1}, { - OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_WITH_DEFAULT_0, - OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_WITH_DEFAULT_1, + // param default ',' TYPE_COMMENT? + OP_RULE, R_PARAM, + OP_RULE, R_DEFAULT, + OP_TOKEN, COMMA, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RETURN, A_PARAM_WITH_DEFAULT_0, + + // param default TYPE_COMMENT? &')' + OP_RULE, R_PARAM, + OP_RULE, R_DEFAULT, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_PARAM_WITH_DEFAULT_1, + }, }, {"param_maybe_default", R_PARAM_MAYBE_DEFAULT, {0, 12, -1}, { - OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_PARAM_MAYBE_DEFAULT_0, - OP_RULE, R_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_PARAM_MAYBE_DEFAULT_1, + // param default? ',' TYPE_COMMENT? + OP_RULE, R_PARAM, + OP_RULE, R_DEFAULT, + OP_OPTIONAL, + OP_TOKEN, COMMA, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_RETURN, A_PARAM_MAYBE_DEFAULT_0, + + // param default? TYPE_COMMENT? &')' + OP_RULE, R_PARAM, + OP_RULE, R_DEFAULT, + OP_OPTIONAL, + OP_TOKEN, TYPE_COMMENT, + OP_OPTIONAL, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_PARAM_MAYBE_DEFAULT_1, + }, }, {"param", R_PARAM, {0, -1}, { - OP_NAME, OP_RULE, R_ANNOTATION, OP_OPTIONAL, OP_RETURN, A_PARAM_0, + // NAME annotation? + OP_NAME, + OP_RULE, R_ANNOTATION, + OP_OPTIONAL, + OP_RETURN, A_PARAM_0, + }, }, {"annotation", R_ANNOTATION, {0, -1}, { - OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_ANNOTATION_0, + // ':' expression + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_ANNOTATION_0, + }, }, {"default", R_DEFAULT, {0, -1}, { - OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_DEFAULT_0, + // '=' expression + OP_TOKEN, EQUAL, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_DEFAULT_0, + }, }, {"decorators", R_DECORATORS, {0, -1}, { - OP_RULE, R__LOOP1_54, OP_RETURN, A_DECORATORS_0, + // (('@' named_expression NEWLINE))+ + OP_RULE, R__LOOP1_54, + OP_RETURN, A_DECORATORS_0, + }, }, {"class_def", R_CLASS_DEF, {0, 6, -1}, { - OP_RULE, R_DECORATORS, OP_RULE, R_CLASS_DEF_RAW, OP_RETURN, A_CLASS_DEF_0, - OP_RULE, R_CLASS_DEF_RAW, OP_RETURN, A_CLASS_DEF_1, + // decorators class_def_raw + OP_RULE, R_DECORATORS, + OP_RULE, R_CLASS_DEF_RAW, + OP_RETURN, A_CLASS_DEF_0, + + // class_def_raw + OP_RULE, R_CLASS_DEF_RAW, + OP_RETURN, A_CLASS_DEF_1, + }, }, {"class_def_raw", R_CLASS_DEF_RAW, {0, -1}, { - OP_TOKEN, 523, OP_NAME, OP_RULE, R__TMP_55, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_CLASS_DEF_RAW_0, + // 'class' NAME ['(' arguments? ')'] ':' block + OP_TOKEN, 523, + OP_NAME, + OP_RULE, R__TMP_55, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_BLOCK, + OP_RETURN, A_CLASS_DEF_RAW_0, + }, }, {"block", R_BLOCK, {0, 10, 14, -1}, { - OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RULE, R_STATEMENTS, OP_TOKEN, DEDENT, OP_RETURN, A_BLOCK_0, - OP_RULE, R_SIMPLE_STMT, OP_RETURN, A_BLOCK_1, - OP_RULE, R_INVALID_BLOCK, OP_RETURN, A_BLOCK_2, + // NEWLINE INDENT statements DEDENT + OP_TOKEN, NEWLINE, + OP_TOKEN, INDENT, + OP_RULE, R_STATEMENTS, + OP_TOKEN, DEDENT, + OP_RETURN, A_BLOCK_0, + + // simple_stmt + OP_RULE, R_SIMPLE_STMT, + OP_RETURN, A_BLOCK_1, + + // invalid_block + OP_RULE, R_INVALID_BLOCK, + OP_RETURN, A_BLOCK_2, + }, }, {"expressions_list", R_EXPRESSIONS_LIST, {0, -1}, { - OP_RULE, R__GATHER_56, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_LIST_0, + // ','.star_expression+ ','? + OP_RULE, R__GATHER_56, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_EXPRESSIONS_LIST_0, + }, }, {"star_expressions", R_STAR_EXPRESSIONS, {0, 9, 15, -1}, { - OP_RULE, R_STAR_EXPRESSION, OP_RULE, R__LOOP1_57, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_EXPRESSIONS_0, - OP_RULE, R_STAR_EXPRESSION, OP_TOKEN, COMMA, OP_RETURN, A_STAR_EXPRESSIONS_1, - OP_RULE, R_STAR_EXPRESSION, OP_RETURN, A_STAR_EXPRESSIONS_2, + // star_expression ((',' star_expression))+ ','? + OP_RULE, R_STAR_EXPRESSION, + OP_RULE, R__LOOP1_57, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_STAR_EXPRESSIONS_0, + + // star_expression ',' + OP_RULE, R_STAR_EXPRESSION, + OP_TOKEN, COMMA, + OP_RETURN, A_STAR_EXPRESSIONS_1, + + // star_expression + OP_RULE, R_STAR_EXPRESSION, + OP_RETURN, A_STAR_EXPRESSIONS_2, + }, }, {"star_expression", R_STAR_EXPRESSION, {0, 6, -1}, { - OP_TOKEN, STAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_STAR_EXPRESSION_0, - OP_RULE, R_EXPRESSION, OP_RETURN, A_STAR_EXPRESSION_1, + // '*' bitwise_or + OP_TOKEN, STAR, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_STAR_EXPRESSION_0, + + // expression + OP_RULE, R_EXPRESSION, + OP_RETURN, A_STAR_EXPRESSION_1, + }, }, {"star_named_expressions", R_STAR_NAMED_EXPRESSIONS, {0, -1}, { - OP_RULE, R__GATHER_58, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_NAMED_EXPRESSIONS_0, + // ','.star_named_expression+ ','? + OP_RULE, R__GATHER_58, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_STAR_NAMED_EXPRESSIONS_0, + }, }, {"star_named_expression", R_STAR_NAMED_EXPRESSION, {0, 6, -1}, { - OP_TOKEN, STAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_STAR_NAMED_EXPRESSION_0, - OP_RULE, R_NAMED_EXPRESSION, OP_RETURN, A_STAR_NAMED_EXPRESSION_1, + // '*' bitwise_or + OP_TOKEN, STAR, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_STAR_NAMED_EXPRESSION_0, + + // named_expression + OP_RULE, R_NAMED_EXPRESSION, + OP_RETURN, A_STAR_NAMED_EXPRESSION_1, + }, }, {"named_expression", R_NAMED_EXPRESSION, {0, 7, 15, -1}, { - OP_NAME, OP_TOKEN, COLONEQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_NAMED_EXPRESSION_0, - OP_RULE, R_EXPRESSION, OP_SAVE_MARK, OP_TOKEN, COLONEQUAL, OP_NEG_LOOKAHEAD, OP_RETURN, A_NAMED_EXPRESSION_1, - OP_RULE, R_INVALID_NAMED_EXPRESSION, OP_RETURN, A_NAMED_EXPRESSION_2, + // NAME ':=' expression + OP_NAME, + OP_TOKEN, COLONEQUAL, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_NAMED_EXPRESSION_0, + + // expression !':=' + OP_RULE, R_EXPRESSION, + OP_SAVE_MARK, + OP_TOKEN, COLONEQUAL, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_NAMED_EXPRESSION_1, + + // invalid_named_expression + OP_RULE, R_INVALID_NAMED_EXPRESSION, + OP_RETURN, A_NAMED_EXPRESSION_2, + }, }, {"annotated_rhs", R_ANNOTATED_RHS, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A_ANNOTATED_RHS_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A_ANNOTATED_RHS_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A_ANNOTATED_RHS_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A_ANNOTATED_RHS_1, + }, }, {"expressions", R_EXPRESSIONS, {0, 9, 15, -1}, { - OP_RULE, R_EXPRESSION, OP_RULE, R__LOOP1_59, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_0, - OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_RETURN, A_EXPRESSIONS_1, - OP_RULE, R_EXPRESSION, OP_RETURN, A_EXPRESSIONS_2, + // expression ((',' expression))+ ','? + OP_RULE, R_EXPRESSION, + OP_RULE, R__LOOP1_59, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_EXPRESSIONS_0, + + // expression ',' + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_RETURN, A_EXPRESSIONS_1, + + // expression + OP_RULE, R_EXPRESSION, + OP_RETURN, A_EXPRESSIONS_2, + }, }, {"expression", R_EXPRESSION, {0, 12, 16, -1}, { - OP_RULE, R_DISJUNCTION, OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_TOKEN, 516, OP_RULE, R_EXPRESSION, OP_RETURN, A_EXPRESSION_0, - OP_RULE, R_DISJUNCTION, OP_RETURN, A_EXPRESSION_1, - OP_RULE, R_LAMBDEF, OP_RETURN, A_EXPRESSION_2, + // disjunction 'if' disjunction 'else' expression + OP_RULE, R_DISJUNCTION, + OP_TOKEN, 510, + OP_RULE, R_DISJUNCTION, + OP_TOKEN, 516, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_EXPRESSION_0, + + // disjunction + OP_RULE, R_DISJUNCTION, + OP_RETURN, A_EXPRESSION_1, + + // lambdef + OP_RULE, R_LAMBDEF, + OP_RETURN, A_EXPRESSION_2, + }, }, {"lambdef", R_LAMBDEF, {0, -1}, { - OP_TOKEN, 524, OP_RULE, R_LAMBDA_PARAMETERS, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_LAMBDEF_0, + // 'lambda' lambda_parameters? ':' expression + OP_TOKEN, 524, + OP_RULE, R_LAMBDA_PARAMETERS, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_LAMBDEF_0, + }, }, {"lambda_parameters", R_LAMBDA_PARAMETERS, {0, 11, 20, 29, 36, -1}, { - OP_RULE, R_LAMBDA_SLASH_NO_DEFAULT, OP_RULE, R__LOOP0_60, OP_RULE, R__LOOP0_61, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_0, - OP_RULE, R_LAMBDA_SLASH_WITH_DEFAULT, OP_RULE, R__LOOP0_62, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_1, - OP_RULE, R__LOOP1_63, OP_RULE, R__LOOP0_64, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_2, - OP_RULE, R__LOOP1_65, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_3, - OP_RULE, R_LAMBDA_STAR_ETC, OP_RETURN, A_LAMBDA_PARAMETERS_4, + // lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc? + OP_RULE, R_LAMBDA_SLASH_NO_DEFAULT, + OP_RULE, R__LOOP0_60, + OP_RULE, R__LOOP0_61, + OP_RULE, R_LAMBDA_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_PARAMETERS_0, + + // lambda_slash_with_default lambda_param_with_default* lambda_star_etc? + OP_RULE, R_LAMBDA_SLASH_WITH_DEFAULT, + OP_RULE, R__LOOP0_62, + OP_RULE, R_LAMBDA_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_PARAMETERS_1, + + // lambda_param_no_default+ lambda_param_with_default* lambda_star_etc? + OP_RULE, R__LOOP1_63, + OP_RULE, R__LOOP0_64, + OP_RULE, R_LAMBDA_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_PARAMETERS_2, + + // lambda_param_with_default+ lambda_star_etc? + OP_RULE, R__LOOP1_65, + OP_RULE, R_LAMBDA_STAR_ETC, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_PARAMETERS_3, + + // lambda_star_etc + OP_RULE, R_LAMBDA_STAR_ETC, + OP_RETURN, A_LAMBDA_PARAMETERS_4, + }, }, {"lambda_slash_no_default", R_LAMBDA_SLASH_NO_DEFAULT, {0, 8, -1}, { - OP_RULE, R__LOOP1_66, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_0, - OP_RULE, R__LOOP1_67, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_1, + // lambda_param_no_default+ '/' ',' + OP_RULE, R__LOOP1_66, + OP_TOKEN, SLASH, + OP_TOKEN, COMMA, + OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_0, + + // lambda_param_no_default+ '/' &':' + OP_RULE, R__LOOP1_67, + OP_TOKEN, SLASH, + OP_SAVE_MARK, + OP_TOKEN, COLON, + OP_POS_LOOKAHEAD, + OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_1, + }, }, {"lambda_slash_with_default", R_LAMBDA_SLASH_WITH_DEFAULT, {0, 10, -1}, { - OP_RULE, R__LOOP0_68, OP_RULE, R__LOOP1_69, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_0, - OP_RULE, R__LOOP0_70, OP_RULE, R__LOOP1_71, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_1, + // lambda_param_no_default* lambda_param_with_default+ '/' ',' + OP_RULE, R__LOOP0_68, + OP_RULE, R__LOOP1_69, + OP_TOKEN, SLASH, + OP_TOKEN, COMMA, + OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_0, + + // lambda_param_no_default* lambda_param_with_default+ '/' &':' + OP_RULE, R__LOOP0_70, + OP_RULE, R__LOOP1_71, + OP_TOKEN, SLASH, + OP_SAVE_MARK, + OP_TOKEN, COLON, + OP_POS_LOOKAHEAD, + OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_1, + }, }, {"lambda_star_etc", R_LAMBDA_STAR_ETC, {0, 11, 22, 26, -1}, { - OP_TOKEN, STAR, OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_RULE, R__LOOP0_72, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_0, - OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_RULE, R__LOOP1_73, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_1, - OP_RULE, R_LAMBDA_KWDS, OP_RETURN, A_LAMBDA_STAR_ETC_2, - OP_RULE, R_INVALID_LAMBDA_STAR_ETC, OP_RETURN, A_LAMBDA_STAR_ETC_3, + // '*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds? + OP_TOKEN, STAR, + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_RULE, R__LOOP0_72, + OP_RULE, R_LAMBDA_KWDS, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_STAR_ETC_0, + + // '*' ',' lambda_param_maybe_default+ lambda_kwds? + OP_TOKEN, STAR, + OP_TOKEN, COMMA, + OP_RULE, R__LOOP1_73, + OP_RULE, R_LAMBDA_KWDS, + OP_OPTIONAL, + OP_RETURN, A_LAMBDA_STAR_ETC_1, + + // lambda_kwds + OP_RULE, R_LAMBDA_KWDS, + OP_RETURN, A_LAMBDA_STAR_ETC_2, + + // invalid_lambda_star_etc + OP_RULE, R_INVALID_LAMBDA_STAR_ETC, + OP_RETURN, A_LAMBDA_STAR_ETC_3, + }, }, {"lambda_kwds", R_LAMBDA_KWDS, {0, -1}, { - OP_TOKEN, DOUBLESTAR, OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_RETURN, A_LAMBDA_KWDS_0, + // '**' lambda_param_no_default + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_RETURN, A_LAMBDA_KWDS_0, + }, }, {"lambda_param_no_default", R_LAMBDA_PARAM_NO_DEFAULT, {0, 6, -1}, { - OP_RULE, R_LAMBDA_PARAM, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_0, - OP_RULE, R_LAMBDA_PARAM, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_1, + // lambda_param ',' + OP_RULE, R_LAMBDA_PARAM, + OP_TOKEN, COMMA, + OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_0, + + // lambda_param &':' + OP_RULE, R_LAMBDA_PARAM, + OP_SAVE_MARK, + OP_TOKEN, COLON, + OP_POS_LOOKAHEAD, + OP_RETURN, A_LAMBDA_PARAM_NO_DEFAULT_1, + }, }, {"lambda_param_with_default", R_LAMBDA_PARAM_WITH_DEFAULT, {0, 8, -1}, { - OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_0, - OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_1, + // lambda_param default ',' + OP_RULE, R_LAMBDA_PARAM, + OP_RULE, R_DEFAULT, + OP_TOKEN, COMMA, + OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_0, + + // lambda_param default &':' + OP_RULE, R_LAMBDA_PARAM, + OP_RULE, R_DEFAULT, + OP_SAVE_MARK, + OP_TOKEN, COLON, + OP_POS_LOOKAHEAD, + OP_RETURN, A_LAMBDA_PARAM_WITH_DEFAULT_1, + }, }, {"lambda_param_maybe_default", R_LAMBDA_PARAM_MAYBE_DEFAULT, {0, 9, -1}, { - OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_0, - OP_RULE, R_LAMBDA_PARAM, OP_RULE, R_DEFAULT, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, COLON, OP_POS_LOOKAHEAD, OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_1, + // lambda_param default? ',' + OP_RULE, R_LAMBDA_PARAM, + OP_RULE, R_DEFAULT, + OP_OPTIONAL, + OP_TOKEN, COMMA, + OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_0, + + // lambda_param default? &':' + OP_RULE, R_LAMBDA_PARAM, + OP_RULE, R_DEFAULT, + OP_OPTIONAL, + OP_SAVE_MARK, + OP_TOKEN, COLON, + OP_POS_LOOKAHEAD, + OP_RETURN, A_LAMBDA_PARAM_MAYBE_DEFAULT_1, + }, }, {"lambda_param", R_LAMBDA_PARAM, {0, -1}, { - OP_NAME, OP_RETURN, A_LAMBDA_PARAM_0, + // NAME + OP_NAME, + OP_RETURN, A_LAMBDA_PARAM_0, + }, }, {"disjunction", R_DISJUNCTION, {0, 6, -1}, { - OP_RULE, R_CONJUNCTION, OP_RULE, R__LOOP1_74, OP_RETURN, A_DISJUNCTION_0, - OP_RULE, R_CONJUNCTION, OP_RETURN, A_DISJUNCTION_1, + // conjunction (('or' conjunction))+ + OP_RULE, R_CONJUNCTION, + OP_RULE, R__LOOP1_74, + OP_RETURN, A_DISJUNCTION_0, + + // conjunction + OP_RULE, R_CONJUNCTION, + OP_RETURN, A_DISJUNCTION_1, + }, }, {"conjunction", R_CONJUNCTION, {0, 6, -1}, { - OP_RULE, R_INVERSION, OP_RULE, R__LOOP1_75, OP_RETURN, A_CONJUNCTION_0, - OP_RULE, R_INVERSION, OP_RETURN, A_CONJUNCTION_1, + // inversion (('and' inversion))+ + OP_RULE, R_INVERSION, + OP_RULE, R__LOOP1_75, + OP_RETURN, A_CONJUNCTION_0, + + // inversion + OP_RULE, R_INVERSION, + OP_RETURN, A_CONJUNCTION_1, + }, }, {"inversion", R_INVERSION, {0, 6, -1}, { - OP_TOKEN, 525, OP_RULE, R_INVERSION, OP_RETURN, A_INVERSION_0, - OP_RULE, R_COMPARISON, OP_RETURN, A_INVERSION_1, + // 'not' inversion + OP_TOKEN, 525, + OP_RULE, R_INVERSION, + OP_RETURN, A_INVERSION_0, + + // comparison + OP_RULE, R_COMPARISON, + OP_RETURN, A_INVERSION_1, + }, }, {"comparison", R_COMPARISON, {0, 6, -1}, { - OP_RULE, R_BITWISE_OR, OP_RULE, R__LOOP1_76, OP_RETURN, A_COMPARISON_0, - OP_RULE, R_BITWISE_OR, OP_RETURN, A_COMPARISON_1, + // bitwise_or compare_op_bitwise_or_pair+ + OP_RULE, R_BITWISE_OR, + OP_RULE, R__LOOP1_76, + OP_RETURN, A_COMPARISON_0, + + // bitwise_or + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_COMPARISON_1, + }, }, {"compare_op_bitwise_or_pair", R_COMPARE_OP_BITWISE_OR_PAIR, {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, -1}, { - OP_RULE, R_EQ_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_0, - OP_RULE, R_NOTEQ_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_1, - OP_RULE, R_LTE_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_2, - OP_RULE, R_LT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_3, - OP_RULE, R_GTE_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_4, - OP_RULE, R_GT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_5, - OP_RULE, R_NOTIN_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_6, - OP_RULE, R_IN_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_7, - OP_RULE, R_ISNOT_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_8, - OP_RULE, R_IS_BITWISE_OR, OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_9, + // eq_bitwise_or + OP_RULE, R_EQ_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_0, + + // noteq_bitwise_or + OP_RULE, R_NOTEQ_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_1, + + // lte_bitwise_or + OP_RULE, R_LTE_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_2, + + // lt_bitwise_or + OP_RULE, R_LT_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_3, + + // gte_bitwise_or + OP_RULE, R_GTE_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_4, + + // gt_bitwise_or + OP_RULE, R_GT_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_5, + + // notin_bitwise_or + OP_RULE, R_NOTIN_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_6, + + // in_bitwise_or + OP_RULE, R_IN_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_7, + + // isnot_bitwise_or + OP_RULE, R_ISNOT_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_8, + + // is_bitwise_or + OP_RULE, R_IS_BITWISE_OR, + OP_RETURN, A_COMPARE_OP_BITWISE_OR_PAIR_9, + }, }, {"eq_bitwise_or", R_EQ_BITWISE_OR, {0, -1}, { - OP_TOKEN, EQEQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_EQ_BITWISE_OR_0, + // '==' bitwise_or + OP_TOKEN, EQEQUAL, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_EQ_BITWISE_OR_0, + }, }, {"noteq_bitwise_or", R_NOTEQ_BITWISE_OR, {0, -1}, { - OP_RULE, R__TMP_77, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTEQ_BITWISE_OR_0, + // ('!=') bitwise_or + OP_RULE, R__TMP_77, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_NOTEQ_BITWISE_OR_0, + }, }, {"lte_bitwise_or", R_LTE_BITWISE_OR, {0, -1}, { - OP_TOKEN, LESSEQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_LTE_BITWISE_OR_0, + // '<=' bitwise_or + OP_TOKEN, LESSEQUAL, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_LTE_BITWISE_OR_0, + }, }, {"lt_bitwise_or", R_LT_BITWISE_OR, {0, -1}, { - OP_TOKEN, LESS, OP_RULE, R_BITWISE_OR, OP_RETURN, A_LT_BITWISE_OR_0, + // '<' bitwise_or + OP_TOKEN, LESS, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_LT_BITWISE_OR_0, + }, }, {"gte_bitwise_or", R_GTE_BITWISE_OR, {0, -1}, { - OP_TOKEN, GREATEREQUAL, OP_RULE, R_BITWISE_OR, OP_RETURN, A_GTE_BITWISE_OR_0, + // '>=' bitwise_or + OP_TOKEN, GREATEREQUAL, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_GTE_BITWISE_OR_0, + }, }, {"gt_bitwise_or", R_GT_BITWISE_OR, {0, -1}, { - OP_TOKEN, GREATER, OP_RULE, R_BITWISE_OR, OP_RETURN, A_GT_BITWISE_OR_0, + // '>' bitwise_or + OP_TOKEN, GREATER, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_GT_BITWISE_OR_0, + }, }, {"notin_bitwise_or", R_NOTIN_BITWISE_OR, {0, -1}, { - OP_TOKEN, 525, OP_TOKEN, 518, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTIN_BITWISE_OR_0, + // 'not' 'in' bitwise_or + OP_TOKEN, 525, + OP_TOKEN, 518, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_NOTIN_BITWISE_OR_0, + }, }, {"in_bitwise_or", R_IN_BITWISE_OR, {0, -1}, { - OP_TOKEN, 518, OP_RULE, R_BITWISE_OR, OP_RETURN, A_IN_BITWISE_OR_0, + // 'in' bitwise_or + OP_TOKEN, 518, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_IN_BITWISE_OR_0, + }, }, {"isnot_bitwise_or", R_ISNOT_BITWISE_OR, {0, -1}, { - OP_TOKEN, 526, OP_TOKEN, 525, OP_RULE, R_BITWISE_OR, OP_RETURN, A_ISNOT_BITWISE_OR_0, + // 'is' 'not' bitwise_or + OP_TOKEN, 526, + OP_TOKEN, 525, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_ISNOT_BITWISE_OR_0, + }, }, {"is_bitwise_or", R_IS_BITWISE_OR, {0, -1}, { - OP_TOKEN, 526, OP_RULE, R_BITWISE_OR, OP_RETURN, A_IS_BITWISE_OR_0, + // 'is' bitwise_or + OP_TOKEN, 526, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_IS_BITWISE_OR_0, + }, }, {"bitwise_or", R_BITWISE_OR, {0, 9, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_OR, OP_TOKEN, VBAR, OP_RULE, R_BITWISE_XOR, OP_RETURN_LEFT_REC, A_BITWISE_OR_0, - OP_RULE, R_BITWISE_XOR, OP_RETURN_LEFT_REC, A_BITWISE_OR_1, + // bitwise_or '|' bitwise_xor + OP_SETUP_LEFT_REC, + OP_RULE, R_BITWISE_OR, + OP_TOKEN, VBAR, + OP_RULE, R_BITWISE_XOR, + OP_RETURN_LEFT_REC, A_BITWISE_OR_0, + + // bitwise_xor + OP_RULE, R_BITWISE_XOR, + OP_RETURN_LEFT_REC, A_BITWISE_OR_1, + }, }, {"bitwise_xor", R_BITWISE_XOR, {0, 9, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_XOR, OP_TOKEN, CIRCUMFLEX, OP_RULE, R_BITWISE_AND, OP_RETURN_LEFT_REC, A_BITWISE_XOR_0, - OP_RULE, R_BITWISE_AND, OP_RETURN_LEFT_REC, A_BITWISE_XOR_1, + // bitwise_xor '^' bitwise_and + OP_SETUP_LEFT_REC, + OP_RULE, R_BITWISE_XOR, + OP_TOKEN, CIRCUMFLEX, + OP_RULE, R_BITWISE_AND, + OP_RETURN_LEFT_REC, A_BITWISE_XOR_0, + + // bitwise_and + OP_RULE, R_BITWISE_AND, + OP_RETURN_LEFT_REC, A_BITWISE_XOR_1, + }, }, {"bitwise_and", R_BITWISE_AND, {0, 9, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_AND, OP_TOKEN, AMPER, OP_RULE, R_SHIFT_EXPR, OP_RETURN_LEFT_REC, A_BITWISE_AND_0, - OP_RULE, R_SHIFT_EXPR, OP_RETURN_LEFT_REC, A_BITWISE_AND_1, + // bitwise_and '&' shift_expr + OP_SETUP_LEFT_REC, + OP_RULE, R_BITWISE_AND, + OP_TOKEN, AMPER, + OP_RULE, R_SHIFT_EXPR, + OP_RETURN_LEFT_REC, A_BITWISE_AND_0, + + // shift_expr + OP_RULE, R_SHIFT_EXPR, + OP_RETURN_LEFT_REC, A_BITWISE_AND_1, + }, }, {"shift_expr", R_SHIFT_EXPR, {0, 9, 17, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_SHIFT_EXPR, OP_TOKEN, LEFTSHIFT, OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_0, - OP_RULE, R_SHIFT_EXPR, OP_TOKEN, RIGHTSHIFT, OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_1, - OP_RULE, R_SUM, OP_RETURN_LEFT_REC, A_SHIFT_EXPR_2, + // shift_expr '<<' sum + OP_SETUP_LEFT_REC, + OP_RULE, R_SHIFT_EXPR, + OP_TOKEN, LEFTSHIFT, + OP_RULE, R_SUM, + OP_RETURN_LEFT_REC, A_SHIFT_EXPR_0, + + // shift_expr '>>' sum + OP_RULE, R_SHIFT_EXPR, + OP_TOKEN, RIGHTSHIFT, + OP_RULE, R_SUM, + OP_RETURN_LEFT_REC, A_SHIFT_EXPR_1, + + // sum + OP_RULE, R_SUM, + OP_RETURN_LEFT_REC, A_SHIFT_EXPR_2, + }, }, {"sum", R_SUM, {0, 9, 17, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_SUM, OP_TOKEN, PLUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_0, - OP_RULE, R_SUM, OP_TOKEN, MINUS, OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_1, - OP_RULE, R_TERM, OP_RETURN_LEFT_REC, A_SUM_2, + // sum '+' term + OP_SETUP_LEFT_REC, + OP_RULE, R_SUM, + OP_TOKEN, PLUS, + OP_RULE, R_TERM, + OP_RETURN_LEFT_REC, A_SUM_0, + + // sum '-' term + OP_RULE, R_SUM, + OP_TOKEN, MINUS, + OP_RULE, R_TERM, + OP_RETURN_LEFT_REC, A_SUM_1, + + // term + OP_RULE, R_TERM, + OP_RETURN_LEFT_REC, A_SUM_2, + }, }, {"term", R_TERM, {0, 9, 17, 25, 33, 41, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_0, - OP_RULE, R_TERM, OP_TOKEN, SLASH, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_1, - OP_RULE, R_TERM, OP_TOKEN, DOUBLESLASH, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_2, - OP_RULE, R_TERM, OP_TOKEN, PERCENT, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_3, - OP_RULE, R_TERM, OP_TOKEN, AT, OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_4, - OP_RULE, R_FACTOR, OP_RETURN_LEFT_REC, A_TERM_5, + // term '*' factor + OP_SETUP_LEFT_REC, + OP_RULE, R_TERM, + OP_TOKEN, STAR, + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_0, + + // term '/' factor + OP_RULE, R_TERM, + OP_TOKEN, SLASH, + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_1, + + // term '//' factor + OP_RULE, R_TERM, + OP_TOKEN, DOUBLESLASH, + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_2, + + // term '%' factor + OP_RULE, R_TERM, + OP_TOKEN, PERCENT, + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_3, + + // term '@' factor + OP_RULE, R_TERM, + OP_TOKEN, AT, + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_4, + + // factor + OP_RULE, R_FACTOR, + OP_RETURN_LEFT_REC, A_TERM_5, + }, }, {"factor", R_FACTOR, {0, 6, 12, 18, -1}, { - OP_TOKEN, PLUS, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_0, - OP_TOKEN, MINUS, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_1, - OP_TOKEN, TILDE, OP_RULE, R_FACTOR, OP_RETURN, A_FACTOR_2, - OP_RULE, R_POWER, OP_RETURN, A_FACTOR_3, + // '+' factor + OP_TOKEN, PLUS, + OP_RULE, R_FACTOR, + OP_RETURN, A_FACTOR_0, + + // '-' factor + OP_TOKEN, MINUS, + OP_RULE, R_FACTOR, + OP_RETURN, A_FACTOR_1, + + // '~' factor + OP_TOKEN, TILDE, + OP_RULE, R_FACTOR, + OP_RETURN, A_FACTOR_2, + + // power + OP_RULE, R_POWER, + OP_RETURN, A_FACTOR_3, + }, }, {"power", R_POWER, {0, 8, -1}, { - OP_RULE, R_AWAIT_PRIMARY, OP_TOKEN, DOUBLESTAR, OP_RULE, R_FACTOR, OP_RETURN, A_POWER_0, - OP_RULE, R_AWAIT_PRIMARY, OP_RETURN, A_POWER_1, + // await_primary '**' factor + OP_RULE, R_AWAIT_PRIMARY, + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_FACTOR, + OP_RETURN, A_POWER_0, + + // await_primary + OP_RULE, R_AWAIT_PRIMARY, + OP_RETURN, A_POWER_1, + }, }, {"await_primary", R_AWAIT_PRIMARY, {0, 6, -1}, { - OP_TOKEN, AWAIT, OP_RULE, R_PRIMARY, OP_RETURN, A_AWAIT_PRIMARY_0, - OP_RULE, R_PRIMARY, OP_RETURN, A_AWAIT_PRIMARY_1, + // AWAIT primary + OP_TOKEN, AWAIT, + OP_RULE, R_PRIMARY, + OP_RETURN, A_AWAIT_PRIMARY_0, + + // primary + OP_RULE, R_PRIMARY, + OP_RETURN, A_AWAIT_PRIMARY_1, + }, }, {"primary", R_PRIMARY, {0, 8, 14, 25, 35, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_RETURN_LEFT_REC, A_PRIMARY_0, - OP_RULE, R_PRIMARY, OP_RULE, R_GENEXP, OP_RETURN_LEFT_REC, A_PRIMARY_1, - OP_RULE, R_PRIMARY, OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN_LEFT_REC, A_PRIMARY_2, - OP_RULE, R_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_RETURN_LEFT_REC, A_PRIMARY_3, - OP_RULE, R_ATOM, OP_RETURN_LEFT_REC, A_PRIMARY_4, + // primary '.' NAME + OP_SETUP_LEFT_REC, + OP_RULE, R_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_RETURN_LEFT_REC, A_PRIMARY_0, + + // primary genexp + OP_RULE, R_PRIMARY, + OP_RULE, R_GENEXP, + OP_RETURN_LEFT_REC, A_PRIMARY_1, + + // primary '(' arguments? ')' + OP_RULE, R_PRIMARY, + OP_TOKEN, LPAR, + OP_RULE, R_ARGUMENTS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN_LEFT_REC, A_PRIMARY_2, + + // primary '[' slices ']' + OP_RULE, R_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_RETURN_LEFT_REC, A_PRIMARY_3, + + // atom + OP_RULE, R_ATOM, + OP_RETURN_LEFT_REC, A_PRIMARY_4, + }, }, {"slices", R_SLICES, {0, 8, -1}, { - OP_RULE, R_SLICE, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_SLICES_0, - OP_RULE, R__GATHER_78, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_SLICES_1, + // slice !',' + OP_RULE, R_SLICE, + OP_SAVE_MARK, + OP_TOKEN, COMMA, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_SLICES_0, + + // ','.slice+ ','? + OP_RULE, R__GATHER_78, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_SLICES_1, + }, }, {"slice", R_SLICE, {0, 13, -1}, { - OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_RULE, R__TMP_79, OP_OPTIONAL, OP_RETURN, A_SLICE_0, - OP_RULE, R_EXPRESSION, OP_RETURN, A_SLICE_1, + // expression? ':' expression? [':' expression?] + OP_RULE, R_EXPRESSION, + OP_OPTIONAL, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_OPTIONAL, + OP_RULE, R__TMP_79, + OP_OPTIONAL, + OP_RETURN, A_SLICE_0, + + // expression + OP_RULE, R_EXPRESSION, + OP_RETURN, A_SLICE_1, + }, }, {"atom", R_ATOM, {0, 3, 7, 11, 15, 19, 26, 29, 37, 45, 53, -1}, { - OP_NAME, OP_RETURN, A_ATOM_0, - OP_TOKEN, 527, OP_RETURN, A_ATOM_1, - OP_TOKEN, 528, OP_RETURN, A_ATOM_2, - OP_TOKEN, 529, OP_RETURN, A_ATOM_3, - OP_TOKEN, 530, OP_RETURN, A_ATOM_4, - OP_SAVE_MARK, OP_STRING, OP_POS_LOOKAHEAD, OP_RULE, R_STRINGS, OP_RETURN, A_ATOM_5, - OP_NUMBER, OP_RETURN, A_ATOM_6, - OP_SAVE_MARK, OP_TOKEN, LPAR, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_80, OP_RETURN, A_ATOM_7, - OP_SAVE_MARK, OP_TOKEN, LSQB, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_81, OP_RETURN, A_ATOM_8, - OP_SAVE_MARK, OP_TOKEN, LBRACE, OP_POS_LOOKAHEAD, OP_RULE, R__TMP_82, OP_RETURN, A_ATOM_9, - OP_TOKEN, ELLIPSIS, OP_RETURN, A_ATOM_10, + // NAME + OP_NAME, + OP_RETURN, A_ATOM_0, + + // 'True' + OP_TOKEN, 527, + OP_RETURN, A_ATOM_1, + + // 'False' + OP_TOKEN, 528, + OP_RETURN, A_ATOM_2, + + // 'None' + OP_TOKEN, 529, + OP_RETURN, A_ATOM_3, + + // '__new_parser__' + OP_TOKEN, 530, + OP_RETURN, A_ATOM_4, + + // &STRING strings + OP_SAVE_MARK, + OP_STRING, + OP_POS_LOOKAHEAD, + OP_RULE, R_STRINGS, + OP_RETURN, A_ATOM_5, + + // NUMBER + OP_NUMBER, + OP_RETURN, A_ATOM_6, + + // &'(' (tuple | group | genexp) + OP_SAVE_MARK, + OP_TOKEN, LPAR, + OP_POS_LOOKAHEAD, + OP_RULE, R__TMP_80, + OP_RETURN, A_ATOM_7, + + // &'[' (list | listcomp) + OP_SAVE_MARK, + OP_TOKEN, LSQB, + OP_POS_LOOKAHEAD, + OP_RULE, R__TMP_81, + OP_RETURN, A_ATOM_8, + + // &'{' (dict | set | dictcomp | setcomp) + OP_SAVE_MARK, + OP_TOKEN, LBRACE, + OP_POS_LOOKAHEAD, + OP_RULE, R__TMP_82, + OP_RETURN, A_ATOM_9, + + // '...' + OP_TOKEN, ELLIPSIS, + OP_RETURN, A_ATOM_10, + }, }, {"strings", R_STRINGS, {0, -1}, { - OP_RULE, R__LOOP1_83, OP_RETURN, A_STRINGS_0, + // STRING+ + OP_RULE, R__LOOP1_83, + OP_RETURN, A_STRINGS_0, + }, }, {"list", R_LIST, {0, -1}, { - OP_TOKEN, LSQB, OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_LIST_0, + // '[' star_named_expressions? ']' + OP_TOKEN, LSQB, + OP_RULE, R_STAR_NAMED_EXPRESSIONS, + OP_OPTIONAL, + OP_TOKEN, RSQB, + OP_RETURN, A_LIST_0, + }, }, {"listcomp", R_LISTCOMP, {0, 10, -1}, { - OP_TOKEN, LSQB, OP_RULE, R_NAMED_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RSQB, OP_RETURN, A_LISTCOMP_0, - OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_LISTCOMP_1, + // '[' named_expression for_if_clauses ']' + OP_TOKEN, LSQB, + OP_RULE, R_NAMED_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, RSQB, + OP_RETURN, A_LISTCOMP_0, + + // invalid_comprehension + OP_RULE, R_INVALID_COMPREHENSION, + OP_RETURN, A_LISTCOMP_1, + }, }, {"tuple", R_TUPLE, {0, -1}, { - OP_TOKEN, LPAR, OP_RULE, R__TMP_84, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_TUPLE_0, + // '(' [star_named_expression ',' star_named_expressions?] ')' + OP_TOKEN, LPAR, + OP_RULE, R__TMP_84, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A_TUPLE_0, + }, }, {"group", R_GROUP, {0, -1}, { - OP_TOKEN, LPAR, OP_RULE, R__TMP_85, OP_TOKEN, RPAR, OP_RETURN, A_GROUP_0, + // '(' (yield_expr | named_expression) ')' + OP_TOKEN, LPAR, + OP_RULE, R__TMP_85, + OP_TOKEN, RPAR, + OP_RETURN, A_GROUP_0, + }, }, {"genexp", R_GENEXP, {0, 10, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RPAR, OP_RETURN, A_GENEXP_0, - OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_GENEXP_1, + // '(' expression for_if_clauses ')' + OP_TOKEN, LPAR, + OP_RULE, R_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, RPAR, + OP_RETURN, A_GENEXP_0, + + // invalid_comprehension + OP_RULE, R_INVALID_COMPREHENSION, + OP_RETURN, A_GENEXP_1, + }, }, {"set", R_SET, {0, -1}, { - OP_TOKEN, LBRACE, OP_RULE, R_EXPRESSIONS_LIST, OP_TOKEN, RBRACE, OP_RETURN, A_SET_0, + // '{' expressions_list '}' + OP_TOKEN, LBRACE, + OP_RULE, R_EXPRESSIONS_LIST, + OP_TOKEN, RBRACE, + OP_RETURN, A_SET_0, + }, }, {"setcomp", R_SETCOMP, {0, 10, -1}, { - OP_TOKEN, LBRACE, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_SETCOMP_0, - OP_RULE, R_INVALID_COMPREHENSION, OP_RETURN, A_SETCOMP_1, + // '{' expression for_if_clauses '}' + OP_TOKEN, LBRACE, + OP_RULE, R_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, RBRACE, + OP_RETURN, A_SETCOMP_0, + + // invalid_comprehension + OP_RULE, R_INVALID_COMPREHENSION, + OP_RETURN, A_SETCOMP_1, + }, }, {"dict", R_DICT, {0, -1}, { - OP_TOKEN, LBRACE, OP_RULE, R_DOUBLE_STARRED_KVPAIRS, OP_OPTIONAL, OP_TOKEN, RBRACE, OP_RETURN, A_DICT_0, + // '{' double_starred_kvpairs? '}' + OP_TOKEN, LBRACE, + OP_RULE, R_DOUBLE_STARRED_KVPAIRS, + OP_OPTIONAL, + OP_TOKEN, RBRACE, + OP_RETURN, A_DICT_0, + }, }, {"dictcomp", R_DICTCOMP, {0, 10, -1}, { - OP_TOKEN, LBRACE, OP_RULE, R_KVPAIR, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_DICTCOMP_0, - OP_RULE, R_INVALID_DICT_COMPREHENSION, OP_RETURN, A_DICTCOMP_1, + // '{' kvpair for_if_clauses '}' + OP_TOKEN, LBRACE, + OP_RULE, R_KVPAIR, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, RBRACE, + OP_RETURN, A_DICTCOMP_0, + + // invalid_dict_comprehension + OP_RULE, R_INVALID_DICT_COMPREHENSION, + OP_RETURN, A_DICTCOMP_1, + }, }, {"double_starred_kvpairs", R_DOUBLE_STARRED_KVPAIRS, {0, -1}, { - OP_RULE, R__GATHER_86, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DOUBLE_STARRED_KVPAIRS_0, + // ','.double_starred_kvpair+ ','? + OP_RULE, R__GATHER_86, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_DOUBLE_STARRED_KVPAIRS_0, + }, }, {"double_starred_kvpair", R_DOUBLE_STARRED_KVPAIR, {0, 6, -1}, { - OP_TOKEN, DOUBLESTAR, OP_RULE, R_BITWISE_OR, OP_RETURN, A_DOUBLE_STARRED_KVPAIR_0, - OP_RULE, R_KVPAIR, OP_RETURN, A_DOUBLE_STARRED_KVPAIR_1, + // '**' bitwise_or + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_BITWISE_OR, + OP_RETURN, A_DOUBLE_STARRED_KVPAIR_0, + + // kvpair + OP_RULE, R_KVPAIR, + OP_RETURN, A_DOUBLE_STARRED_KVPAIR_1, + }, }, {"kvpair", R_KVPAIR, {0, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RETURN, A_KVPAIR_0, + // expression ':' expression + OP_RULE, R_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_KVPAIR_0, + }, }, {"for_if_clauses", R_FOR_IF_CLAUSES, {0, -1}, { - OP_RULE, R__LOOP1_87, OP_RETURN, A_FOR_IF_CLAUSES_0, + // for_if_clause+ + OP_RULE, R__LOOP1_87, + OP_RETURN, A_FOR_IF_CLAUSES_0, + }, }, {"for_if_clause", R_FOR_IF_CLAUSE, {0, 14, -1}, { - OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_DISJUNCTION, OP_RULE, R__LOOP0_88, OP_RETURN, A_FOR_IF_CLAUSE_0, - OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, OP_RULE, R_DISJUNCTION, OP_RULE, R__LOOP0_89, OP_RETURN, A_FOR_IF_CLAUSE_1, + // ASYNC 'for' star_targets 'in' disjunction (('if' disjunction))* + OP_TOKEN, ASYNC, + OP_TOKEN, 517, + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, 518, + OP_RULE, R_DISJUNCTION, + OP_RULE, R__LOOP0_88, + OP_RETURN, A_FOR_IF_CLAUSE_0, + + // 'for' star_targets 'in' disjunction (('if' disjunction))* + OP_TOKEN, 517, + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, 518, + OP_RULE, R_DISJUNCTION, + OP_RULE, R__LOOP0_89, + OP_RETURN, A_FOR_IF_CLAUSE_1, + }, }, {"yield_expr", R_YIELD_EXPR, {0, 8, -1}, { - OP_TOKEN, 504, OP_TOKEN, 514, OP_RULE, R_EXPRESSION, OP_RETURN, A_YIELD_EXPR_0, - OP_TOKEN, 504, OP_RULE, R_STAR_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A_YIELD_EXPR_1, + // 'yield' 'from' expression + OP_TOKEN, 504, + OP_TOKEN, 514, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_YIELD_EXPR_0, + + // 'yield' star_expressions? + OP_TOKEN, 504, + OP_RULE, R_STAR_EXPRESSIONS, + OP_OPTIONAL, + OP_RETURN, A_YIELD_EXPR_1, + }, }, {"arguments", R_ARGUMENTS, {0, 11, -1}, { - OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_OPTIONAL, OP_SAVE_MARK, OP_TOKEN, RPAR, OP_POS_LOOKAHEAD, OP_RETURN, A_ARGUMENTS_0, - OP_RULE, R_INCORRECT_ARGUMENTS, OP_RETURN, A_ARGUMENTS_1, + // args ','? &')' + OP_RULE, R_ARGS, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_SAVE_MARK, + OP_TOKEN, RPAR, + OP_POS_LOOKAHEAD, + OP_RETURN, A_ARGUMENTS_0, + + // incorrect_arguments + OP_RULE, R_INCORRECT_ARGUMENTS, + OP_RETURN, A_ARGUMENTS_1, + }, }, {"args", R_ARGS, {0, 7, 11, -1}, { - OP_RULE, R_STARRED_EXPRESSION, OP_RULE, R__TMP_90, OP_OPTIONAL, OP_RETURN, A_ARGS_0, - OP_RULE, R_KWARGS, OP_RETURN, A_ARGS_1, - OP_RULE, R_NAMED_EXPRESSION, OP_RULE, R__TMP_91, OP_OPTIONAL, OP_RETURN, A_ARGS_2, + // starred_expression [',' args] + OP_RULE, R_STARRED_EXPRESSION, + OP_RULE, R__TMP_90, + OP_OPTIONAL, + OP_RETURN, A_ARGS_0, + + // kwargs + OP_RULE, R_KWARGS, + OP_RETURN, A_ARGS_1, + + // named_expression [',' args] + OP_RULE, R_NAMED_EXPRESSION, + OP_RULE, R__TMP_91, + OP_OPTIONAL, + OP_RETURN, A_ARGS_2, + }, }, {"kwargs", R_KWARGS, {0, 8, 12, -1}, { - OP_RULE, R__GATHER_92, OP_TOKEN, COMMA, OP_RULE, R__GATHER_93, OP_RETURN, A_KWARGS_0, - OP_RULE, R__GATHER_94, OP_RETURN, A_KWARGS_1, - OP_RULE, R__GATHER_95, OP_RETURN, A_KWARGS_2, + // ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ + OP_RULE, R__GATHER_92, + OP_TOKEN, COMMA, + OP_RULE, R__GATHER_93, + OP_RETURN, A_KWARGS_0, + + // ','.kwarg_or_starred+ + OP_RULE, R__GATHER_94, + OP_RETURN, A_KWARGS_1, + + // ','.kwarg_or_double_starred+ + OP_RULE, R__GATHER_95, + OP_RETURN, A_KWARGS_2, + }, }, {"starred_expression", R_STARRED_EXPRESSION, {0, -1}, { - OP_TOKEN, STAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_STARRED_EXPRESSION_0, + // '*' expression + OP_TOKEN, STAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_STARRED_EXPRESSION_0, + }, }, {"kwarg_or_starred", R_KWARG_OR_STARRED, {0, 7, 11, -1}, { - OP_NAME, OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_STARRED_0, - OP_RULE, R_STARRED_EXPRESSION, OP_RETURN, A_KWARG_OR_STARRED_1, - OP_RULE, R_INVALID_KWARG, OP_RETURN, A_KWARG_OR_STARRED_2, + // NAME '=' expression + OP_NAME, + OP_TOKEN, EQUAL, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_KWARG_OR_STARRED_0, + + // starred_expression + OP_RULE, R_STARRED_EXPRESSION, + OP_RETURN, A_KWARG_OR_STARRED_1, + + // invalid_kwarg + OP_RULE, R_INVALID_KWARG, + OP_RETURN, A_KWARG_OR_STARRED_2, + }, }, {"kwarg_or_double_starred", R_KWARG_OR_DOUBLE_STARRED, {0, 7, 13, -1}, { - OP_NAME, OP_TOKEN, EQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_0, - OP_TOKEN, DOUBLESTAR, OP_RULE, R_EXPRESSION, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_1, - OP_RULE, R_INVALID_KWARG, OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_2, + // NAME '=' expression + OP_NAME, + OP_TOKEN, EQUAL, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_0, + + // '**' expression + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_1, + + // invalid_kwarg + OP_RULE, R_INVALID_KWARG, + OP_RETURN, A_KWARG_OR_DOUBLE_STARRED_2, + }, }, {"star_targets", R_STAR_TARGETS, {0, 8, -1}, { - OP_RULE, R_STAR_TARGET, OP_SAVE_MARK, OP_TOKEN, COMMA, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGETS_0, - OP_RULE, R_STAR_TARGET, OP_RULE, R__LOOP0_96, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_1, + // star_target !',' + OP_RULE, R_STAR_TARGET, + OP_SAVE_MARK, + OP_TOKEN, COMMA, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_STAR_TARGETS_0, + + // star_target ((',' star_target))* ','? + OP_RULE, R_STAR_TARGET, + OP_RULE, R__LOOP0_96, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_STAR_TARGETS_1, + }, }, {"star_targets_seq", R_STAR_TARGETS_SEQ, {0, -1}, { - OP_RULE, R__GATHER_97, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_SEQ_0, + // ','.star_target+ ','? + OP_RULE, R__GATHER_97, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_STAR_TARGETS_SEQ_0, + }, }, {"star_target", R_STAR_TARGET, {0, 6, 17, 31, -1}, { - OP_TOKEN, STAR, OP_RULE, R__TMP_98, OP_RETURN, A_STAR_TARGET_0, - OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGET_1, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_STAR_TARGET_2, - OP_RULE, R_STAR_ATOM, OP_RETURN, A_STAR_TARGET_3, + // '*' (!'*' star_target) + OP_TOKEN, STAR, + OP_RULE, R__TMP_98, + OP_RETURN, A_STAR_TARGET_0, + + // t_primary '.' NAME !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_STAR_TARGET_1, + + // t_primary '[' slices ']' !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_STAR_TARGET_2, + + // star_atom + OP_RULE, R_STAR_ATOM, + OP_RETURN, A_STAR_TARGET_3, + }, }, {"star_atom", R_STAR_ATOM, {0, 3, 11, 20, -1}, { - OP_NAME, OP_RETURN, A_STAR_ATOM_0, - OP_TOKEN, LPAR, OP_RULE, R_STAR_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_STAR_ATOM_1, - OP_TOKEN, LPAR, OP_RULE, R_STAR_TARGETS_SEQ, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_STAR_ATOM_2, - OP_TOKEN, LSQB, OP_RULE, R_STAR_TARGETS_SEQ, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_STAR_ATOM_3, + // NAME + OP_NAME, + OP_RETURN, A_STAR_ATOM_0, + + // '(' star_target ')' + OP_TOKEN, LPAR, + OP_RULE, R_STAR_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A_STAR_ATOM_1, + + // '(' star_targets_seq? ')' + OP_TOKEN, LPAR, + OP_RULE, R_STAR_TARGETS_SEQ, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A_STAR_ATOM_2, + + // '[' star_targets_seq? ']' + OP_TOKEN, LSQB, + OP_RULE, R_STAR_TARGETS_SEQ, + OP_OPTIONAL, + OP_TOKEN, RSQB, + OP_RETURN, A_STAR_ATOM_3, + }, }, {"single_target", R_SINGLE_TARGET, {0, 4, 7, -1}, { - OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, OP_RETURN, A_SINGLE_TARGET_0, - OP_NAME, OP_RETURN, A_SINGLE_TARGET_1, - OP_TOKEN, LPAR, OP_RULE, R_SINGLE_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_SINGLE_TARGET_2, + // single_subscript_attribute_target + OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, + OP_RETURN, A_SINGLE_TARGET_0, + + // NAME + OP_NAME, + OP_RETURN, A_SINGLE_TARGET_1, + + // '(' single_target ')' + OP_TOKEN, LPAR, + OP_RULE, R_SINGLE_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A_SINGLE_TARGET_2, + }, }, {"single_subscript_attribute_target", R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, {0, 11, -1}, { - OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_0, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_1, + // t_primary '.' NAME !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_0, + + // t_primary '[' slices ']' !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET_1, + }, }, {"del_targets", R_DEL_TARGETS, {0, -1}, { - OP_RULE, R__GATHER_99, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DEL_TARGETS_0, + // ','.del_target+ ','? + OP_RULE, R__GATHER_99, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_DEL_TARGETS_0, + }, }, {"del_target", R_DEL_TARGET, {0, 11, 25, -1}, { - OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_0, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_1, - OP_RULE, R_DEL_T_ATOM, OP_RETURN, A_DEL_TARGET_2, + // t_primary '.' NAME &del_target_end + OP_RULE, R_T_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_DEL_TARGET_END, + OP_POS_LOOKAHEAD, + OP_RETURN, A_DEL_TARGET_0, + + // t_primary '[' slices ']' &del_target_end + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_SAVE_MARK, + OP_RULE, R_DEL_TARGET_END, + OP_POS_LOOKAHEAD, + OP_RETURN, A_DEL_TARGET_1, + + // del_t_atom + OP_RULE, R_DEL_T_ATOM, + OP_RETURN, A_DEL_TARGET_2, + }, }, {"del_t_atom", R_DEL_T_ATOM, {0, 7, 15, 24, 33, -1}, { - OP_NAME, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_T_ATOM_0, - OP_TOKEN, LPAR, OP_RULE, R_DEL_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_DEL_T_ATOM_1, - OP_TOKEN, LPAR, OP_RULE, R_DEL_TARGETS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_DEL_T_ATOM_2, - OP_TOKEN, LSQB, OP_RULE, R_DEL_TARGETS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_DEL_T_ATOM_3, - OP_RULE, R_INVALID_DEL_TARGET, OP_RETURN, A_DEL_T_ATOM_4, + // NAME &del_target_end + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_DEL_TARGET_END, + OP_POS_LOOKAHEAD, + OP_RETURN, A_DEL_T_ATOM_0, + + // '(' del_target ')' + OP_TOKEN, LPAR, + OP_RULE, R_DEL_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A_DEL_T_ATOM_1, + + // '(' del_targets? ')' + OP_TOKEN, LPAR, + OP_RULE, R_DEL_TARGETS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A_DEL_T_ATOM_2, + + // '[' del_targets? ']' + OP_TOKEN, LSQB, + OP_RULE, R_DEL_TARGETS, + OP_OPTIONAL, + OP_TOKEN, RSQB, + OP_RETURN, A_DEL_T_ATOM_3, + + // invalid_del_target + OP_RULE, R_INVALID_DEL_TARGET, + OP_RETURN, A_DEL_T_ATOM_4, + }, }, {"del_target_end", R_DEL_TARGET_END, {0, 4, 8, 12, 16, -1}, { - OP_TOKEN, RPAR, OP_RETURN, A_DEL_TARGET_END_0, - OP_TOKEN, RSQB, OP_RETURN, A_DEL_TARGET_END_1, - OP_TOKEN, COMMA, OP_RETURN, A_DEL_TARGET_END_2, - OP_TOKEN, SEMI, OP_RETURN, A_DEL_TARGET_END_3, - OP_TOKEN, NEWLINE, OP_RETURN, A_DEL_TARGET_END_4, + // ')' + OP_TOKEN, RPAR, + OP_RETURN, A_DEL_TARGET_END_0, + + // ']' + OP_TOKEN, RSQB, + OP_RETURN, A_DEL_TARGET_END_1, + + // ',' + OP_TOKEN, COMMA, + OP_RETURN, A_DEL_TARGET_END_2, + + // ';' + OP_TOKEN, SEMI, + OP_RETURN, A_DEL_TARGET_END_3, + + // NEWLINE + OP_TOKEN, NEWLINE, + OP_RETURN, A_DEL_TARGET_END_4, + }, }, {"targets", R_TARGETS, {0, -1}, { - OP_RULE, R__GATHER_100, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_TARGETS_0, + // ','.target+ ','? + OP_RULE, R__GATHER_100, + OP_TOKEN, COMMA, + OP_OPTIONAL, + OP_RETURN, A_TARGETS_0, + }, }, {"target", R_TARGET, {0, 11, 25, -1}, { - OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_TARGET_0, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_NEG_LOOKAHEAD, OP_RETURN, A_TARGET_1, - OP_RULE, R_T_ATOM, OP_RETURN, A_TARGET_2, + // t_primary '.' NAME !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_TARGET_0, + + // t_primary '[' slices ']' !t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_TARGET_1, + + // t_atom + OP_RULE, R_T_ATOM, + OP_RETURN, A_TARGET_2, + }, }, {"t_primary", R_T_PRIMARY, {0, 12, 26, 36, 51, -1}, { - OP_SETUP_LEFT_REC, OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_0, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_1, - OP_RULE, R_T_PRIMARY, OP_RULE, R_GENEXP, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_2, - OP_RULE, R_T_PRIMARY, OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_3, - OP_RULE, R_ATOM, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, OP_RETURN_LEFT_REC, A_T_PRIMARY_4, + // t_primary '.' NAME &t_lookahead + OP_SETUP_LEFT_REC, + OP_RULE, R_T_PRIMARY, + OP_TOKEN, DOT, + OP_NAME, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_POS_LOOKAHEAD, + OP_RETURN_LEFT_REC, A_T_PRIMARY_0, + + // t_primary '[' slices ']' &t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LSQB, + OP_RULE, R_SLICES, + OP_TOKEN, RSQB, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_POS_LOOKAHEAD, + OP_RETURN_LEFT_REC, A_T_PRIMARY_1, + + // t_primary genexp &t_lookahead + OP_RULE, R_T_PRIMARY, + OP_RULE, R_GENEXP, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_POS_LOOKAHEAD, + OP_RETURN_LEFT_REC, A_T_PRIMARY_2, + + // t_primary '(' arguments? ')' &t_lookahead + OP_RULE, R_T_PRIMARY, + OP_TOKEN, LPAR, + OP_RULE, R_ARGUMENTS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_POS_LOOKAHEAD, + OP_RETURN_LEFT_REC, A_T_PRIMARY_3, + + // atom &t_lookahead + OP_RULE, R_ATOM, + OP_SAVE_MARK, + OP_RULE, R_T_LOOKAHEAD, + OP_POS_LOOKAHEAD, + OP_RETURN_LEFT_REC, A_T_PRIMARY_4, + }, }, {"t_lookahead", R_T_LOOKAHEAD, {0, 4, 8, -1}, { - OP_TOKEN, LPAR, OP_RETURN, A_T_LOOKAHEAD_0, - OP_TOKEN, LSQB, OP_RETURN, A_T_LOOKAHEAD_1, - OP_TOKEN, DOT, OP_RETURN, A_T_LOOKAHEAD_2, + // '(' + OP_TOKEN, LPAR, + OP_RETURN, A_T_LOOKAHEAD_0, + + // '[' + OP_TOKEN, LSQB, + OP_RETURN, A_T_LOOKAHEAD_1, + + // '.' + OP_TOKEN, DOT, + OP_RETURN, A_T_LOOKAHEAD_2, + }, }, {"t_atom", R_T_ATOM, {0, 3, 11, 20, -1}, { - OP_NAME, OP_RETURN, A_T_ATOM_0, - OP_TOKEN, LPAR, OP_RULE, R_TARGET, OP_TOKEN, RPAR, OP_RETURN, A_T_ATOM_1, - OP_TOKEN, LPAR, OP_RULE, R_TARGETS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_T_ATOM_2, - OP_TOKEN, LSQB, OP_RULE, R_TARGETS, OP_OPTIONAL, OP_TOKEN, RSQB, OP_RETURN, A_T_ATOM_3, + // NAME + OP_NAME, + OP_RETURN, A_T_ATOM_0, + + // '(' target ')' + OP_TOKEN, LPAR, + OP_RULE, R_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A_T_ATOM_1, + + // '(' targets? ')' + OP_TOKEN, LPAR, + OP_RULE, R_TARGETS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A_T_ATOM_2, + + // '[' targets? ']' + OP_TOKEN, LSQB, + OP_RULE, R_TARGETS, + OP_OPTIONAL, + OP_TOKEN, RSQB, + OP_RETURN, A_T_ATOM_3, + }, }, {"incorrect_arguments", R_INCORRECT_ARGUMENTS, {0, 8, 19, 25, 35, -1}, { - OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_TOKEN, STAR, OP_RETURN, A_INCORRECT_ARGUMENTS_0, - OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, COMMA, OP_RULE, R__TMP_101, OP_OPTIONAL, OP_RETURN, A_INCORRECT_ARGUMENTS_1, - OP_RULE, R_ARGS, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INCORRECT_ARGUMENTS_2, - OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INCORRECT_ARGUMENTS_3, - OP_RULE, R_ARGS, OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A_INCORRECT_ARGUMENTS_4, + // args ',' '*' + OP_RULE, R_ARGS, + OP_TOKEN, COMMA, + OP_TOKEN, STAR, + OP_RETURN, A_INCORRECT_ARGUMENTS_0, + + // expression for_if_clauses ',' [args | expression for_if_clauses] + OP_RULE, R_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, COMMA, + OP_RULE, R__TMP_101, + OP_OPTIONAL, + OP_RETURN, A_INCORRECT_ARGUMENTS_1, + + // args for_if_clauses + OP_RULE, R_ARGS, + OP_RULE, R_FOR_IF_CLAUSES, + OP_RETURN, A_INCORRECT_ARGUMENTS_2, + + // args ',' expression for_if_clauses + OP_RULE, R_ARGS, + OP_TOKEN, COMMA, + OP_RULE, R_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_RETURN, A_INCORRECT_ARGUMENTS_3, + + // args ',' args + OP_RULE, R_ARGS, + OP_TOKEN, COMMA, + OP_RULE, R_ARGS, + OP_RETURN, A_INCORRECT_ARGUMENTS_4, + }, }, {"invalid_kwarg", R_INVALID_KWARG, {0, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, EQUAL, OP_RETURN, A_INVALID_KWARG_0, + // expression '=' + OP_RULE, R_EXPRESSION, + OP_TOKEN, EQUAL, + OP_RETURN, A_INVALID_KWARG_0, + }, }, {"invalid_named_expression", R_INVALID_NAMED_EXPRESSION, {0, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COLONEQUAL, OP_RULE, R_EXPRESSION, OP_RETURN, A_INVALID_NAMED_EXPRESSION_0, + // expression ':=' expression + OP_RULE, R_EXPRESSION, + OP_TOKEN, COLONEQUAL, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_INVALID_NAMED_EXPRESSION_0, + }, }, {"invalid_assignment", R_INVALID_ASSIGNMENT, {0, 6, 12, 22, 33, 41, -1}, { - OP_RULE, R_LIST, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_0, - OP_RULE, R_TUPLE, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_1, - OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_RULE, R__LOOP0_102, OP_TOKEN, COLON, OP_RETURN, A_INVALID_ASSIGNMENT_2, - OP_RULE, R_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_RULE, R__TMP_103, OP_OPTIONAL, OP_RETURN, A_INVALID_ASSIGNMENT_3, - OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, EQUAL, OP_RULE, R__TMP_104, OP_RETURN, A_INVALID_ASSIGNMENT_4, - OP_RULE, R_STAR_EXPRESSIONS, OP_RULE, R_AUGASSIGN, OP_RULE, R__TMP_105, OP_RETURN, A_INVALID_ASSIGNMENT_5, + // list ':' + OP_RULE, R_LIST, + OP_TOKEN, COLON, + OP_RETURN, A_INVALID_ASSIGNMENT_0, + + // tuple ':' + OP_RULE, R_TUPLE, + OP_TOKEN, COLON, + OP_RETURN, A_INVALID_ASSIGNMENT_1, + + // star_named_expression ',' star_named_expressions* ':' + OP_RULE, R_STAR_NAMED_EXPRESSION, + OP_TOKEN, COMMA, + OP_RULE, R__LOOP0_102, + OP_TOKEN, COLON, + OP_RETURN, A_INVALID_ASSIGNMENT_2, + + // expression ':' expression ['=' annotated_rhs] + OP_RULE, R_EXPRESSION, + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_RULE, R__TMP_103, + OP_OPTIONAL, + OP_RETURN, A_INVALID_ASSIGNMENT_3, + + // star_expressions '=' (yield_expr | star_expressions) + OP_RULE, R_STAR_EXPRESSIONS, + OP_TOKEN, EQUAL, + OP_RULE, R__TMP_104, + OP_RETURN, A_INVALID_ASSIGNMENT_4, + + // star_expressions augassign (yield_expr | star_expressions) + OP_RULE, R_STAR_EXPRESSIONS, + OP_RULE, R_AUGASSIGN, + OP_RULE, R__TMP_105, + OP_RETURN, A_INVALID_ASSIGNMENT_5, + }, }, {"invalid_block", R_INVALID_BLOCK, {0, -1}, { - OP_TOKEN, NEWLINE, OP_SAVE_MARK, OP_TOKEN, INDENT, OP_NEG_LOOKAHEAD, OP_RETURN, A_INVALID_BLOCK_0, + // NEWLINE !INDENT + OP_TOKEN, NEWLINE, + OP_SAVE_MARK, + OP_TOKEN, INDENT, + OP_NEG_LOOKAHEAD, + OP_RETURN, A_INVALID_BLOCK_0, + }, }, {"invalid_comprehension", R_INVALID_COMPREHENSION, {0, -1}, { - OP_RULE, R__TMP_106, OP_RULE, R_STARRED_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INVALID_COMPREHENSION_0, + // ('[' | '(' | '{') starred_expression for_if_clauses + OP_RULE, R__TMP_106, + OP_RULE, R_STARRED_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_RETURN, A_INVALID_COMPREHENSION_0, + }, }, {"invalid_dict_comprehension", R_INVALID_DICT_COMPREHENSION, {0, -1}, { - OP_TOKEN, LBRACE, OP_TOKEN, DOUBLESTAR, OP_RULE, R_BITWISE_OR, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_INVALID_DICT_COMPREHENSION_0, + // '{' '**' bitwise_or for_if_clauses '}' + OP_TOKEN, LBRACE, + OP_TOKEN, DOUBLESTAR, + OP_RULE, R_BITWISE_OR, + OP_RULE, R_FOR_IF_CLAUSES, + OP_TOKEN, RBRACE, + OP_RETURN, A_INVALID_DICT_COMPREHENSION_0, + }, }, {"invalid_parameters", R_INVALID_PARAMETERS, {0, -1}, { - OP_RULE, R__LOOP0_107, OP_RULE, R__TMP_108, OP_RULE, R_PARAM_NO_DEFAULT, OP_RETURN, A_INVALID_PARAMETERS_0, + // param_no_default* (slash_with_default | param_with_default+) param_no_default + OP_RULE, R__LOOP0_107, + OP_RULE, R__TMP_108, + OP_RULE, R_PARAM_NO_DEFAULT, + OP_RETURN, A_INVALID_PARAMETERS_0, + }, }, {"invalid_star_etc", R_INVALID_STAR_ETC, {0, 6, -1}, { - OP_TOKEN, STAR, OP_RULE, R__TMP_109, OP_RETURN, A_INVALID_STAR_ETC_0, - OP_TOKEN, STAR, OP_TOKEN, COMMA, OP_TOKEN, TYPE_COMMENT, OP_RETURN, A_INVALID_STAR_ETC_1, + // '*' (')' | ',' (')' | '**')) + OP_TOKEN, STAR, + OP_RULE, R__TMP_109, + OP_RETURN, A_INVALID_STAR_ETC_0, + + // '*' ',' TYPE_COMMENT + OP_TOKEN, STAR, + OP_TOKEN, COMMA, + OP_TOKEN, TYPE_COMMENT, + OP_RETURN, A_INVALID_STAR_ETC_1, + }, }, {"invalid_lambda_star_etc", R_INVALID_LAMBDA_STAR_ETC, {0, -1}, { - OP_TOKEN, STAR, OP_RULE, R__TMP_110, OP_RETURN, A_INVALID_LAMBDA_STAR_ETC_0, + // '*' (':' | ',' (':' | '**')) + OP_TOKEN, STAR, + OP_RULE, R__TMP_110, + OP_RETURN, A_INVALID_LAMBDA_STAR_ETC_0, + }, }, {"invalid_double_type_comments", R_INVALID_DOUBLE_TYPE_COMMENTS, {0, -1}, { - OP_TOKEN, TYPE_COMMENT, OP_TOKEN, NEWLINE, OP_TOKEN, TYPE_COMMENT, OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RETURN, A_INVALID_DOUBLE_TYPE_COMMENTS_0, + // TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT + OP_TOKEN, TYPE_COMMENT, + OP_TOKEN, NEWLINE, + OP_TOKEN, TYPE_COMMENT, + OP_TOKEN, NEWLINE, + OP_TOKEN, INDENT, + OP_RETURN, A_INVALID_DOUBLE_TYPE_COMMENTS_0, + }, }, {"invalid_del_target", R_INVALID_DEL_TARGET, {0, -1}, { - OP_RULE, R_STAR_EXPRESSION, OP_SAVE_MARK, OP_RULE, R_DEL_TARGET_END, OP_POS_LOOKAHEAD, OP_RETURN, A_INVALID_DEL_TARGET_0, + // star_expression &del_target_end + OP_RULE, R_STAR_EXPRESSION, + OP_SAVE_MARK, + OP_RULE, R_DEL_TARGET_END, + OP_POS_LOOKAHEAD, + OP_RETURN, A_INVALID_DEL_TARGET_0, + }, }, {"invalid_import_from_targets", R_INVALID_IMPORT_FROM_TARGETS, {0, -1}, { - OP_RULE, R_IMPORT_FROM_AS_NAMES, OP_TOKEN, COMMA, OP_RETURN, A_INVALID_IMPORT_FROM_TARGETS_0, + // import_from_as_names ',' + OP_RULE, R_IMPORT_FROM_AS_NAMES, + OP_TOKEN, COMMA, + OP_RETURN, A_INVALID_IMPORT_FROM_TARGETS_0, + }, }, {"root", R_ROOT, {0, 3, -1}, { - OP_RULE, R_FILE, OP_SUCCESS, - OP_FAILURE, + // + OP_RULE, R_FILE, + OP_SUCCESS, + + // + OP_FAILURE, + }, }, {"_loop0_1", R__LOOP0_1, {0, 3, -1}, { - OP_TOKEN, NEWLINE, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // NEWLINE + OP_TOKEN, NEWLINE, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_2", R__LOOP0_2, {0, 3, -1}, { - OP_TOKEN, NEWLINE, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // NEWLINE + OP_TOKEN, NEWLINE, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_gather_3", R__GATHER_3, {0, 5, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // expression ',' + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // expression + OP_RULE, R_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_4", R__GATHER_4, {0, 5, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // expression ',' + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // expression + OP_RULE, R_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_5", R__GATHER_5, {0, 5, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // expression ',' + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // expression + OP_RULE, R_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_6", R__GATHER_6, {0, 5, -1}, { - OP_RULE, R_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // expression ',' + OP_RULE, R_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // expression + OP_RULE, R_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_loop1_7", R__LOOP1_7, {0, 3, -1}, { - OP_RULE, R_STATEMENT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // statement + OP_RULE, R_STATEMENT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_gather_8", R__GATHER_8, {0, 5, -1}, { - OP_RULE, R_SMALL_STMT, OP_TOKEN, SEMI, OP_LOOP_ITERATE, - OP_RULE, R_SMALL_STMT, OP_LOOP_COLLECT_DELIMITED, + // small_stmt ';' + OP_RULE, R_SMALL_STMT, + OP_TOKEN, SEMI, + OP_LOOP_ITERATE, + + // small_stmt + OP_RULE, R_SMALL_STMT, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_9", R__TMP_9, {0, 4, -1}, { - OP_TOKEN, 513, OP_RETURN, A__TMP_9_0, - OP_TOKEN, 514, OP_RETURN, A__TMP_9_1, + // 'import' + OP_TOKEN, 513, + OP_RETURN, A__TMP_9_0, + + // 'from' + OP_TOKEN, 514, + OP_RETURN, A__TMP_9_1, + }, }, {"_tmp_10", R__TMP_10, {0, 4, 8, -1}, { - OP_TOKEN, 522, OP_RETURN, A__TMP_10_0, - OP_TOKEN, AT, OP_RETURN, A__TMP_10_1, - OP_TOKEN, ASYNC, OP_RETURN, A__TMP_10_2, + // 'def' + OP_TOKEN, 522, + OP_RETURN, A__TMP_10_0, + + // '@' + OP_TOKEN, AT, + OP_RETURN, A__TMP_10_1, + + // ASYNC + OP_TOKEN, ASYNC, + OP_RETURN, A__TMP_10_2, + }, }, {"_tmp_11", R__TMP_11, {0, 4, -1}, { - OP_TOKEN, 523, OP_RETURN, A__TMP_11_0, - OP_TOKEN, AT, OP_RETURN, A__TMP_11_1, + // 'class' + OP_TOKEN, 523, + OP_RETURN, A__TMP_11_0, + + // '@' + OP_TOKEN, AT, + OP_RETURN, A__TMP_11_1, + }, }, {"_tmp_12", R__TMP_12, {0, 4, -1}, { - OP_TOKEN, 519, OP_RETURN, A__TMP_12_0, - OP_TOKEN, ASYNC, OP_RETURN, A__TMP_12_1, + // 'with' + OP_TOKEN, 519, + OP_RETURN, A__TMP_12_0, + + // ASYNC + OP_TOKEN, ASYNC, + OP_RETURN, A__TMP_12_1, + }, }, {"_tmp_13", R__TMP_13, {0, 4, -1}, { - OP_TOKEN, 517, OP_RETURN, A__TMP_13_0, - OP_TOKEN, ASYNC, OP_RETURN, A__TMP_13_1, + // 'for' + OP_TOKEN, 517, + OP_RETURN, A__TMP_13_0, + + // ASYNC + OP_TOKEN, ASYNC, + OP_RETURN, A__TMP_13_1, + }, }, {"_tmp_14", R__TMP_14, {0, -1}, { - OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_14_0, + // '=' annotated_rhs + OP_TOKEN, EQUAL, + OP_RULE, R_ANNOTATED_RHS, + OP_RETURN, A__TMP_14_0, + }, }, {"_tmp_15", R__TMP_15, {0, 8, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_SINGLE_TARGET, OP_TOKEN, RPAR, OP_RETURN, A__TMP_15_0, - OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, OP_RETURN, A__TMP_15_1, + // '(' single_target ')' + OP_TOKEN, LPAR, + OP_RULE, R_SINGLE_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A__TMP_15_0, + + // single_subscript_attribute_target + OP_RULE, R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, + OP_RETURN, A__TMP_15_1, + }, }, {"_tmp_16", R__TMP_16, {0, -1}, { - OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_16_0, + // '=' annotated_rhs + OP_TOKEN, EQUAL, + OP_RULE, R_ANNOTATED_RHS, + OP_RETURN, A__TMP_16_0, + }, }, {"_loop1_17", R__LOOP1_17, {0, 3, -1}, { - OP_RULE, R__TMP_111, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // (star_targets '=') + OP_RULE, R__TMP_111, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_18", R__TMP_18, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_18_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_18_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A__TMP_18_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A__TMP_18_1, + }, }, {"_tmp_19", R__TMP_19, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_19_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_19_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A__TMP_19_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A__TMP_19_1, + }, }, {"_gather_20", R__GATHER_20, {0, 4, -1}, { - OP_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_NAME, OP_LOOP_COLLECT_DELIMITED, + // NAME ',' + OP_NAME, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // NAME + OP_NAME, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_21", R__GATHER_21, {0, 4, -1}, { - OP_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_NAME, OP_LOOP_COLLECT_DELIMITED, + // NAME ',' + OP_NAME, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // NAME + OP_NAME, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_22", R__TMP_22, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_22_0, + // ',' expression + OP_TOKEN, COMMA, + OP_RULE, R_EXPRESSION, + OP_RETURN, A__TMP_22_0, + }, }, {"_loop0_23", R__LOOP0_23, {0, 3, -1}, { - OP_RULE, R__TMP_112, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // ('.' | '...') + OP_RULE, R__TMP_112, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_24", R__LOOP1_24, {0, 3, -1}, { - OP_RULE, R__TMP_113, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // ('.' | '...') + OP_RULE, R__TMP_113, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_gather_25", R__GATHER_25, {0, 5, -1}, { - OP_RULE, R_IMPORT_FROM_AS_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_IMPORT_FROM_AS_NAME, OP_LOOP_COLLECT_DELIMITED, + // import_from_as_name ',' + OP_RULE, R_IMPORT_FROM_AS_NAME, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // import_from_as_name + OP_RULE, R_IMPORT_FROM_AS_NAME, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_26", R__TMP_26, {0, -1}, { - OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_26_0, + // 'as' NAME + OP_TOKEN, 531, + OP_NAME, + OP_RETURN, A__TMP_26_0, + }, }, {"_gather_27", R__GATHER_27, {0, 5, -1}, { - OP_RULE, R_DOTTED_AS_NAME, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_DOTTED_AS_NAME, OP_LOOP_COLLECT_DELIMITED, + // dotted_as_name ',' + OP_RULE, R_DOTTED_AS_NAME, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // dotted_as_name + OP_RULE, R_DOTTED_AS_NAME, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_28", R__TMP_28, {0, -1}, { - OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_28_0, + // 'as' NAME + OP_TOKEN, 531, + OP_NAME, + OP_RETURN, A__TMP_28_0, + }, }, {"_gather_29", R__GATHER_29, {0, 5, -1}, { - OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + // with_item ',' + OP_RULE, R_WITH_ITEM, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // with_item + OP_RULE, R_WITH_ITEM, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_30", R__GATHER_30, {0, 5, -1}, { - OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + // with_item ',' + OP_RULE, R_WITH_ITEM, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // with_item + OP_RULE, R_WITH_ITEM, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_31", R__GATHER_31, {0, 5, -1}, { - OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + // with_item ',' + OP_RULE, R_WITH_ITEM, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // with_item + OP_RULE, R_WITH_ITEM, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_32", R__GATHER_32, {0, 5, -1}, { - OP_RULE, R_WITH_ITEM, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_WITH_ITEM, OP_LOOP_COLLECT_DELIMITED, + // with_item ',' + OP_RULE, R_WITH_ITEM, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // with_item + OP_RULE, R_WITH_ITEM, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_33", R__TMP_33, {0, -1}, { - OP_TOKEN, 531, OP_RULE, R_TARGET, OP_RETURN, A__TMP_33_0, + // 'as' target + OP_TOKEN, 531, + OP_RULE, R_TARGET, + OP_RETURN, A__TMP_33_0, + }, }, {"_loop1_34", R__LOOP1_34, {0, 3, -1}, { - OP_RULE, R_EXCEPT_BLOCK, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // except_block + OP_RULE, R_EXCEPT_BLOCK, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_35", R__TMP_35, {0, -1}, { - OP_TOKEN, 531, OP_NAME, OP_RETURN, A__TMP_35_0, + // 'as' NAME + OP_TOKEN, 531, + OP_NAME, + OP_RETURN, A__TMP_35_0, + }, }, {"_tmp_36", R__TMP_36, {0, -1}, { - OP_TOKEN, 514, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_36_0, + // 'from' expression + OP_TOKEN, 514, + OP_RULE, R_EXPRESSION, + OP_RETURN, A__TMP_36_0, + }, }, {"_tmp_37", R__TMP_37, {0, -1}, { - OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_37_0, + // '->' expression + OP_TOKEN, RARROW, + OP_RULE, R_EXPRESSION, + OP_RETURN, A__TMP_37_0, + }, }, {"_tmp_38", R__TMP_38, {0, -1}, { - OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_38_0, + // '->' expression + OP_TOKEN, RARROW, + OP_RULE, R_EXPRESSION, + OP_RETURN, A__TMP_38_0, + }, }, {"_tmp_39", R__TMP_39, {0, -1}, { - OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, OP_RETURN, A__TMP_39_0, + // NEWLINE INDENT + OP_TOKEN, NEWLINE, + OP_TOKEN, INDENT, + OP_RETURN, A__TMP_39_0, + }, }, {"_loop0_40", R__LOOP0_40, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_41", R__LOOP0_41, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_42", R__LOOP0_42, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_43", R__LOOP1_43, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_44", R__LOOP0_44, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_45", R__LOOP1_45, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_46", R__LOOP1_46, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_47", R__LOOP1_47, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_48", R__LOOP0_48, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_49", R__LOOP1_49, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_50", R__LOOP0_50, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_51", R__LOOP1_51, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_52", R__LOOP0_52, {0, 3, -1}, { - OP_RULE, R_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_maybe_default + OP_RULE, R_PARAM_MAYBE_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_53", R__LOOP1_53, {0, 3, -1}, { - OP_RULE, R_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_maybe_default + OP_RULE, R_PARAM_MAYBE_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_54", R__LOOP1_54, {0, 3, -1}, { - OP_RULE, R__TMP_114, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // ('@' named_expression NEWLINE) + OP_RULE, R__TMP_114, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_55", R__TMP_55, {0, -1}, { - OP_TOKEN, LPAR, OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A__TMP_55_0, + // '(' arguments? ')' + OP_TOKEN, LPAR, + OP_RULE, R_ARGUMENTS, + OP_OPTIONAL, + OP_TOKEN, RPAR, + OP_RETURN, A__TMP_55_0, + }, }, {"_gather_56", R__GATHER_56, {0, 5, -1}, { - OP_RULE, R_STAR_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_STAR_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // star_expression ',' + OP_RULE, R_STAR_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // star_expression + OP_RULE, R_STAR_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_loop1_57", R__LOOP1_57, {0, 3, -1}, { - OP_RULE, R__TMP_115, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // (',' star_expression) + OP_RULE, R__TMP_115, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_gather_58", R__GATHER_58, {0, 5, -1}, { - OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_STAR_NAMED_EXPRESSION, OP_LOOP_COLLECT_DELIMITED, + // star_named_expression ',' + OP_RULE, R_STAR_NAMED_EXPRESSION, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // star_named_expression + OP_RULE, R_STAR_NAMED_EXPRESSION, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_loop1_59", R__LOOP1_59, {0, 3, -1}, { - OP_RULE, R__TMP_116, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // (',' expression) + OP_RULE, R__TMP_116, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_60", R__LOOP0_60, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_61", R__LOOP0_61, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_62", R__LOOP0_62, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_63", R__LOOP1_63, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_64", R__LOOP0_64, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_65", R__LOOP1_65, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_66", R__LOOP1_66, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_67", R__LOOP1_67, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_68", R__LOOP0_68, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_69", R__LOOP1_69, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_70", R__LOOP0_70, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_71", R__LOOP1_71, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_72", R__LOOP0_72, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // lambda_param_maybe_default + OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop1_73", R__LOOP1_73, {0, 3, -1}, { - OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // lambda_param_maybe_default + OP_RULE, R_LAMBDA_PARAM_MAYBE_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_74", R__LOOP1_74, {0, 3, -1}, { - OP_RULE, R__TMP_117, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // ('or' conjunction) + OP_RULE, R__TMP_117, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_75", R__LOOP1_75, {0, 3, -1}, { - OP_RULE, R__TMP_118, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // ('and' inversion) + OP_RULE, R__TMP_118, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop1_76", R__LOOP1_76, {0, 3, -1}, { - OP_RULE, R_COMPARE_OP_BITWISE_OR_PAIR, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // compare_op_bitwise_or_pair + OP_RULE, R_COMPARE_OP_BITWISE_OR_PAIR, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_77", R__TMP_77, {0, -1}, { - OP_TOKEN, NOTEQUAL, OP_RETURN, A__TMP_77_0, + // '!=' + OP_TOKEN, NOTEQUAL, + OP_RETURN, A__TMP_77_0, + }, }, {"_gather_78", R__GATHER_78, {0, 5, -1}, { - OP_RULE, R_SLICE, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_SLICE, OP_LOOP_COLLECT_DELIMITED, + // slice ',' + OP_RULE, R_SLICE, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // slice + OP_RULE, R_SLICE, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_79", R__TMP_79, {0, -1}, { - OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, OP_RETURN, A__TMP_79_0, + // ':' expression? + OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, + OP_OPTIONAL, + OP_RETURN, A__TMP_79_0, + }, }, {"_tmp_80", R__TMP_80, {0, 4, 8, -1}, { - OP_RULE, R_TUPLE, OP_RETURN, A__TMP_80_0, - OP_RULE, R_GROUP, OP_RETURN, A__TMP_80_1, - OP_RULE, R_GENEXP, OP_RETURN, A__TMP_80_2, + // tuple + OP_RULE, R_TUPLE, + OP_RETURN, A__TMP_80_0, + + // group + OP_RULE, R_GROUP, + OP_RETURN, A__TMP_80_1, + + // genexp + OP_RULE, R_GENEXP, + OP_RETURN, A__TMP_80_2, + }, }, {"_tmp_81", R__TMP_81, {0, 4, -1}, { - OP_RULE, R_LIST, OP_RETURN, A__TMP_81_0, - OP_RULE, R_LISTCOMP, OP_RETURN, A__TMP_81_1, + // list + OP_RULE, R_LIST, + OP_RETURN, A__TMP_81_0, + + // listcomp + OP_RULE, R_LISTCOMP, + OP_RETURN, A__TMP_81_1, + }, }, {"_tmp_82", R__TMP_82, {0, 4, 8, 12, -1}, { - OP_RULE, R_DICT, OP_RETURN, A__TMP_82_0, - OP_RULE, R_SET, OP_RETURN, A__TMP_82_1, - OP_RULE, R_DICTCOMP, OP_RETURN, A__TMP_82_2, - OP_RULE, R_SETCOMP, OP_RETURN, A__TMP_82_3, + // dict + OP_RULE, R_DICT, + OP_RETURN, A__TMP_82_0, + + // set + OP_RULE, R_SET, + OP_RETURN, A__TMP_82_1, + + // dictcomp + OP_RULE, R_DICTCOMP, + OP_RETURN, A__TMP_82_2, + + // setcomp + OP_RULE, R_SETCOMP, + OP_RETURN, A__TMP_82_3, + }, }, {"_loop1_83", R__LOOP1_83, {0, 2, -1}, { - OP_STRING, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // STRING + OP_STRING, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_84", R__TMP_84, {0, -1}, { - OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_OPTIONAL, OP_RETURN, A__TMP_84_0, + // star_named_expression ',' star_named_expressions? + OP_RULE, R_STAR_NAMED_EXPRESSION, + OP_TOKEN, COMMA, + OP_RULE, R_STAR_NAMED_EXPRESSIONS, + OP_OPTIONAL, + OP_RETURN, A__TMP_84_0, + }, }, {"_tmp_85", R__TMP_85, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_85_0, - OP_RULE, R_NAMED_EXPRESSION, OP_RETURN, A__TMP_85_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A__TMP_85_0, + + // named_expression + OP_RULE, R_NAMED_EXPRESSION, + OP_RETURN, A__TMP_85_1, + }, }, {"_gather_86", R__GATHER_86, {0, 5, -1}, { - OP_RULE, R_DOUBLE_STARRED_KVPAIR, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_DOUBLE_STARRED_KVPAIR, OP_LOOP_COLLECT_DELIMITED, + // double_starred_kvpair ',' + OP_RULE, R_DOUBLE_STARRED_KVPAIR, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // double_starred_kvpair + OP_RULE, R_DOUBLE_STARRED_KVPAIR, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_loop1_87", R__LOOP1_87, {0, 3, -1}, { - OP_RULE, R_FOR_IF_CLAUSE, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // for_if_clause + OP_RULE, R_FOR_IF_CLAUSE, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_loop0_88", R__LOOP0_88, {0, 3, -1}, { - OP_RULE, R__TMP_119, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // ('if' disjunction) + OP_RULE, R__TMP_119, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_loop0_89", R__LOOP0_89, {0, 3, -1}, { - OP_RULE, R__TMP_120, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // ('if' disjunction) + OP_RULE, R__TMP_120, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_tmp_90", R__TMP_90, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A__TMP_90_0, + // ',' args + OP_TOKEN, COMMA, + OP_RULE, R_ARGS, + OP_RETURN, A__TMP_90_0, + }, }, {"_tmp_91", R__TMP_91, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_ARGS, OP_RETURN, A__TMP_91_0, + // ',' args + OP_TOKEN, COMMA, + OP_RULE, R_ARGS, + OP_RETURN, A__TMP_91_0, + }, }, {"_gather_92", R__GATHER_92, {0, 5, -1}, { - OP_RULE, R_KWARG_OR_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_KWARG_OR_STARRED, OP_LOOP_COLLECT_DELIMITED, + // kwarg_or_starred ',' + OP_RULE, R_KWARG_OR_STARRED, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // kwarg_or_starred + OP_RULE, R_KWARG_OR_STARRED, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_93", R__GATHER_93, {0, 5, -1}, { - OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_LOOP_COLLECT_DELIMITED, + // kwarg_or_double_starred ',' + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // kwarg_or_double_starred + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_94", R__GATHER_94, {0, 5, -1}, { - OP_RULE, R_KWARG_OR_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_KWARG_OR_STARRED, OP_LOOP_COLLECT_DELIMITED, + // kwarg_or_starred ',' + OP_RULE, R_KWARG_OR_STARRED, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // kwarg_or_starred + OP_RULE, R_KWARG_OR_STARRED, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_95", R__GATHER_95, {0, 5, -1}, { - OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_KWARG_OR_DOUBLE_STARRED, OP_LOOP_COLLECT_DELIMITED, + // kwarg_or_double_starred ',' + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // kwarg_or_double_starred + OP_RULE, R_KWARG_OR_DOUBLE_STARRED, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_loop0_96", R__LOOP0_96, {0, 3, -1}, { - OP_RULE, R__TMP_121, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // (',' star_target) + OP_RULE, R__TMP_121, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_gather_97", R__GATHER_97, {0, 5, -1}, { - OP_RULE, R_STAR_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_STAR_TARGET, OP_LOOP_COLLECT_DELIMITED, + // star_target ',' + OP_RULE, R_STAR_TARGET, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // star_target + OP_RULE, R_STAR_TARGET, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_98", R__TMP_98, {0, -1}, { - OP_SAVE_MARK, OP_TOKEN, STAR, OP_NEG_LOOKAHEAD, OP_RULE, R_STAR_TARGET, OP_RETURN, A__TMP_98_0, + // !'*' star_target + OP_SAVE_MARK, + OP_TOKEN, STAR, + OP_NEG_LOOKAHEAD, + OP_RULE, R_STAR_TARGET, + OP_RETURN, A__TMP_98_0, + }, }, {"_gather_99", R__GATHER_99, {0, 5, -1}, { - OP_RULE, R_DEL_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_DEL_TARGET, OP_LOOP_COLLECT_DELIMITED, + // del_target ',' + OP_RULE, R_DEL_TARGET, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // del_target + OP_RULE, R_DEL_TARGET, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_gather_100", R__GATHER_100, {0, 5, -1}, { - OP_RULE, R_TARGET, OP_TOKEN, COMMA, OP_LOOP_ITERATE, - OP_RULE, R_TARGET, OP_LOOP_COLLECT_DELIMITED, + // target ',' + OP_RULE, R_TARGET, + OP_TOKEN, COMMA, + OP_LOOP_ITERATE, + + // target + OP_RULE, R_TARGET, + OP_LOOP_COLLECT_DELIMITED, + }, }, {"_tmp_101", R__TMP_101, {0, 4, -1}, { - OP_RULE, R_ARGS, OP_RETURN, A__TMP_101_0, - OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A__TMP_101_1, + // args + OP_RULE, R_ARGS, + OP_RETURN, A__TMP_101_0, + + // expression for_if_clauses + OP_RULE, R_EXPRESSION, + OP_RULE, R_FOR_IF_CLAUSES, + OP_RETURN, A__TMP_101_1, + }, }, {"_loop0_102", R__LOOP0_102, {0, 3, -1}, { - OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // star_named_expressions + OP_RULE, R_STAR_NAMED_EXPRESSIONS, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_tmp_103", R__TMP_103, {0, -1}, { - OP_TOKEN, EQUAL, OP_RULE, R_ANNOTATED_RHS, OP_RETURN, A__TMP_103_0, + // '=' annotated_rhs + OP_TOKEN, EQUAL, + OP_RULE, R_ANNOTATED_RHS, + OP_RETURN, A__TMP_103_0, + }, }, {"_tmp_104", R__TMP_104, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_104_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_104_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A__TMP_104_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A__TMP_104_1, + }, }, {"_tmp_105", R__TMP_105, {0, 4, -1}, { - OP_RULE, R_YIELD_EXPR, OP_RETURN, A__TMP_105_0, - OP_RULE, R_STAR_EXPRESSIONS, OP_RETURN, A__TMP_105_1, + // yield_expr + OP_RULE, R_YIELD_EXPR, + OP_RETURN, A__TMP_105_0, + + // star_expressions + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A__TMP_105_1, + }, }, {"_tmp_106", R__TMP_106, {0, 4, 8, -1}, { - OP_TOKEN, LSQB, OP_RETURN, A__TMP_106_0, - OP_TOKEN, LPAR, OP_RETURN, A__TMP_106_1, - OP_TOKEN, LBRACE, OP_RETURN, A__TMP_106_2, + // '[' + OP_TOKEN, LSQB, + OP_RETURN, A__TMP_106_0, + + // '(' + OP_TOKEN, LPAR, + OP_RETURN, A__TMP_106_1, + + // '{' + OP_TOKEN, LBRACE, + OP_RETURN, A__TMP_106_2, + }, }, {"_loop0_107", R__LOOP0_107, {0, 3, -1}, { - OP_RULE, R_PARAM_NO_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT, + // param_no_default + OP_RULE, R_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + }, }, {"_tmp_108", R__TMP_108, {0, 4, -1}, { - OP_RULE, R_SLASH_WITH_DEFAULT, OP_RETURN, A__TMP_108_0, - OP_RULE, R__LOOP1_122, OP_RETURN, A__TMP_108_1, + // slash_with_default + OP_RULE, R_SLASH_WITH_DEFAULT, + OP_RETURN, A__TMP_108_0, + + // param_with_default+ + OP_RULE, R__LOOP1_122, + OP_RETURN, A__TMP_108_1, + }, }, {"_tmp_109", R__TMP_109, {0, 4, -1}, { - OP_TOKEN, RPAR, OP_RETURN, A__TMP_109_0, - OP_TOKEN, COMMA, OP_RULE, R__TMP_123, OP_RETURN, A__TMP_109_1, + // ')' + OP_TOKEN, RPAR, + OP_RETURN, A__TMP_109_0, + + // ',' (')' | '**') + OP_TOKEN, COMMA, + OP_RULE, R__TMP_123, + OP_RETURN, A__TMP_109_1, + }, }, {"_tmp_110", R__TMP_110, {0, 4, -1}, { - OP_TOKEN, COLON, OP_RETURN, A__TMP_110_0, - OP_TOKEN, COMMA, OP_RULE, R__TMP_124, OP_RETURN, A__TMP_110_1, + // ':' + OP_TOKEN, COLON, + OP_RETURN, A__TMP_110_0, + + // ',' (':' | '**') + OP_TOKEN, COMMA, + OP_RULE, R__TMP_124, + OP_RETURN, A__TMP_110_1, + }, }, {"_tmp_111", R__TMP_111, {0, -1}, { - OP_RULE, R_STAR_TARGETS, OP_TOKEN, EQUAL, OP_RETURN, A__TMP_111_0, + // star_targets '=' + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, EQUAL, + OP_RETURN, A__TMP_111_0, + }, }, {"_tmp_112", R__TMP_112, {0, 4, -1}, { - OP_TOKEN, DOT, OP_RETURN, A__TMP_112_0, - OP_TOKEN, ELLIPSIS, OP_RETURN, A__TMP_112_1, + // '.' + OP_TOKEN, DOT, + OP_RETURN, A__TMP_112_0, + + // '...' + OP_TOKEN, ELLIPSIS, + OP_RETURN, A__TMP_112_1, + }, }, {"_tmp_113", R__TMP_113, {0, 4, -1}, { - OP_TOKEN, DOT, OP_RETURN, A__TMP_113_0, - OP_TOKEN, ELLIPSIS, OP_RETURN, A__TMP_113_1, + // '.' + OP_TOKEN, DOT, + OP_RETURN, A__TMP_113_0, + + // '...' + OP_TOKEN, ELLIPSIS, + OP_RETURN, A__TMP_113_1, + }, }, {"_tmp_114", R__TMP_114, {0, -1}, { - OP_TOKEN, AT, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, NEWLINE, OP_RETURN, A__TMP_114_0, + // '@' named_expression NEWLINE + OP_TOKEN, AT, + OP_RULE, R_NAMED_EXPRESSION, + OP_TOKEN, NEWLINE, + OP_RETURN, A__TMP_114_0, + }, }, {"_tmp_115", R__TMP_115, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_STAR_EXPRESSION, OP_RETURN, A__TMP_115_0, + // ',' star_expression + OP_TOKEN, COMMA, + OP_RULE, R_STAR_EXPRESSION, + OP_RETURN, A__TMP_115_0, + }, }, {"_tmp_116", R__TMP_116, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, OP_RETURN, A__TMP_116_0, + // ',' expression + OP_TOKEN, COMMA, + OP_RULE, R_EXPRESSION, + OP_RETURN, A__TMP_116_0, + }, }, {"_tmp_117", R__TMP_117, {0, -1}, { - OP_TOKEN, 532, OP_RULE, R_CONJUNCTION, OP_RETURN, A__TMP_117_0, + // 'or' conjunction + OP_TOKEN, 532, + OP_RULE, R_CONJUNCTION, + OP_RETURN, A__TMP_117_0, + }, }, {"_tmp_118", R__TMP_118, {0, -1}, { - OP_TOKEN, 533, OP_RULE, R_INVERSION, OP_RETURN, A__TMP_118_0, + // 'and' inversion + OP_TOKEN, 533, + OP_RULE, R_INVERSION, + OP_RETURN, A__TMP_118_0, + }, }, {"_tmp_119", R__TMP_119, {0, -1}, { - OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_RETURN, A__TMP_119_0, + // 'if' disjunction + OP_TOKEN, 510, + OP_RULE, R_DISJUNCTION, + OP_RETURN, A__TMP_119_0, + }, }, {"_tmp_120", R__TMP_120, {0, -1}, { - OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, OP_RETURN, A__TMP_120_0, + // 'if' disjunction + OP_TOKEN, 510, + OP_RULE, R_DISJUNCTION, + OP_RETURN, A__TMP_120_0, + }, }, {"_tmp_121", R__TMP_121, {0, -1}, { - OP_TOKEN, COMMA, OP_RULE, R_STAR_TARGET, OP_RETURN, A__TMP_121_0, + // ',' star_target + OP_TOKEN, COMMA, + OP_RULE, R_STAR_TARGET, + OP_RETURN, A__TMP_121_0, + }, }, {"_loop1_122", R__LOOP1_122, {0, 3, -1}, { - OP_RULE, R_PARAM_WITH_DEFAULT, OP_LOOP_ITERATE, - OP_LOOP_COLLECT_NONEMPTY, + // param_with_default + OP_RULE, R_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + }, }, {"_tmp_123", R__TMP_123, {0, 4, -1}, { - OP_TOKEN, RPAR, OP_RETURN, A__TMP_123_0, - OP_TOKEN, DOUBLESTAR, OP_RETURN, A__TMP_123_1, + // ')' + OP_TOKEN, RPAR, + OP_RETURN, A__TMP_123_0, + + // '**' + OP_TOKEN, DOUBLESTAR, + OP_RETURN, A__TMP_123_1, + }, }, {"_tmp_124", R__TMP_124, {0, 4, -1}, { - OP_TOKEN, COLON, OP_RETURN, A__TMP_124_0, - OP_TOKEN, DOUBLESTAR, OP_RETURN, A__TMP_124_1, + // ':' + OP_TOKEN, COLON, + OP_RETURN, A__TMP_124_0, + + // '**' + OP_TOKEN, DOUBLESTAR, + OP_RETURN, A__TMP_124_1, + }, }, }; diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 53b16172ed06b6..cf71abf2926888 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -1,13 +1,13 @@ import ast import contextlib -import sys +import dataclasses import re +import sys import token import tokenize from collections import defaultdict from itertools import accumulate -import sys -from typing import Any, Dict, Iterator, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from pegen import grammar from pegen.build import build_parser @@ -34,6 +34,16 @@ ) from pegen.parser_generator import ParserGenerator + +@dataclasses.dataclass +class Opcode: + opcode: str + oparg: Optional[Union[int, str]] = None + + def __len__(self) -> int: + return 1 if self.oparg is None else 2 + + CALL_ACTION_HEAD = """ static void * call_action(Parser *p, Frame *_f, int _iaction) @@ -58,6 +68,7 @@ } """ + class RootRule(Rule): def __init__(self, name: str, startrulename: str): super().__init__(name, "void *", Rhs([]), None) @@ -142,11 +153,11 @@ def __init__( ): super().__init__(grammar, token.tok_name, sys.stdout) - self.opcode_buffer: Optional[List[str]] = None + self.opcode_buffer: Optional[List[Opcode]] = None self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor(self) @contextlib.contextmanager - def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: + def set_opcode_buffer(self, buffer: List[Opcode]) -> Iterator[None]: self.opcode_buffer = buffer yield self.opcode_buffer = None @@ -154,9 +165,8 @@ def set_opcode_buffer(self, buffer: List[str]) -> Iterator[None]: def add_opcode(self, opcode: str, oparg: Optional[str] = None) -> None: assert not isinstance(oparg, int) assert self.opcode_buffer is not None - self.opcode_buffer.append(opcode) - if oparg is not None: - self.opcode_buffer.append(oparg) + + self.opcode_buffer.append(Opcode(opcode, oparg)) def name_gather(self, node: Gather) -> str: self.counter += 1 @@ -187,13 +197,13 @@ def generate(self, filename: str) -> None: self.print("};") self.print() + self.print("static Rule all_rules[] = {") while self.todo: - self.print("static Rule all_rules[] = {") for rulename, rule in list(self.todo.items()): del self.todo[rulename] with self.indent(): self.visit(rule) - self.print("};") + self.print("};") self.printblock(CALL_ACTION_HEAD) with self.indent(): @@ -327,27 +337,19 @@ def add_root_rules(self) -> None: self.rules["root"] = root def visit_RootRule(self, node: RootRule) -> None: - # TODO: Refactor visit_Rule() so we can share code. self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") - opcodes_by_alt: Dict[int, List[str]] = {0: [], 1: []} - with self.set_opcode_buffer(opcodes_by_alt[0]): + first_alt = Alt([]) + second_alt = Alt([]) + + opcodes_by_alt: Dict[Alt, List[Opcode]] = {first_alt: [], second_alt: []} + with self.set_opcode_buffer(opcodes_by_alt[first_alt]): self.add_opcode("OP_RULE", f"R_{node.startrulename.upper()}") self.add_opcode("OP_SUCCESS") - with self.set_opcode_buffer(opcodes_by_alt[1]): + with self.set_opcode_buffer(opcodes_by_alt[second_alt]): self.add_opcode("OP_FAILURE") - - indexes = [0, len(opcodes_by_alt[0]), -1] - - self.print(f" {{{', '.join(map(str, indexes))}}},") - - self.print(" {") - with self.indent(): - for index, opcodes in opcodes_by_alt.items(): - self.print(", ".join(opcodes) + ",") - self.print(" },") - + self.generate_opcodes(opcodes_by_alt) self.print("},") def visit_Rule(self, node: Rule) -> None: @@ -358,7 +360,13 @@ def visit_Rule(self, node: Rule) -> None: self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") self.current_rule = node # TODO: make this a context manager - self.visit(rhs, is_loop=is_loop, is_loop1=is_loop1, is_gather=is_gather, is_leftrec=node.left_recursive) + self.visit( + rhs, + is_loop=is_loop, + is_loop1=is_loop1, + is_gather=is_gather, + is_leftrec=node.left_recursive, + ) self.print("},") def visit_NamedItem(self, node: NamedItem) -> None: @@ -381,7 +389,15 @@ def visit_NameLeaf(self, node: NameLeaf) -> None: name = node.value if name in ("NAME", "NUMBER", "STRING"): self.add_opcode(f"OP_{name}") - elif name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER", "ASYNC", "AWAIT", "TYPE_COMMENT"): + elif name in ( + "NEWLINE", + "DEDENT", + "INDENT", + "ENDMARKER", + "ASYNC", + "AWAIT", + "TYPE_COMMENT", + ): self.add_opcode("OP_TOKEN", name) else: self.add_opcode("OP_RULE", self._get_rule_opcode(name)) @@ -394,27 +410,30 @@ def visit_Cut(self, node: Cut) -> None: self.add_opcode("OP_CUT") def handle_loop_rhs( - self, node: Rhs, opcodes_by_alt: Dict[int, List[str]], collect_opcode: str, + self, node: Rhs, opcodes_by_alt: Dict[Alt, List[Opcode]], collect_opcode: str, ) -> None: self.handle_default_rhs(node, opcodes_by_alt, is_loop=True, is_gather=False) - with self.set_opcode_buffer(opcodes_by_alt[len(node.alts)]): + loop_alt = Alt([]) + with self.set_opcode_buffer(opcodes_by_alt[loop_alt]): self.add_opcode(collect_opcode) def handle_default_rhs( self, node: Rhs, - opcodes_by_alt: Dict[int, List[str]], + opcodes_by_alt: Dict[Alt, List[Opcode]], is_loop: bool = False, is_loop1: bool = False, is_gather: bool = False, is_leftrec: bool = False, ) -> None: for index, alt in enumerate(node.alts): - with self.set_opcode_buffer(opcodes_by_alt[index]): + with self.set_opcode_buffer(opcodes_by_alt[alt]): if is_leftrec and index == 0: self.add_opcode("OP_SETUP_LEFT_REC") self.visit(alt, is_loop=False, is_loop1=False, is_gather=False) - assert not (alt.action and (is_loop or is_gather)) # A loop rule can't have actions + assert not ( + alt.action and (is_loop or is_gather) + ) # A loop rule can't have actions if is_loop or is_gather: if index == 0: self.add_opcode("OP_LOOP_ITERATE") @@ -427,26 +446,40 @@ def handle_default_rhs( self.add_opcode(opcode, f"A_{self.current_rule.name.upper()}_{index}") def visit_Rhs( - self, node: Rhs, is_loop: bool = False, is_loop1: bool = False, is_gather: bool = False, + self, + node: Rhs, + is_loop: bool = False, + is_loop1: bool = False, + is_gather: bool = False, is_leftrec: bool = False, ) -> None: - opcodes_by_alt: Dict[int, List[str]] = defaultdict(list) + opcodes_by_alt: Dict[Alt, List[Opcode]] = defaultdict(list) if is_loop: opcode = "OP_LOOP_COLLECT" if is_loop1: opcode += "_NONEMPTY" self.handle_loop_rhs(node, opcodes_by_alt, opcode) else: - self.handle_default_rhs(node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather, - is_leftrec=is_leftrec) - *indexes, _ = accumulate(map(len, opcodes_by_alt.values())) + self.handle_default_rhs( + node, opcodes_by_alt, is_loop=is_loop, is_gather=is_gather, is_leftrec=is_leftrec + ) + self.generate_opcodes(opcodes_by_alt) + + def generate_opcodes(self, opcodes_by_alt: Dict[Alt, List[Opcode]]) -> None: + *indexes, _ = accumulate( + map(lambda opcodes: sum(map(len, opcodes)), opcodes_by_alt.values()) + ) indexes = [0, *indexes, -1] self.print(f" {{{', '.join(map(str, indexes))}}},") self.print(" {") with self.indent(): - for index, opcodes in opcodes_by_alt.items(): - self.print(", ".join(opcodes) + ",") + for alt, opcodes in opcodes_by_alt.items(): + self.print(f"// {alt if str(alt) else ''}") + for opcode in opcodes: + oparg_text = str(opcode.oparg) + "," if opcode.oparg else "" + self.print(f"{opcode.opcode}, {oparg_text}") + self.print() self.print(" },") def visit_Alt(self, node: Alt, is_loop: bool, is_loop1: bool, is_gather: bool) -> None: From a9a4115a9aa4ee54688b0d4f284fa23c0117c5b5 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 3 Jun 2020 18:15:43 +0300 Subject: [PATCH 59/67] Add vm target to pegen script to generate the vm parser (#130) --- Tools/peg_generator/Makefile | 3 ++ Tools/peg_generator/pegen/__main__.py | 37 +++++++++++++++++++++++ Tools/peg_generator/pegen/build.py | 37 +++++++++++++++++++++++ Tools/peg_generator/pegen/vm_generator.py | 28 ++++++++++++----- 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index e7a190c1bcd13d..1840e574f7a658 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -25,6 +25,9 @@ build: peg_extension/parse.c peg_extension/parse.c: $(GRAMMAR) $(TOKENS) pegen/*.py peg_extension/peg_extension.c ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h pegen/grammar_parser.py $(PYTHON) -m pegen -q c $(GRAMMAR) $(TOKENS) -o peg_extension/parse.c --compile-extension +generate_vm: $(GRAMMAR) $(TOKENS) pegen/*.py ../../Parser/pegen/pegen.c ../../Parser/pegen/parse_string.c ../../Parser/pegen/*.h + $(PYTHON) -m pegen -q vm $(GRAMMAR) $(TOKENS) -o ../../Parser/pegen/vmparse.h + clean: -rm -f peg_extension/*.o peg_extension/*.so peg_extension/parse.c -rm -f data/xxl.py diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 1dcbaad1c38870..2da260b517b1f4 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -16,6 +16,31 @@ from pegen.build import Grammar, Parser, Tokenizer, ParserGenerator +def generate_vm_code( + args: argparse.Namespace, +) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: + from pegen.build import build_vm_parser_and_generator + + verbose = args.verbose + verbose_tokenizer = verbose >= 3 + verbose_parser = verbose == 2 or verbose >= 4 + try: + grammar, parser, tokenizer, gen = build_vm_parser_and_generator( + args.grammar_filename, + args.tokens_filename, + args.output, + verbose_tokenizer, + verbose_parser, + ) + return grammar, parser, tokenizer, gen + except Exception as err: + if args.verbose: + raise # Show traceback + traceback.print_exception(err.__class__, err, None) + sys.stderr.write("For full traceback, use -v\n") + sys.exit(1) + + def generate_c_code( args: argparse.Namespace, ) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: @@ -116,6 +141,18 @@ def generate_python_code( "--skip-actions", action="store_true", help="Suppress code emission for rule actions", ) +vm_parser = subparsers.add_parser("vm", help="Generate the new VM parser generator") +vm_parser.set_defaults(func=generate_vm_code) +vm_parser.add_argument("grammar_filename", help="Grammar description") +vm_parser.add_argument("tokens_filename", help="Tokens description") +vm_parser.add_argument( + "-o", + "--output", + metavar="OUT", + default="vmparse.h", + help="Where to write the generated parser", +) + def main() -> None: from pegen.testutil import print_memstats diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 931ffc787523b5..58443dbf2285eb 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -8,6 +8,7 @@ from typing import Optional, Tuple, List, IO, Set, Dict from pegen.c_generator import CParserGenerator +from pegen.vm_generator import VMParserGenerator from pegen.grammar import Grammar from pegen.grammar_parser import GeneratedParser as GrammarParser from pegen.parser import Parser @@ -181,6 +182,19 @@ def build_python_generator( return gen +def build_vm_generator( + grammar: Grammar, grammar_file: str, tokens_file: str, output_file: str, +) -> ParserGenerator: + with open(tokens_file, "r") as tok_file: + all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file) + with open(output_file, "w") as file: + gen: ParserGenerator = VMParserGenerator( + grammar, all_tokens, exact_tok, non_exact_tok, file + ) + gen.generate(grammar_file) + return gen + + def build_c_parser_and_generator( grammar_file: str, tokens_file: str, @@ -246,3 +260,26 @@ def build_python_parser_and_generator( grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser) gen = build_python_generator(grammar, grammar_file, output_file, skip_actions=skip_actions,) return grammar, parser, tokenizer, gen + + +def build_vm_parser_and_generator( + grammar_file: str, + tokens_file: str, + output_file: str, + verbose_tokenizer: bool = False, + verbose_parser: bool = False, +) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: + """Generate rules, C parser, tokenizer, parser generator for a given grammar + + Args: + grammar_file (string): Path for the grammar file + tokens_file (string): Path for the tokens file + output_file (string): Path for the output file + verbose_tokenizer (bool, optional): Whether to display additional output + when generating the tokenizer. Defaults to False. + verbose_parser (bool, optional): Whether to display additional output + when generating the parser. Defaults to False. + """ + grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser) + gen = build_vm_generator(grammar, grammar_file, tokens_file, output_file) + return grammar, parser, tokenizer, gen diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index cf71abf2926888..e7b64ee9d452fe 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -7,10 +7,9 @@ import tokenize from collections import defaultdict from itertools import accumulate -from typing import Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import Any, Dict, Iterator, List, Optional, Tuple, Set, IO, Text, Union from pegen import grammar -from pegen.build import build_parser from pegen.grammar import ( Alt, Cut, @@ -77,9 +76,14 @@ def __init__(self, name: str, startrulename: str): class VMCallMakerVisitor(GrammarVisitor): def __init__( - self, parser_generator: ParserGenerator, + self, + parser_generator: ParserGenerator, + exact_tokens: Dict[str, int], + non_exact_tokens: Set[str], ): self.gen = parser_generator + self.exact_tokens = exact_tokens + self.non_exact_tokens = non_exact_tokens self.cache: Dict[Any, Any] = {} self.keyword_cache: Dict[str, int] = {} self.soft_keyword_cache: List[str] = [] @@ -101,8 +105,8 @@ def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: return self.keyword_helper(val) else: return self.soft_keyword_helper(val) - tok_num: int = token.EXACT_TOKEN_TYPES[val] # type: ignore [attr-defined] - return "OP_TOKEN", token.tok_name[tok_num] + tok_num: int = self.exact_tokens[val] + return "OP_TOKEN", self.gen.tokens[tok_num] def visit_Repeat0(self, node: Repeat0) -> str: if node in self.cache: @@ -149,12 +153,19 @@ def can_we_inline(node: Rhs) -> int: class VMParserGenerator(ParserGenerator, GrammarVisitor): def __init__( - self, grammar: grammar.Grammar, + self, + grammar: grammar.Grammar, + tokens: Dict[str, int], + exact_tokens: Dict[str, int], + non_exact_tokens: Set[str], + file: Optional[IO[Text]], ): - super().__init__(grammar, token.tok_name, sys.stdout) + super().__init__(grammar, tokens, file) self.opcode_buffer: Optional[List[Opcode]] = None - self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor(self) + self.callmakervisitor: VMCallMakerVisitor = VMCallMakerVisitor( + self, exact_tokens, non_exact_tokens, + ) @contextlib.contextmanager def set_opcode_buffer(self, buffer: List[Opcode]) -> Iterator[None]: @@ -517,6 +528,7 @@ def visit_Gather(self, node: Gather) -> None: def main() -> None: + from pegen.build import build_parser filename = "../../Grammar/python.gram" if sys.argv[1:]: filename = sys.argv[1] From 5bb2f57edcaa92b0c05078906a4959c6defcabbf Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 3 Jun 2020 08:34:33 -0700 Subject: [PATCH 60/67] Selective memoization --- Parser/pegen/vm.c | 38 ++- Parser/pegen/vm.h | 1 + Parser/pegen/vmparse.h | 281 ++++++++++++++++++++++ Tools/peg_generator/pegen/vm_generator.py | 9 + 4 files changed, 320 insertions(+), 9 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index c6fb772c79241e..4397bfd6d95f7a 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -51,8 +51,11 @@ pop_frame(Stack *stack, void *v) if (f->collection) { PyMem_Free(f->collection); } - // TODO: Selective memoization - _PyPegen_insert_memo(stack->p, f->mark, f->rule->type + 1000, v); + if (f->rule->memo) { + if (_PyPegen_insert_memo(stack->p, f->mark, f->rule->type + 1000, v) == -1) { + return NULL; + } + } f = &stack->frames[stack->top - 1]; // New top of stack D(printf(" pop %s\n", f->rule->name)); return f; @@ -145,15 +148,21 @@ run_vm(Parser *p, Rule rules[], int root) D(printf("Nothing collected for %s\n", f->rule->name)); v = NULL; f = pop_frame(&stack, v); + if (!f) { + return NULL; + } break; } // Fallthrough! case OP_LOOP_COLLECT: v = make_asdl_seq(p, f->collection, f->ncollected); - f = pop_frame(&stack, v); if (!v) { return PyErr_NoMemory(); } + f = pop_frame(&stack, v); + if (!f) { + return NULL; + } break; case OP_SAVE_MARK: @@ -193,12 +202,14 @@ run_vm(Parser *p, Rule rules[], int root) case OP_RULE: oparg = f->rule->opcodes[f->iop++]; Rule *rule = &rules[oparg]; - // TODO: Selective memoization - v = NULL; // In case is_memoized ran into an error - int memo = _PyPegen_is_memoized(p, rule->type + 1000, &v); - if (memo) { - // The result is v; if v != NULL, p->mark has been updated - break; + if (rule->memo) { + v = NULL; // In case is_memoized ran into an error + int memo = _PyPegen_is_memoized(p, rule->type + 1000, &v); + if (memo) { + D(printf(" Memo hit %s\n", rule->name)); + // The result is v; if v != NULL, p->mark has been updated + break; + } } f = push_frame(&stack, rule); goto top; @@ -206,6 +217,9 @@ run_vm(Parser *p, Rule rules[], int root) oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); f = pop_frame(&stack, v); + if (!f) { + return NULL; + } break; case OP_RETURN_LEFT_REC: @@ -241,6 +255,9 @@ run_vm(Parser *p, Rule rules[], int root) // End the recursion loop, popping the frame v = w; f = pop_frame(&stack, v); + if (!f) { + return NULL; + } } break; @@ -289,6 +306,9 @@ run_vm(Parser *p, Rule rules[], int root) pop: f = pop_frame(&stack, NULL); + if (!f) { + return NULL; + } goto fail; } diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 1f54c9f6e75e2b..96be1d6ee1117e 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -54,6 +54,7 @@ static char *opcode_names[] = { typedef struct _rule { char *name; int type; + int memo; // To memo or not int alts[MAXALTS]; int opcodes[MAXOPCODES]; } Rule; diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 88c9855fca4afa..737b9137b23954 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -849,6 +849,7 @@ enum { static Rule all_rules[] = { {"file", R_FILE, + 0, {0, -1}, { // statements? $ @@ -861,6 +862,7 @@ static Rule all_rules[] = { }, {"interactive", R_INTERACTIVE, + 0, {0, -1}, { // statement_newline @@ -871,6 +873,7 @@ static Rule all_rules[] = { }, {"eval", R_EVAL, + 0, {0, -1}, { // expressions NEWLINE* $ @@ -883,6 +886,7 @@ static Rule all_rules[] = { }, {"func_type", R_FUNC_TYPE, + 0, {0, -1}, { // '(' type_expressions? ')' '->' expression NEWLINE* $ @@ -900,6 +904,7 @@ static Rule all_rules[] = { }, {"fstring", R_FSTRING, + 0, {0, -1}, { // star_expressions @@ -910,6 +915,7 @@ static Rule all_rules[] = { }, {"type_expressions", R_TYPE_EXPRESSIONS, + 0, {0, 16, 26, 36, 48, 54, 60, -1}, { // ','.expression+ ',' '*' expression ',' '**' expression @@ -962,6 +968,7 @@ static Rule all_rules[] = { }, {"statements", R_STATEMENTS, + 0, {0, -1}, { // statement+ @@ -972,6 +979,7 @@ static Rule all_rules[] = { }, {"statement", R_STATEMENT, + 0, {0, 4, -1}, { // compound_stmt @@ -986,6 +994,7 @@ static Rule all_rules[] = { }, {"statement_newline", R_STATEMENT_NEWLINE, + 0, {0, 6, 10, 14, -1}, { // compound_stmt NEWLINE @@ -1009,6 +1018,7 @@ static Rule all_rules[] = { }, {"simple_stmt", R_SIMPLE_STMT, + 0, {0, 10, -1}, { // small_stmt !';' NEWLINE @@ -1030,6 +1040,7 @@ static Rule all_rules[] = { }, {"small_stmt", R_SMALL_STMT, + 1, // memo {0, 4, 8, 16, 24, 32, 36, 44, 52, 60, 64, 68, 76, -1}, { // assignment @@ -1112,6 +1123,7 @@ static Rule all_rules[] = { }, {"compound_stmt", R_COMPOUND_STMT, + 0, {0, 8, 16, 24, 32, 40, 48, -1}, { // &('def' | '@' | ASYNC) function_def @@ -1167,6 +1179,7 @@ static Rule all_rules[] = { }, {"assignment", R_ASSIGNMENT, + 0, {0, 10, 21, 30, 38, -1}, { // NAME ':' expression ['=' annotated_rhs] @@ -1206,6 +1219,7 @@ static Rule all_rules[] = { }, {"augassign", R_AUGASSIGN, + 0, {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, -1}, { // '+=' @@ -1264,6 +1278,7 @@ static Rule all_rules[] = { }, {"global_stmt", R_GLOBAL_STMT, + 0, {0, -1}, { // 'global' ','.NAME+ @@ -1275,6 +1290,7 @@ static Rule all_rules[] = { }, {"nonlocal_stmt", R_NONLOCAL_STMT, + 0, {0, -1}, { // 'nonlocal' ','.NAME+ @@ -1286,6 +1302,7 @@ static Rule all_rules[] = { }, {"yield_stmt", R_YIELD_STMT, + 0, {0, -1}, { // yield_expr @@ -1296,6 +1313,7 @@ static Rule all_rules[] = { }, {"assert_stmt", R_ASSERT_STMT, + 0, {0, -1}, { // 'assert' expression [',' expression] @@ -1309,6 +1327,7 @@ static Rule all_rules[] = { }, {"del_stmt", R_DEL_STMT, + 0, {0, -1}, { // 'del' del_targets @@ -1320,6 +1339,7 @@ static Rule all_rules[] = { }, {"import_stmt", R_IMPORT_STMT, + 0, {0, 4, -1}, { // import_name @@ -1334,6 +1354,7 @@ static Rule all_rules[] = { }, {"import_name", R_IMPORT_NAME, + 0, {0, -1}, { // 'import' dotted_as_names @@ -1345,6 +1366,7 @@ static Rule all_rules[] = { }, {"import_from", R_IMPORT_FROM, + 0, {0, 12, -1}, { // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets @@ -1366,6 +1388,7 @@ static Rule all_rules[] = { }, {"import_from_targets", R_IMPORT_FROM_TARGETS, + 0, {0, 11, 19, 23, -1}, { // '(' import_from_as_names ','? ')' @@ -1395,6 +1418,7 @@ static Rule all_rules[] = { }, {"import_from_as_names", R_IMPORT_FROM_AS_NAMES, + 0, {0, -1}, { // ','.import_from_as_name+ @@ -1405,6 +1429,7 @@ static Rule all_rules[] = { }, {"import_from_as_name", R_IMPORT_FROM_AS_NAME, + 0, {0, -1}, { // NAME ['as' NAME] @@ -1417,6 +1442,7 @@ static Rule all_rules[] = { }, {"dotted_as_names", R_DOTTED_AS_NAMES, + 0, {0, -1}, { // ','.dotted_as_name+ @@ -1427,6 +1453,7 @@ static Rule all_rules[] = { }, {"dotted_as_name", R_DOTTED_AS_NAME, + 0, {0, -1}, { // dotted_name ['as' NAME] @@ -1439,6 +1466,7 @@ static Rule all_rules[] = { }, {"dotted_name", R_DOTTED_NAME, + 1, // leftrec {0, 8, -1}, { // dotted_name '.' NAME @@ -1456,6 +1484,7 @@ static Rule all_rules[] = { }, {"if_stmt", R_IF_STMT, + 0, {0, 12, -1}, { // 'if' named_expression ':' block elif_stmt @@ -1479,6 +1508,7 @@ static Rule all_rules[] = { }, {"elif_stmt", R_ELIF_STMT, + 0, {0, 12, -1}, { // 'elif' named_expression ':' block elif_stmt @@ -1502,6 +1532,7 @@ static Rule all_rules[] = { }, {"else_block", R_ELSE_BLOCK, + 0, {0, -1}, { // 'else' ':' block @@ -1514,6 +1545,7 @@ static Rule all_rules[] = { }, {"while_stmt", R_WHILE_STMT, + 0, {0, -1}, { // 'while' named_expression ':' block else_block? @@ -1529,6 +1561,7 @@ static Rule all_rules[] = { }, {"for_stmt", R_FOR_STMT, + 0, {0, 20, -1}, { // 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? @@ -1562,6 +1595,7 @@ static Rule all_rules[] = { }, {"with_stmt", R_WITH_STMT, + 0, {0, 17, 30, 49, -1}, { // 'with' '(' ','.with_item+ ','? ')' ':' block @@ -1610,6 +1644,7 @@ static Rule all_rules[] = { }, {"with_item", R_WITH_ITEM, + 0, {0, -1}, { // expression ['as' target] @@ -1622,6 +1657,7 @@ static Rule all_rules[] = { }, {"try_stmt", R_TRY_STMT, + 0, {0, 10, -1}, { // 'try' ':' block finally_block @@ -1646,6 +1682,7 @@ static Rule all_rules[] = { }, {"except_block", R_EXCEPT_BLOCK, + 0, {0, 13, -1}, { // 'except' expression ['as' NAME] ':' block @@ -1667,6 +1704,7 @@ static Rule all_rules[] = { }, {"finally_block", R_FINALLY_BLOCK, + 0, {0, -1}, { // 'finally' ':' block @@ -1679,6 +1717,7 @@ static Rule all_rules[] = { }, {"return_stmt", R_RETURN_STMT, + 0, {0, -1}, { // 'return' star_expressions? @@ -1691,6 +1730,7 @@ static Rule all_rules[] = { }, {"raise_stmt", R_RAISE_STMT, + 0, {0, 9, -1}, { // 'raise' expression ['from' expression] @@ -1708,6 +1748,7 @@ static Rule all_rules[] = { }, {"function_def", R_FUNCTION_DEF, + 0, {0, 6, -1}, { // decorators function_def_raw @@ -1723,6 +1764,7 @@ static Rule all_rules[] = { }, {"function_def_raw", R_FUNCTION_DEF_RAW, + 0, {0, 22, -1}, { // 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block @@ -1760,6 +1802,7 @@ static Rule all_rules[] = { }, {"func_type_comment", R_FUNC_TYPE_COMMENT, + 0, {0, 10, 14, -1}, { // NEWLINE TYPE_COMMENT &(NEWLINE INDENT) @@ -1782,6 +1825,7 @@ static Rule all_rules[] = { }, {"params", R_PARAMS, + 0, {0, 4, -1}, { // invalid_parameters @@ -1796,6 +1840,7 @@ static Rule all_rules[] = { }, {"parameters", R_PARAMETERS, + 0, {0, 11, 20, 29, 36, -1}, { // slash_no_default param_no_default* param_with_default* star_etc? @@ -1834,6 +1879,7 @@ static Rule all_rules[] = { }, {"slash_no_default", R_SLASH_NO_DEFAULT, + 0, {0, 8, -1}, { // param_no_default+ '/' ',' @@ -1854,6 +1900,7 @@ static Rule all_rules[] = { }, {"slash_with_default", R_SLASH_WITH_DEFAULT, + 0, {0, 10, -1}, { // param_no_default* param_with_default+ '/' ',' @@ -1876,6 +1923,7 @@ static Rule all_rules[] = { }, {"star_etc", R_STAR_ETC, + 0, {0, 11, 22, 26, -1}, { // '*' param_no_default param_maybe_default* kwds? @@ -1906,6 +1954,7 @@ static Rule all_rules[] = { }, {"kwds", R_KWDS, + 0, {0, -1}, { // '**' param_no_default @@ -1917,6 +1966,7 @@ static Rule all_rules[] = { }, {"param_no_default", R_PARAM_NO_DEFAULT, + 0, {0, 9, -1}, { // param ',' TYPE_COMMENT? @@ -1939,6 +1989,7 @@ static Rule all_rules[] = { }, {"param_with_default", R_PARAM_WITH_DEFAULT, + 0, {0, 11, -1}, { // param default ',' TYPE_COMMENT? @@ -1963,6 +2014,7 @@ static Rule all_rules[] = { }, {"param_maybe_default", R_PARAM_MAYBE_DEFAULT, + 0, {0, 12, -1}, { // param default? ',' TYPE_COMMENT? @@ -1989,6 +2041,7 @@ static Rule all_rules[] = { }, {"param", R_PARAM, + 0, {0, -1}, { // NAME annotation? @@ -2001,6 +2054,7 @@ static Rule all_rules[] = { }, {"annotation", R_ANNOTATION, + 0, {0, -1}, { // ':' expression @@ -2012,6 +2066,7 @@ static Rule all_rules[] = { }, {"default", R_DEFAULT, + 0, {0, -1}, { // '=' expression @@ -2023,6 +2078,7 @@ static Rule all_rules[] = { }, {"decorators", R_DECORATORS, + 0, {0, -1}, { // (('@' named_expression NEWLINE))+ @@ -2033,6 +2089,7 @@ static Rule all_rules[] = { }, {"class_def", R_CLASS_DEF, + 0, {0, 6, -1}, { // decorators class_def_raw @@ -2048,6 +2105,7 @@ static Rule all_rules[] = { }, {"class_def_raw", R_CLASS_DEF_RAW, + 0, {0, -1}, { // 'class' NAME ['(' arguments? ')'] ':' block @@ -2063,6 +2121,7 @@ static Rule all_rules[] = { }, {"block", R_BLOCK, + 1, // memo {0, 10, 14, -1}, { // NEWLINE INDENT statements DEDENT @@ -2084,6 +2143,7 @@ static Rule all_rules[] = { }, {"expressions_list", R_EXPRESSIONS_LIST, + 0, {0, -1}, { // ','.star_expression+ ','? @@ -2096,6 +2156,7 @@ static Rule all_rules[] = { }, {"star_expressions", R_STAR_EXPRESSIONS, + 0, {0, 9, 15, -1}, { // star_expression ((',' star_expression))+ ','? @@ -2118,6 +2179,7 @@ static Rule all_rules[] = { }, {"star_expression", R_STAR_EXPRESSION, + 1, // memo {0, 6, -1}, { // '*' bitwise_or @@ -2133,6 +2195,7 @@ static Rule all_rules[] = { }, {"star_named_expressions", R_STAR_NAMED_EXPRESSIONS, + 0, {0, -1}, { // ','.star_named_expression+ ','? @@ -2145,6 +2208,7 @@ static Rule all_rules[] = { }, {"star_named_expression", R_STAR_NAMED_EXPRESSION, + 0, {0, 6, -1}, { // '*' bitwise_or @@ -2160,6 +2224,7 @@ static Rule all_rules[] = { }, {"named_expression", R_NAMED_EXPRESSION, + 0, {0, 7, 15, -1}, { // NAME ':=' expression @@ -2183,6 +2248,7 @@ static Rule all_rules[] = { }, {"annotated_rhs", R_ANNOTATED_RHS, + 0, {0, 4, -1}, { // yield_expr @@ -2197,6 +2263,7 @@ static Rule all_rules[] = { }, {"expressions", R_EXPRESSIONS, + 0, {0, 9, 15, -1}, { // expression ((',' expression))+ ','? @@ -2219,6 +2286,7 @@ static Rule all_rules[] = { }, {"expression", R_EXPRESSION, + 1, // memo {0, 12, 16, -1}, { // disjunction 'if' disjunction 'else' expression @@ -2241,6 +2309,7 @@ static Rule all_rules[] = { }, {"lambdef", R_LAMBDEF, + 0, {0, -1}, { // 'lambda' lambda_parameters? ':' expression @@ -2255,6 +2324,7 @@ static Rule all_rules[] = { }, {"lambda_parameters", R_LAMBDA_PARAMETERS, + 0, {0, 11, 20, 29, 36, -1}, { // lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc? @@ -2293,6 +2363,7 @@ static Rule all_rules[] = { }, {"lambda_slash_no_default", R_LAMBDA_SLASH_NO_DEFAULT, + 0, {0, 8, -1}, { // lambda_param_no_default+ '/' ',' @@ -2313,6 +2384,7 @@ static Rule all_rules[] = { }, {"lambda_slash_with_default", R_LAMBDA_SLASH_WITH_DEFAULT, + 0, {0, 10, -1}, { // lambda_param_no_default* lambda_param_with_default+ '/' ',' @@ -2335,6 +2407,7 @@ static Rule all_rules[] = { }, {"lambda_star_etc", R_LAMBDA_STAR_ETC, + 0, {0, 11, 22, 26, -1}, { // '*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds? @@ -2365,6 +2438,7 @@ static Rule all_rules[] = { }, {"lambda_kwds", R_LAMBDA_KWDS, + 0, {0, -1}, { // '**' lambda_param_no_default @@ -2376,6 +2450,7 @@ static Rule all_rules[] = { }, {"lambda_param_no_default", R_LAMBDA_PARAM_NO_DEFAULT, + 0, {0, 6, -1}, { // lambda_param ',' @@ -2394,6 +2469,7 @@ static Rule all_rules[] = { }, {"lambda_param_with_default", R_LAMBDA_PARAM_WITH_DEFAULT, + 0, {0, 8, -1}, { // lambda_param default ',' @@ -2414,6 +2490,7 @@ static Rule all_rules[] = { }, {"lambda_param_maybe_default", R_LAMBDA_PARAM_MAYBE_DEFAULT, + 0, {0, 9, -1}, { // lambda_param default? ',' @@ -2436,6 +2513,7 @@ static Rule all_rules[] = { }, {"lambda_param", R_LAMBDA_PARAM, + 0, {0, -1}, { // NAME @@ -2446,6 +2524,7 @@ static Rule all_rules[] = { }, {"disjunction", R_DISJUNCTION, + 1, // memo {0, 6, -1}, { // conjunction (('or' conjunction))+ @@ -2461,6 +2540,7 @@ static Rule all_rules[] = { }, {"conjunction", R_CONJUNCTION, + 1, // memo {0, 6, -1}, { // inversion (('and' inversion))+ @@ -2476,6 +2556,7 @@ static Rule all_rules[] = { }, {"inversion", R_INVERSION, + 1, // memo {0, 6, -1}, { // 'not' inversion @@ -2491,6 +2572,7 @@ static Rule all_rules[] = { }, {"comparison", R_COMPARISON, + 0, {0, 6, -1}, { // bitwise_or compare_op_bitwise_or_pair+ @@ -2506,6 +2588,7 @@ static Rule all_rules[] = { }, {"compare_op_bitwise_or_pair", R_COMPARE_OP_BITWISE_OR_PAIR, + 0, {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, -1}, { // eq_bitwise_or @@ -2552,6 +2635,7 @@ static Rule all_rules[] = { }, {"eq_bitwise_or", R_EQ_BITWISE_OR, + 0, {0, -1}, { // '==' bitwise_or @@ -2563,6 +2647,7 @@ static Rule all_rules[] = { }, {"noteq_bitwise_or", R_NOTEQ_BITWISE_OR, + 0, {0, -1}, { // ('!=') bitwise_or @@ -2574,6 +2659,7 @@ static Rule all_rules[] = { }, {"lte_bitwise_or", R_LTE_BITWISE_OR, + 0, {0, -1}, { // '<=' bitwise_or @@ -2585,6 +2671,7 @@ static Rule all_rules[] = { }, {"lt_bitwise_or", R_LT_BITWISE_OR, + 0, {0, -1}, { // '<' bitwise_or @@ -2596,6 +2683,7 @@ static Rule all_rules[] = { }, {"gte_bitwise_or", R_GTE_BITWISE_OR, + 0, {0, -1}, { // '>=' bitwise_or @@ -2607,6 +2695,7 @@ static Rule all_rules[] = { }, {"gt_bitwise_or", R_GT_BITWISE_OR, + 0, {0, -1}, { // '>' bitwise_or @@ -2618,6 +2707,7 @@ static Rule all_rules[] = { }, {"notin_bitwise_or", R_NOTIN_BITWISE_OR, + 0, {0, -1}, { // 'not' 'in' bitwise_or @@ -2630,6 +2720,7 @@ static Rule all_rules[] = { }, {"in_bitwise_or", R_IN_BITWISE_OR, + 0, {0, -1}, { // 'in' bitwise_or @@ -2641,6 +2732,7 @@ static Rule all_rules[] = { }, {"isnot_bitwise_or", R_ISNOT_BITWISE_OR, + 0, {0, -1}, { // 'is' 'not' bitwise_or @@ -2653,6 +2745,7 @@ static Rule all_rules[] = { }, {"is_bitwise_or", R_IS_BITWISE_OR, + 0, {0, -1}, { // 'is' bitwise_or @@ -2664,6 +2757,7 @@ static Rule all_rules[] = { }, {"bitwise_or", R_BITWISE_OR, + 1, // leftrec {0, 9, -1}, { // bitwise_or '|' bitwise_xor @@ -2681,6 +2775,7 @@ static Rule all_rules[] = { }, {"bitwise_xor", R_BITWISE_XOR, + 1, // leftrec {0, 9, -1}, { // bitwise_xor '^' bitwise_and @@ -2698,6 +2793,7 @@ static Rule all_rules[] = { }, {"bitwise_and", R_BITWISE_AND, + 1, // leftrec {0, 9, -1}, { // bitwise_and '&' shift_expr @@ -2715,6 +2811,7 @@ static Rule all_rules[] = { }, {"shift_expr", R_SHIFT_EXPR, + 1, // leftrec {0, 9, 17, -1}, { // shift_expr '<<' sum @@ -2738,6 +2835,7 @@ static Rule all_rules[] = { }, {"sum", R_SUM, + 1, // leftrec {0, 9, 17, -1}, { // sum '+' term @@ -2761,6 +2859,7 @@ static Rule all_rules[] = { }, {"term", R_TERM, + 1, // leftrec {0, 9, 17, 25, 33, 41, -1}, { // term '*' factor @@ -2802,6 +2901,7 @@ static Rule all_rules[] = { }, {"factor", R_FACTOR, + 1, // memo {0, 6, 12, 18, -1}, { // '+' factor @@ -2827,6 +2927,7 @@ static Rule all_rules[] = { }, {"power", R_POWER, + 0, {0, 8, -1}, { // await_primary '**' factor @@ -2843,6 +2944,7 @@ static Rule all_rules[] = { }, {"await_primary", R_AWAIT_PRIMARY, + 1, // memo {0, 6, -1}, { // AWAIT primary @@ -2858,6 +2960,7 @@ static Rule all_rules[] = { }, {"primary", R_PRIMARY, + 1, // leftrec {0, 8, 14, 25, 35, -1}, { // primary '.' NAME @@ -2895,6 +2998,7 @@ static Rule all_rules[] = { }, {"slices", R_SLICES, + 0, {0, 8, -1}, { // slice !',' @@ -2914,6 +3018,7 @@ static Rule all_rules[] = { }, {"slice", R_SLICE, + 0, {0, 13, -1}, { // expression? ':' expression? [':' expression?] @@ -2934,6 +3039,7 @@ static Rule all_rules[] = { }, {"atom", R_ATOM, + 0, {0, 3, 7, 11, 15, 19, 26, 29, 37, 45, 53, -1}, { // NAME @@ -2996,6 +3102,7 @@ static Rule all_rules[] = { }, {"strings", R_STRINGS, + 1, // memo {0, -1}, { // STRING+ @@ -3006,6 +3113,7 @@ static Rule all_rules[] = { }, {"list", R_LIST, + 0, {0, -1}, { // '[' star_named_expressions? ']' @@ -3019,6 +3127,7 @@ static Rule all_rules[] = { }, {"listcomp", R_LISTCOMP, + 0, {0, 10, -1}, { // '[' named_expression for_if_clauses ']' @@ -3036,6 +3145,7 @@ static Rule all_rules[] = { }, {"tuple", R_TUPLE, + 0, {0, -1}, { // '(' [star_named_expression ',' star_named_expressions?] ')' @@ -3049,6 +3159,7 @@ static Rule all_rules[] = { }, {"group", R_GROUP, + 0, {0, -1}, { // '(' (yield_expr | named_expression) ')' @@ -3061,6 +3172,7 @@ static Rule all_rules[] = { }, {"genexp", R_GENEXP, + 0, {0, 10, -1}, { // '(' expression for_if_clauses ')' @@ -3078,6 +3190,7 @@ static Rule all_rules[] = { }, {"set", R_SET, + 0, {0, -1}, { // '{' expressions_list '}' @@ -3090,6 +3203,7 @@ static Rule all_rules[] = { }, {"setcomp", R_SETCOMP, + 0, {0, 10, -1}, { // '{' expression for_if_clauses '}' @@ -3107,6 +3221,7 @@ static Rule all_rules[] = { }, {"dict", R_DICT, + 0, {0, -1}, { // '{' double_starred_kvpairs? '}' @@ -3120,6 +3235,7 @@ static Rule all_rules[] = { }, {"dictcomp", R_DICTCOMP, + 0, {0, 10, -1}, { // '{' kvpair for_if_clauses '}' @@ -3137,6 +3253,7 @@ static Rule all_rules[] = { }, {"double_starred_kvpairs", R_DOUBLE_STARRED_KVPAIRS, + 0, {0, -1}, { // ','.double_starred_kvpair+ ','? @@ -3149,6 +3266,7 @@ static Rule all_rules[] = { }, {"double_starred_kvpair", R_DOUBLE_STARRED_KVPAIR, + 0, {0, 6, -1}, { // '**' bitwise_or @@ -3164,6 +3282,7 @@ static Rule all_rules[] = { }, {"kvpair", R_KVPAIR, + 0, {0, -1}, { // expression ':' expression @@ -3176,6 +3295,7 @@ static Rule all_rules[] = { }, {"for_if_clauses", R_FOR_IF_CLAUSES, + 0, {0, -1}, { // for_if_clause+ @@ -3186,6 +3306,7 @@ static Rule all_rules[] = { }, {"for_if_clause", R_FOR_IF_CLAUSE, + 0, {0, 14, -1}, { // ASYNC 'for' star_targets 'in' disjunction (('if' disjunction))* @@ -3209,6 +3330,7 @@ static Rule all_rules[] = { }, {"yield_expr", R_YIELD_EXPR, + 0, {0, 8, -1}, { // 'yield' 'from' expression @@ -3227,6 +3349,7 @@ static Rule all_rules[] = { }, {"arguments", R_ARGUMENTS, + 1, // memo {0, 11, -1}, { // args ','? &')' @@ -3246,6 +3369,7 @@ static Rule all_rules[] = { }, {"args", R_ARGS, + 0, {0, 7, 11, -1}, { // starred_expression [',' args] @@ -3268,6 +3392,7 @@ static Rule all_rules[] = { }, {"kwargs", R_KWARGS, + 0, {0, 8, 12, -1}, { // ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ @@ -3288,6 +3413,7 @@ static Rule all_rules[] = { }, {"starred_expression", R_STARRED_EXPRESSION, + 0, {0, -1}, { // '*' expression @@ -3299,6 +3425,7 @@ static Rule all_rules[] = { }, {"kwarg_or_starred", R_KWARG_OR_STARRED, + 0, {0, 7, 11, -1}, { // NAME '=' expression @@ -3319,6 +3446,7 @@ static Rule all_rules[] = { }, {"kwarg_or_double_starred", R_KWARG_OR_DOUBLE_STARRED, + 0, {0, 7, 13, -1}, { // NAME '=' expression @@ -3340,6 +3468,7 @@ static Rule all_rules[] = { }, {"star_targets", R_STAR_TARGETS, + 0, {0, 8, -1}, { // star_target !',' @@ -3360,6 +3489,7 @@ static Rule all_rules[] = { }, {"star_targets_seq", R_STAR_TARGETS_SEQ, + 0, {0, -1}, { // ','.star_target+ ','? @@ -3372,6 +3502,7 @@ static Rule all_rules[] = { }, {"star_target", R_STAR_TARGET, + 1, // memo {0, 6, 17, 31, -1}, { // '*' (!'*' star_target) @@ -3406,6 +3537,7 @@ static Rule all_rules[] = { }, {"star_atom", R_STAR_ATOM, + 0, {0, 3, 11, 20, -1}, { // NAME @@ -3436,6 +3568,7 @@ static Rule all_rules[] = { }, {"single_target", R_SINGLE_TARGET, + 0, {0, 4, 7, -1}, { // single_subscript_attribute_target @@ -3456,6 +3589,7 @@ static Rule all_rules[] = { }, {"single_subscript_attribute_target", R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, + 0, {0, 11, -1}, { // t_primary '.' NAME !t_lookahead @@ -3481,6 +3615,7 @@ static Rule all_rules[] = { }, {"del_targets", R_DEL_TARGETS, + 0, {0, -1}, { // ','.del_target+ ','? @@ -3493,6 +3628,7 @@ static Rule all_rules[] = { }, {"del_target", R_DEL_TARGET, + 1, // memo {0, 11, 25, -1}, { // t_primary '.' NAME &del_target_end @@ -3522,6 +3658,7 @@ static Rule all_rules[] = { }, {"del_t_atom", R_DEL_T_ATOM, + 0, {0, 7, 15, 24, 33, -1}, { // NAME &del_target_end @@ -3559,6 +3696,7 @@ static Rule all_rules[] = { }, {"del_target_end", R_DEL_TARGET_END, + 0, {0, 4, 8, 12, 16, -1}, { // ')' @@ -3585,6 +3723,7 @@ static Rule all_rules[] = { }, {"targets", R_TARGETS, + 0, {0, -1}, { // ','.target+ ','? @@ -3597,6 +3736,7 @@ static Rule all_rules[] = { }, {"target", R_TARGET, + 1, // memo {0, 11, 25, -1}, { // t_primary '.' NAME !t_lookahead @@ -3626,6 +3766,7 @@ static Rule all_rules[] = { }, {"t_primary", R_T_PRIMARY, + 1, // leftrec {0, 12, 26, 36, 51, -1}, { // t_primary '.' NAME &t_lookahead @@ -3678,6 +3819,7 @@ static Rule all_rules[] = { }, {"t_lookahead", R_T_LOOKAHEAD, + 0, {0, 4, 8, -1}, { // '(' @@ -3696,6 +3838,7 @@ static Rule all_rules[] = { }, {"t_atom", R_T_ATOM, + 0, {0, 3, 11, 20, -1}, { // NAME @@ -3726,6 +3869,7 @@ static Rule all_rules[] = { }, {"incorrect_arguments", R_INCORRECT_ARGUMENTS, + 0, {0, 8, 19, 25, 35, -1}, { // args ',' '*' @@ -3764,6 +3908,7 @@ static Rule all_rules[] = { }, {"invalid_kwarg", R_INVALID_KWARG, + 0, {0, -1}, { // expression '=' @@ -3775,6 +3920,7 @@ static Rule all_rules[] = { }, {"invalid_named_expression", R_INVALID_NAMED_EXPRESSION, + 0, {0, -1}, { // expression ':=' expression @@ -3787,6 +3933,7 @@ static Rule all_rules[] = { }, {"invalid_assignment", R_INVALID_ASSIGNMENT, + 0, {0, 6, 12, 22, 33, 41, -1}, { // list ':' @@ -3830,6 +3977,7 @@ static Rule all_rules[] = { }, {"invalid_block", R_INVALID_BLOCK, + 0, {0, -1}, { // NEWLINE !INDENT @@ -3843,6 +3991,7 @@ static Rule all_rules[] = { }, {"invalid_comprehension", R_INVALID_COMPREHENSION, + 0, {0, -1}, { // ('[' | '(' | '{') starred_expression for_if_clauses @@ -3855,6 +4004,7 @@ static Rule all_rules[] = { }, {"invalid_dict_comprehension", R_INVALID_DICT_COMPREHENSION, + 0, {0, -1}, { // '{' '**' bitwise_or for_if_clauses '}' @@ -3869,6 +4019,7 @@ static Rule all_rules[] = { }, {"invalid_parameters", R_INVALID_PARAMETERS, + 0, {0, -1}, { // param_no_default* (slash_with_default | param_with_default+) param_no_default @@ -3881,6 +4032,7 @@ static Rule all_rules[] = { }, {"invalid_star_etc", R_INVALID_STAR_ETC, + 0, {0, 6, -1}, { // '*' (')' | ',' (')' | '**')) @@ -3898,6 +4050,7 @@ static Rule all_rules[] = { }, {"invalid_lambda_star_etc", R_INVALID_LAMBDA_STAR_ETC, + 0, {0, -1}, { // '*' (':' | ',' (':' | '**')) @@ -3909,6 +4062,7 @@ static Rule all_rules[] = { }, {"invalid_double_type_comments", R_INVALID_DOUBLE_TYPE_COMMENTS, + 0, {0, -1}, { // TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT @@ -3923,6 +4077,7 @@ static Rule all_rules[] = { }, {"invalid_del_target", R_INVALID_DEL_TARGET, + 0, {0, -1}, { // star_expression &del_target_end @@ -3936,6 +4091,7 @@ static Rule all_rules[] = { }, {"invalid_import_from_targets", R_INVALID_IMPORT_FROM_TARGETS, + 0, {0, -1}, { // import_from_as_names ',' @@ -3947,6 +4103,7 @@ static Rule all_rules[] = { }, {"root", R_ROOT, + 0, {0, 3, -1}, { // @@ -3960,6 +4117,7 @@ static Rule all_rules[] = { }, {"_loop0_1", R__LOOP0_1, + 0, {0, 3, -1}, { // NEWLINE @@ -3973,6 +4131,7 @@ static Rule all_rules[] = { }, {"_loop0_2", R__LOOP0_2, + 0, {0, 3, -1}, { // NEWLINE @@ -3986,6 +4145,7 @@ static Rule all_rules[] = { }, {"_gather_3", R__GATHER_3, + 0, {0, 5, -1}, { // expression ',' @@ -4001,6 +4161,7 @@ static Rule all_rules[] = { }, {"_gather_4", R__GATHER_4, + 0, {0, 5, -1}, { // expression ',' @@ -4016,6 +4177,7 @@ static Rule all_rules[] = { }, {"_gather_5", R__GATHER_5, + 0, {0, 5, -1}, { // expression ',' @@ -4031,6 +4193,7 @@ static Rule all_rules[] = { }, {"_gather_6", R__GATHER_6, + 0, {0, 5, -1}, { // expression ',' @@ -4046,6 +4209,7 @@ static Rule all_rules[] = { }, {"_loop1_7", R__LOOP1_7, + 0, {0, 3, -1}, { // statement @@ -4059,6 +4223,7 @@ static Rule all_rules[] = { }, {"_gather_8", R__GATHER_8, + 0, {0, 5, -1}, { // small_stmt ';' @@ -4074,6 +4239,7 @@ static Rule all_rules[] = { }, {"_tmp_9", R__TMP_9, + 0, {0, 4, -1}, { // 'import' @@ -4088,6 +4254,7 @@ static Rule all_rules[] = { }, {"_tmp_10", R__TMP_10, + 0, {0, 4, 8, -1}, { // 'def' @@ -4106,6 +4273,7 @@ static Rule all_rules[] = { }, {"_tmp_11", R__TMP_11, + 0, {0, 4, -1}, { // 'class' @@ -4120,6 +4288,7 @@ static Rule all_rules[] = { }, {"_tmp_12", R__TMP_12, + 0, {0, 4, -1}, { // 'with' @@ -4134,6 +4303,7 @@ static Rule all_rules[] = { }, {"_tmp_13", R__TMP_13, + 0, {0, 4, -1}, { // 'for' @@ -4148,6 +4318,7 @@ static Rule all_rules[] = { }, {"_tmp_14", R__TMP_14, + 0, {0, -1}, { // '=' annotated_rhs @@ -4159,6 +4330,7 @@ static Rule all_rules[] = { }, {"_tmp_15", R__TMP_15, + 0, {0, 8, -1}, { // '(' single_target ')' @@ -4175,6 +4347,7 @@ static Rule all_rules[] = { }, {"_tmp_16", R__TMP_16, + 0, {0, -1}, { // '=' annotated_rhs @@ -4186,6 +4359,7 @@ static Rule all_rules[] = { }, {"_loop1_17", R__LOOP1_17, + 0, {0, 3, -1}, { // (star_targets '=') @@ -4199,6 +4373,7 @@ static Rule all_rules[] = { }, {"_tmp_18", R__TMP_18, + 0, {0, 4, -1}, { // yield_expr @@ -4213,6 +4388,7 @@ static Rule all_rules[] = { }, {"_tmp_19", R__TMP_19, + 0, {0, 4, -1}, { // yield_expr @@ -4227,6 +4403,7 @@ static Rule all_rules[] = { }, {"_gather_20", R__GATHER_20, + 0, {0, 4, -1}, { // NAME ',' @@ -4242,6 +4419,7 @@ static Rule all_rules[] = { }, {"_gather_21", R__GATHER_21, + 0, {0, 4, -1}, { // NAME ',' @@ -4257,6 +4435,7 @@ static Rule all_rules[] = { }, {"_tmp_22", R__TMP_22, + 0, {0, -1}, { // ',' expression @@ -4268,6 +4447,7 @@ static Rule all_rules[] = { }, {"_loop0_23", R__LOOP0_23, + 0, {0, 3, -1}, { // ('.' | '...') @@ -4281,6 +4461,7 @@ static Rule all_rules[] = { }, {"_loop1_24", R__LOOP1_24, + 0, {0, 3, -1}, { // ('.' | '...') @@ -4294,6 +4475,7 @@ static Rule all_rules[] = { }, {"_gather_25", R__GATHER_25, + 0, {0, 5, -1}, { // import_from_as_name ',' @@ -4309,6 +4491,7 @@ static Rule all_rules[] = { }, {"_tmp_26", R__TMP_26, + 0, {0, -1}, { // 'as' NAME @@ -4320,6 +4503,7 @@ static Rule all_rules[] = { }, {"_gather_27", R__GATHER_27, + 0, {0, 5, -1}, { // dotted_as_name ',' @@ -4335,6 +4519,7 @@ static Rule all_rules[] = { }, {"_tmp_28", R__TMP_28, + 0, {0, -1}, { // 'as' NAME @@ -4346,6 +4531,7 @@ static Rule all_rules[] = { }, {"_gather_29", R__GATHER_29, + 0, {0, 5, -1}, { // with_item ',' @@ -4361,6 +4547,7 @@ static Rule all_rules[] = { }, {"_gather_30", R__GATHER_30, + 0, {0, 5, -1}, { // with_item ',' @@ -4376,6 +4563,7 @@ static Rule all_rules[] = { }, {"_gather_31", R__GATHER_31, + 0, {0, 5, -1}, { // with_item ',' @@ -4391,6 +4579,7 @@ static Rule all_rules[] = { }, {"_gather_32", R__GATHER_32, + 0, {0, 5, -1}, { // with_item ',' @@ -4406,6 +4595,7 @@ static Rule all_rules[] = { }, {"_tmp_33", R__TMP_33, + 0, {0, -1}, { // 'as' target @@ -4417,6 +4607,7 @@ static Rule all_rules[] = { }, {"_loop1_34", R__LOOP1_34, + 0, {0, 3, -1}, { // except_block @@ -4430,6 +4621,7 @@ static Rule all_rules[] = { }, {"_tmp_35", R__TMP_35, + 0, {0, -1}, { // 'as' NAME @@ -4441,6 +4633,7 @@ static Rule all_rules[] = { }, {"_tmp_36", R__TMP_36, + 0, {0, -1}, { // 'from' expression @@ -4452,6 +4645,7 @@ static Rule all_rules[] = { }, {"_tmp_37", R__TMP_37, + 0, {0, -1}, { // '->' expression @@ -4463,6 +4657,7 @@ static Rule all_rules[] = { }, {"_tmp_38", R__TMP_38, + 0, {0, -1}, { // '->' expression @@ -4474,6 +4669,7 @@ static Rule all_rules[] = { }, {"_tmp_39", R__TMP_39, + 0, {0, -1}, { // NEWLINE INDENT @@ -4485,6 +4681,7 @@ static Rule all_rules[] = { }, {"_loop0_40", R__LOOP0_40, + 0, {0, 3, -1}, { // param_no_default @@ -4498,6 +4695,7 @@ static Rule all_rules[] = { }, {"_loop0_41", R__LOOP0_41, + 0, {0, 3, -1}, { // param_with_default @@ -4511,6 +4709,7 @@ static Rule all_rules[] = { }, {"_loop0_42", R__LOOP0_42, + 0, {0, 3, -1}, { // param_with_default @@ -4524,6 +4723,7 @@ static Rule all_rules[] = { }, {"_loop1_43", R__LOOP1_43, + 0, {0, 3, -1}, { // param_no_default @@ -4537,6 +4737,7 @@ static Rule all_rules[] = { }, {"_loop0_44", R__LOOP0_44, + 0, {0, 3, -1}, { // param_with_default @@ -4550,6 +4751,7 @@ static Rule all_rules[] = { }, {"_loop1_45", R__LOOP1_45, + 0, {0, 3, -1}, { // param_with_default @@ -4563,6 +4765,7 @@ static Rule all_rules[] = { }, {"_loop1_46", R__LOOP1_46, + 0, {0, 3, -1}, { // param_no_default @@ -4576,6 +4779,7 @@ static Rule all_rules[] = { }, {"_loop1_47", R__LOOP1_47, + 0, {0, 3, -1}, { // param_no_default @@ -4589,6 +4793,7 @@ static Rule all_rules[] = { }, {"_loop0_48", R__LOOP0_48, + 0, {0, 3, -1}, { // param_no_default @@ -4602,6 +4807,7 @@ static Rule all_rules[] = { }, {"_loop1_49", R__LOOP1_49, + 0, {0, 3, -1}, { // param_with_default @@ -4615,6 +4821,7 @@ static Rule all_rules[] = { }, {"_loop0_50", R__LOOP0_50, + 0, {0, 3, -1}, { // param_no_default @@ -4628,6 +4835,7 @@ static Rule all_rules[] = { }, {"_loop1_51", R__LOOP1_51, + 0, {0, 3, -1}, { // param_with_default @@ -4641,6 +4849,7 @@ static Rule all_rules[] = { }, {"_loop0_52", R__LOOP0_52, + 0, {0, 3, -1}, { // param_maybe_default @@ -4654,6 +4863,7 @@ static Rule all_rules[] = { }, {"_loop1_53", R__LOOP1_53, + 0, {0, 3, -1}, { // param_maybe_default @@ -4667,6 +4877,7 @@ static Rule all_rules[] = { }, {"_loop1_54", R__LOOP1_54, + 0, {0, 3, -1}, { // ('@' named_expression NEWLINE) @@ -4680,6 +4891,7 @@ static Rule all_rules[] = { }, {"_tmp_55", R__TMP_55, + 0, {0, -1}, { // '(' arguments? ')' @@ -4693,6 +4905,7 @@ static Rule all_rules[] = { }, {"_gather_56", R__GATHER_56, + 0, {0, 5, -1}, { // star_expression ',' @@ -4708,6 +4921,7 @@ static Rule all_rules[] = { }, {"_loop1_57", R__LOOP1_57, + 0, {0, 3, -1}, { // (',' star_expression) @@ -4721,6 +4935,7 @@ static Rule all_rules[] = { }, {"_gather_58", R__GATHER_58, + 0, {0, 5, -1}, { // star_named_expression ',' @@ -4736,6 +4951,7 @@ static Rule all_rules[] = { }, {"_loop1_59", R__LOOP1_59, + 0, {0, 3, -1}, { // (',' expression) @@ -4749,6 +4965,7 @@ static Rule all_rules[] = { }, {"_loop0_60", R__LOOP0_60, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4762,6 +4979,7 @@ static Rule all_rules[] = { }, {"_loop0_61", R__LOOP0_61, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4775,6 +4993,7 @@ static Rule all_rules[] = { }, {"_loop0_62", R__LOOP0_62, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4788,6 +5007,7 @@ static Rule all_rules[] = { }, {"_loop1_63", R__LOOP1_63, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4801,6 +5021,7 @@ static Rule all_rules[] = { }, {"_loop0_64", R__LOOP0_64, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4814,6 +5035,7 @@ static Rule all_rules[] = { }, {"_loop1_65", R__LOOP1_65, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4827,6 +5049,7 @@ static Rule all_rules[] = { }, {"_loop1_66", R__LOOP1_66, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4840,6 +5063,7 @@ static Rule all_rules[] = { }, {"_loop1_67", R__LOOP1_67, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4853,6 +5077,7 @@ static Rule all_rules[] = { }, {"_loop0_68", R__LOOP0_68, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4866,6 +5091,7 @@ static Rule all_rules[] = { }, {"_loop1_69", R__LOOP1_69, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4879,6 +5105,7 @@ static Rule all_rules[] = { }, {"_loop0_70", R__LOOP0_70, + 0, {0, 3, -1}, { // lambda_param_no_default @@ -4892,6 +5119,7 @@ static Rule all_rules[] = { }, {"_loop1_71", R__LOOP1_71, + 0, {0, 3, -1}, { // lambda_param_with_default @@ -4905,6 +5133,7 @@ static Rule all_rules[] = { }, {"_loop0_72", R__LOOP0_72, + 0, {0, 3, -1}, { // lambda_param_maybe_default @@ -4918,6 +5147,7 @@ static Rule all_rules[] = { }, {"_loop1_73", R__LOOP1_73, + 0, {0, 3, -1}, { // lambda_param_maybe_default @@ -4931,6 +5161,7 @@ static Rule all_rules[] = { }, {"_loop1_74", R__LOOP1_74, + 0, {0, 3, -1}, { // ('or' conjunction) @@ -4944,6 +5175,7 @@ static Rule all_rules[] = { }, {"_loop1_75", R__LOOP1_75, + 0, {0, 3, -1}, { // ('and' inversion) @@ -4957,6 +5189,7 @@ static Rule all_rules[] = { }, {"_loop1_76", R__LOOP1_76, + 0, {0, 3, -1}, { // compare_op_bitwise_or_pair @@ -4970,6 +5203,7 @@ static Rule all_rules[] = { }, {"_tmp_77", R__TMP_77, + 0, {0, -1}, { // '!=' @@ -4980,6 +5214,7 @@ static Rule all_rules[] = { }, {"_gather_78", R__GATHER_78, + 0, {0, 5, -1}, { // slice ',' @@ -4995,6 +5230,7 @@ static Rule all_rules[] = { }, {"_tmp_79", R__TMP_79, + 0, {0, -1}, { // ':' expression? @@ -5007,6 +5243,7 @@ static Rule all_rules[] = { }, {"_tmp_80", R__TMP_80, + 0, {0, 4, 8, -1}, { // tuple @@ -5025,6 +5262,7 @@ static Rule all_rules[] = { }, {"_tmp_81", R__TMP_81, + 0, {0, 4, -1}, { // list @@ -5039,6 +5277,7 @@ static Rule all_rules[] = { }, {"_tmp_82", R__TMP_82, + 0, {0, 4, 8, 12, -1}, { // dict @@ -5061,6 +5300,7 @@ static Rule all_rules[] = { }, {"_loop1_83", R__LOOP1_83, + 0, {0, 2, -1}, { // STRING @@ -5074,6 +5314,7 @@ static Rule all_rules[] = { }, {"_tmp_84", R__TMP_84, + 0, {0, -1}, { // star_named_expression ',' star_named_expressions? @@ -5087,6 +5328,7 @@ static Rule all_rules[] = { }, {"_tmp_85", R__TMP_85, + 0, {0, 4, -1}, { // yield_expr @@ -5101,6 +5343,7 @@ static Rule all_rules[] = { }, {"_gather_86", R__GATHER_86, + 0, {0, 5, -1}, { // double_starred_kvpair ',' @@ -5116,6 +5359,7 @@ static Rule all_rules[] = { }, {"_loop1_87", R__LOOP1_87, + 0, {0, 3, -1}, { // for_if_clause @@ -5129,6 +5373,7 @@ static Rule all_rules[] = { }, {"_loop0_88", R__LOOP0_88, + 0, {0, 3, -1}, { // ('if' disjunction) @@ -5142,6 +5387,7 @@ static Rule all_rules[] = { }, {"_loop0_89", R__LOOP0_89, + 0, {0, 3, -1}, { // ('if' disjunction) @@ -5155,6 +5401,7 @@ static Rule all_rules[] = { }, {"_tmp_90", R__TMP_90, + 0, {0, -1}, { // ',' args @@ -5166,6 +5413,7 @@ static Rule all_rules[] = { }, {"_tmp_91", R__TMP_91, + 0, {0, -1}, { // ',' args @@ -5177,6 +5425,7 @@ static Rule all_rules[] = { }, {"_gather_92", R__GATHER_92, + 0, {0, 5, -1}, { // kwarg_or_starred ',' @@ -5192,6 +5441,7 @@ static Rule all_rules[] = { }, {"_gather_93", R__GATHER_93, + 0, {0, 5, -1}, { // kwarg_or_double_starred ',' @@ -5207,6 +5457,7 @@ static Rule all_rules[] = { }, {"_gather_94", R__GATHER_94, + 0, {0, 5, -1}, { // kwarg_or_starred ',' @@ -5222,6 +5473,7 @@ static Rule all_rules[] = { }, {"_gather_95", R__GATHER_95, + 0, {0, 5, -1}, { // kwarg_or_double_starred ',' @@ -5237,6 +5489,7 @@ static Rule all_rules[] = { }, {"_loop0_96", R__LOOP0_96, + 0, {0, 3, -1}, { // (',' star_target) @@ -5250,6 +5503,7 @@ static Rule all_rules[] = { }, {"_gather_97", R__GATHER_97, + 0, {0, 5, -1}, { // star_target ',' @@ -5265,6 +5519,7 @@ static Rule all_rules[] = { }, {"_tmp_98", R__TMP_98, + 0, {0, -1}, { // !'*' star_target @@ -5278,6 +5533,7 @@ static Rule all_rules[] = { }, {"_gather_99", R__GATHER_99, + 0, {0, 5, -1}, { // del_target ',' @@ -5293,6 +5549,7 @@ static Rule all_rules[] = { }, {"_gather_100", R__GATHER_100, + 0, {0, 5, -1}, { // target ',' @@ -5308,6 +5565,7 @@ static Rule all_rules[] = { }, {"_tmp_101", R__TMP_101, + 0, {0, 4, -1}, { // args @@ -5323,6 +5581,7 @@ static Rule all_rules[] = { }, {"_loop0_102", R__LOOP0_102, + 0, {0, 3, -1}, { // star_named_expressions @@ -5336,6 +5595,7 @@ static Rule all_rules[] = { }, {"_tmp_103", R__TMP_103, + 0, {0, -1}, { // '=' annotated_rhs @@ -5347,6 +5607,7 @@ static Rule all_rules[] = { }, {"_tmp_104", R__TMP_104, + 0, {0, 4, -1}, { // yield_expr @@ -5361,6 +5622,7 @@ static Rule all_rules[] = { }, {"_tmp_105", R__TMP_105, + 0, {0, 4, -1}, { // yield_expr @@ -5375,6 +5637,7 @@ static Rule all_rules[] = { }, {"_tmp_106", R__TMP_106, + 0, {0, 4, 8, -1}, { // '[' @@ -5393,6 +5656,7 @@ static Rule all_rules[] = { }, {"_loop0_107", R__LOOP0_107, + 0, {0, 3, -1}, { // param_no_default @@ -5406,6 +5670,7 @@ static Rule all_rules[] = { }, {"_tmp_108", R__TMP_108, + 0, {0, 4, -1}, { // slash_with_default @@ -5420,6 +5685,7 @@ static Rule all_rules[] = { }, {"_tmp_109", R__TMP_109, + 0, {0, 4, -1}, { // ')' @@ -5435,6 +5701,7 @@ static Rule all_rules[] = { }, {"_tmp_110", R__TMP_110, + 0, {0, 4, -1}, { // ':' @@ -5450,6 +5717,7 @@ static Rule all_rules[] = { }, {"_tmp_111", R__TMP_111, + 0, {0, -1}, { // star_targets '=' @@ -5461,6 +5729,7 @@ static Rule all_rules[] = { }, {"_tmp_112", R__TMP_112, + 0, {0, 4, -1}, { // '.' @@ -5475,6 +5744,7 @@ static Rule all_rules[] = { }, {"_tmp_113", R__TMP_113, + 0, {0, 4, -1}, { // '.' @@ -5489,6 +5759,7 @@ static Rule all_rules[] = { }, {"_tmp_114", R__TMP_114, + 0, {0, -1}, { // '@' named_expression NEWLINE @@ -5501,6 +5772,7 @@ static Rule all_rules[] = { }, {"_tmp_115", R__TMP_115, + 0, {0, -1}, { // ',' star_expression @@ -5512,6 +5784,7 @@ static Rule all_rules[] = { }, {"_tmp_116", R__TMP_116, + 0, {0, -1}, { // ',' expression @@ -5523,6 +5796,7 @@ static Rule all_rules[] = { }, {"_tmp_117", R__TMP_117, + 0, {0, -1}, { // 'or' conjunction @@ -5534,6 +5808,7 @@ static Rule all_rules[] = { }, {"_tmp_118", R__TMP_118, + 0, {0, -1}, { // 'and' inversion @@ -5545,6 +5820,7 @@ static Rule all_rules[] = { }, {"_tmp_119", R__TMP_119, + 0, {0, -1}, { // 'if' disjunction @@ -5556,6 +5832,7 @@ static Rule all_rules[] = { }, {"_tmp_120", R__TMP_120, + 0, {0, -1}, { // 'if' disjunction @@ -5567,6 +5844,7 @@ static Rule all_rules[] = { }, {"_tmp_121", R__TMP_121, + 0, {0, -1}, { // ',' star_target @@ -5578,6 +5856,7 @@ static Rule all_rules[] = { }, {"_loop1_122", R__LOOP1_122, + 0, {0, 3, -1}, { // param_with_default @@ -5591,6 +5870,7 @@ static Rule all_rules[] = { }, {"_tmp_123", R__TMP_123, + 0, {0, 4, -1}, { // ')' @@ -5605,6 +5885,7 @@ static Rule all_rules[] = { }, {"_tmp_124", R__TMP_124, + 0, {0, 4, -1}, { // ':' diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index e7b64ee9d452fe..3a8e7536d8dbd3 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -350,6 +350,7 @@ def add_root_rules(self) -> None: def visit_RootRule(self, node: RootRule) -> None: self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") + self.print(f" 0,") first_alt = Alt([]) second_alt = Alt([]) @@ -370,6 +371,14 @@ def visit_Rule(self, node: Rule) -> None: rhs = node.flatten() self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") + if node.memo or node.left_recursive: + if node.left_recursive: + tag = "leftrec" + else: + tag = "memo" + self.print(f" 1, // {tag}") + else: + self.print(f" 0,") self.current_rule = node # TODO: make this a context manager self.visit( rhs, From ba8783bdb241b90b75b2504e2e124f51aacf3d9a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 3 Jun 2020 12:20:01 -0700 Subject: [PATCH 61/67] Don't call is_memoized in OP_RETURN_LEFT_REC This reduces the number of is_memoized calls dramatically, to what it is for the recursive-descent parser (+1 for the root). However we still have 50% more calls to insert_memo. This has to be investigated later. --- Parser/pegen/vm.c | 23 +++++++++-------------- Parser/pegen/vm.h | 2 ++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 4397bfd6d95f7a..9beba90281aa73 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -179,10 +179,11 @@ run_vm(Parser *p, Rule rules[], int root) case OP_SETUP_LEFT_REC: assert(p->mark == f->mark); - assert (_PyPegen_is_memoized(p, f->rule->type + 1000, &v) == 0); if (_PyPegen_insert_memo(p, f->mark, f->rule->type + 1000, NULL) == -1) { return NULL; } + f->lastmark = p->mark; + f->lastval = NULL; goto top; case OP_SUCCESS: @@ -226,19 +227,11 @@ run_vm(Parser *p, Rule rules[], int root) oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); if (v) { - // It's a little tricky to recover the cached position: - // we must call is_memoized() for the start position, and then - // is_memoized() updates p->mark to the cached end position - int newmark = p->mark; - p->mark = f->mark; - void *w; - if (_PyPegen_is_memoized(p, f->rule->type + 1000, &w) == -1) { - return NULL; - } - if (newmark > p->mark) { - D(printf(" newmark wins\n")); + if (p->mark > f->lastmark) { + D(printf(" new mark wins\n")); // The new result is better than the cached value; update memo - p->mark = newmark; + f->lastmark = p->mark; + f->lastval = v; int ok = _PyPegen_update_memo(p, f->mark, f->rule->type + 1000, v); if (ok == -1) { return NULL; @@ -252,8 +245,10 @@ run_vm(Parser *p, Rule rules[], int root) f->iop++; // Skip over OP_SETUP_LEFT_REC goto top; } + // Restore last saved position and value + p->mark = f->lastmark; + v = f->lastval; // End the recursion loop, popping the frame - v = w; f = pop_frame(&stack, v); if (!f) { return NULL; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 96be1d6ee1117e..75419278941e02 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -71,6 +71,8 @@ typedef struct _frame { int ncollected; int capacity; int ival; + int lastmark; void **collection; void *vals[MAXVALS]; + void *lastval; } Frame; From bccd5c82b996691d3d48e45853784b25aedd9349 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 5 Jun 2020 11:56:14 -0700 Subject: [PATCH 62/67] Different way of doing left-recursion It's not any faster than before though. --- Parser/pegen/vm.c | 84 ++- Parser/pegen/vm.h | 25 +- Parser/pegen/vmparse.h | 882 ++++++++++++++-------- Tools/peg_generator/pegen/vm_generator.py | 15 +- 4 files changed, 634 insertions(+), 372 deletions(-) diff --git a/Parser/pegen/vm.c b/Parser/pegen/vm.c index 9beba90281aa73..b3f5066659c94f 100644 --- a/Parser/pegen/vm.c +++ b/Parser/pegen/vm.c @@ -52,6 +52,7 @@ pop_frame(Stack *stack, void *v) PyMem_Free(f->collection); } if (f->rule->memo) { + D(printf(" insert memo %s: val=%p, mark=%d\n", f->rule->name, v, stack->p->mark)); if (_PyPegen_insert_memo(stack->p, f->mark, f->rule->type + 1000, v) == -1) { return NULL; } @@ -145,7 +146,7 @@ run_vm(Parser *p, Rule rules[], int root) // Fallthrough! case OP_LOOP_COLLECT_NONEMPTY: if (!f->ncollected) { - D(printf("Nothing collected for %s\n", f->rule->name)); + D(printf(" Nothing collected for %s\n", f->rule->name)); v = NULL; f = pop_frame(&stack, v); if (!f) { @@ -177,15 +178,6 @@ run_vm(Parser *p, Rule rules[], int root) v = NULL; break; - case OP_SETUP_LEFT_REC: - assert(p->mark == f->mark); - if (_PyPegen_insert_memo(p, f->mark, f->rule->type + 1000, NULL) == -1) { - return NULL; - } - f->lastmark = p->mark; - f->lastval = NULL; - goto top; - case OP_SUCCESS: v = f->vals[0]; return v; @@ -203,7 +195,7 @@ run_vm(Parser *p, Rule rules[], int root) case OP_RULE: oparg = f->rule->opcodes[f->iop++]; Rule *rule = &rules[oparg]; - if (rule->memo) { + if (rule->memo || rule->leftrec) { v = NULL; // In case is_memoized ran into an error int memo = _PyPegen_is_memoized(p, rule->type + 1000, &v); if (memo) { @@ -213,42 +205,40 @@ run_vm(Parser *p, Rule rules[], int root) } } f = push_frame(&stack, rule); + if (rule->leftrec) { + D(printf(" leftrec %s prep: lastval=NULL, lastmark=%d\n", rule->name, f->mark)); + f->lastval = NULL; + f->lastmark = f->mark; + if (_PyPegen_insert_memo(p, f->mark, rule->type + 1000, NULL) == -1) { + return NULL; + } + } goto top; case OP_RETURN: - oparg = f->rule->opcodes[f->iop++]; - v = call_action(p, f, oparg); - f = pop_frame(&stack, v); - if (!f) { - return NULL; - } - break; - - case OP_RETURN_LEFT_REC: oparg = f->rule->opcodes[f->iop++]; v = call_action(p, f, oparg); if (v) { - if (p->mark > f->lastmark) { - D(printf(" new mark wins\n")); - // The new result is better than the cached value; update memo - f->lastmark = p->mark; - f->lastval = v; - int ok = _PyPegen_update_memo(p, f->mark, f->rule->type + 1000, v); - if (ok == -1) { - return NULL; + if (f->rule->leftrec) { + D(printf(" leftrec %s check\n", f->rule->name)); + if (p->mark > f->lastmark) { // We improved, recurse again + D(printf(" leftrec improved: lastval=%p, lastmark=%d\n", v, p->mark)); + f->lastval = v; + f->lastmark = p->mark; + if (_PyPegen_update_memo(p, f->mark, f->rule->type + 1000, v) == -1) { + return NULL; + } + f->ialt = 0; + f->iop = 0; + f->ival = 0; + p->mark = f->mark; + goto top; + } + else { // End recursion + D(printf(" leftrec end: lastval=%p, lastmark=%d\n", f->lastval, f->lastmark)); + p->mark = f->lastmark; + v = f->lastval; } - // Restart parsing this rule from the top - p->mark = f->mark; - f->ival = 0; - f->ialt = 0; - f->iop = f->rule->alts[0]; - assert(f->rule->opcodes[f->iop] == OP_SETUP_LEFT_REC); - f->iop++; // Skip over OP_SETUP_LEFT_REC - goto top; } - // Restore last saved position and value - p->mark = f->lastmark; - v = f->lastval; - // End the recursion loop, popping the frame f = pop_frame(&stack, v); if (!f) { return NULL; @@ -261,6 +251,7 @@ run_vm(Parser *p, Rule rules[], int root) assert(0); } + ok: if (v) { D(printf(" OK\n")); assert(f->ival < MAXVALS); @@ -289,7 +280,7 @@ run_vm(Parser *p, Rule rules[], int root) goto top; } - D(printf(" fail\n")); + D(printf(" alternative fails\n")); p->mark = f->mark; if (f->cut) goto pop; @@ -300,6 +291,17 @@ run_vm(Parser *p, Rule rules[], int root) goto top; pop: + if (f->rule->leftrec) { + D(printf(" leftrec %s pop!! lastval=%p, lastmark=%d\n", f->rule->name, f->lastval, f->lastmark)); + v = f->lastval; + p->mark = f->lastmark; + if (v) { + D(printf(" leftrec pop okay\n")); + goto ok; + } + D(printf(" leftrec pop fail\n")); + } + f = pop_frame(&stack, NULL); if (!f) { return NULL; diff --git a/Parser/pegen/vm.h b/Parser/pegen/vm.h index 75419278941e02..8039e11961100e 100644 --- a/Parser/pegen/vm.h +++ b/Parser/pegen/vm.h @@ -12,7 +12,6 @@ typedef enum _opcodes { OP_SAVE_MARK, OP_POS_LOOKAHEAD, OP_NEG_LOOKAHEAD, - OP_SETUP_LEFT_REC, OP_SUCCESS, OP_FAILURE, // The rest have an argument @@ -20,7 +19,6 @@ typedef enum _opcodes { OP_TOKEN, OP_RULE, OP_RETURN, - OP_RETURN_LEFT_REC, } Opcode; static char *opcode_names[] = { @@ -37,7 +35,6 @@ static char *opcode_names[] = { "OP_SAVE_MARK", "OP_POS_LOOKAHEAD", "OP_NEG_LOOKAHEAD", - "OP_SETUP_LEFT_REC", "OP_SUCCESS", "OP_FAILURE", // The rest have an argument @@ -45,7 +42,6 @@ static char *opcode_names[] = { "OP_TOKEN", "OP_RULE", "OP_RETURN", - "OP_RETURN_LEFT_REC", }; #define MAXALTS 15 @@ -53,10 +49,11 @@ static char *opcode_names[] = { typedef struct _rule { char *name; - int type; - int memo; // To memo or not - int alts[MAXALTS]; - int opcodes[MAXOPCODES]; + short type; + short memo; // memoized rule (not left-recursive) + short leftrec; // left-recursive rule (needs memo lookup) + short alts[MAXALTS]; + short opcodes[MAXOPCODES]; } Rule; #define MAXVALS 10 @@ -65,14 +62,14 @@ typedef struct _frame { Rule *rule; int mark; int savemark; - int ialt; - int iop; - int cut; + int lastmark; + short ialt; + short iop; + short ival; + short cut; int ncollected; int capacity; - int ival; - int lastmark; + void *lastval; void **collection; void *vals[MAXVALS]; - void *lastval; } Frame; diff --git a/Parser/pegen/vmparse.h b/Parser/pegen/vmparse.h index 737b9137b23954..31910e3e7acb6c 100644 --- a/Parser/pegen/vmparse.h +++ b/Parser/pegen/vmparse.h @@ -849,7 +849,8 @@ enum { static Rule all_rules[] = { {"file", R_FILE, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // statements? $ @@ -862,7 +863,8 @@ static Rule all_rules[] = { }, {"interactive", R_INTERACTIVE, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // statement_newline @@ -873,7 +875,8 @@ static Rule all_rules[] = { }, {"eval", R_EVAL, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // expressions NEWLINE* $ @@ -886,7 +889,8 @@ static Rule all_rules[] = { }, {"func_type", R_FUNC_TYPE, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '(' type_expressions? ')' '->' expression NEWLINE* $ @@ -904,7 +908,8 @@ static Rule all_rules[] = { }, {"fstring", R_FSTRING, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // star_expressions @@ -915,7 +920,8 @@ static Rule all_rules[] = { }, {"type_expressions", R_TYPE_EXPRESSIONS, - 0, + 0, // memo + 0, // leftrec {0, 16, 26, 36, 48, 54, 60, -1}, { // ','.expression+ ',' '*' expression ',' '**' expression @@ -968,7 +974,8 @@ static Rule all_rules[] = { }, {"statements", R_STATEMENTS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // statement+ @@ -979,7 +986,8 @@ static Rule all_rules[] = { }, {"statement", R_STATEMENT, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // compound_stmt @@ -994,7 +1002,8 @@ static Rule all_rules[] = { }, {"statement_newline", R_STATEMENT_NEWLINE, - 0, + 0, // memo + 0, // leftrec {0, 6, 10, 14, -1}, { // compound_stmt NEWLINE @@ -1018,7 +1027,8 @@ static Rule all_rules[] = { }, {"simple_stmt", R_SIMPLE_STMT, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // small_stmt !';' NEWLINE @@ -1041,6 +1051,7 @@ static Rule all_rules[] = { {"small_stmt", R_SMALL_STMT, 1, // memo + 0, // leftrec {0, 4, 8, 16, 24, 32, 36, 44, 52, 60, 64, 68, 76, -1}, { // assignment @@ -1123,7 +1134,8 @@ static Rule all_rules[] = { }, {"compound_stmt", R_COMPOUND_STMT, - 0, + 0, // memo + 0, // leftrec {0, 8, 16, 24, 32, 40, 48, -1}, { // &('def' | '@' | ASYNC) function_def @@ -1179,7 +1191,8 @@ static Rule all_rules[] = { }, {"assignment", R_ASSIGNMENT, - 0, + 0, // memo + 0, // leftrec {0, 10, 21, 30, 38, -1}, { // NAME ':' expression ['=' annotated_rhs] @@ -1219,7 +1232,8 @@ static Rule all_rules[] = { }, {"augassign", R_AUGASSIGN, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, -1}, { // '+=' @@ -1278,7 +1292,8 @@ static Rule all_rules[] = { }, {"global_stmt", R_GLOBAL_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'global' ','.NAME+ @@ -1290,7 +1305,8 @@ static Rule all_rules[] = { }, {"nonlocal_stmt", R_NONLOCAL_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'nonlocal' ','.NAME+ @@ -1302,7 +1318,8 @@ static Rule all_rules[] = { }, {"yield_stmt", R_YIELD_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // yield_expr @@ -1313,7 +1330,8 @@ static Rule all_rules[] = { }, {"assert_stmt", R_ASSERT_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'assert' expression [',' expression] @@ -1327,7 +1345,8 @@ static Rule all_rules[] = { }, {"del_stmt", R_DEL_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'del' del_targets @@ -1339,7 +1358,8 @@ static Rule all_rules[] = { }, {"import_stmt", R_IMPORT_STMT, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // import_name @@ -1354,7 +1374,8 @@ static Rule all_rules[] = { }, {"import_name", R_IMPORT_NAME, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'import' dotted_as_names @@ -1366,7 +1387,8 @@ static Rule all_rules[] = { }, {"import_from", R_IMPORT_FROM, - 0, + 0, // memo + 0, // leftrec {0, 12, -1}, { // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets @@ -1388,7 +1410,8 @@ static Rule all_rules[] = { }, {"import_from_targets", R_IMPORT_FROM_TARGETS, - 0, + 0, // memo + 0, // leftrec {0, 11, 19, 23, -1}, { // '(' import_from_as_names ','? ')' @@ -1418,7 +1441,8 @@ static Rule all_rules[] = { }, {"import_from_as_names", R_IMPORT_FROM_AS_NAMES, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.import_from_as_name+ @@ -1429,7 +1453,8 @@ static Rule all_rules[] = { }, {"import_from_as_name", R_IMPORT_FROM_AS_NAME, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // NAME ['as' NAME] @@ -1442,7 +1467,8 @@ static Rule all_rules[] = { }, {"dotted_as_names", R_DOTTED_AS_NAMES, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.dotted_as_name+ @@ -1453,7 +1479,8 @@ static Rule all_rules[] = { }, {"dotted_as_name", R_DOTTED_AS_NAME, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // dotted_name ['as' NAME] @@ -1466,25 +1493,26 @@ static Rule all_rules[] = { }, {"dotted_name", R_DOTTED_NAME, + 0, // memo 1, // leftrec - {0, 8, -1}, + {0, 7, -1}, { // dotted_name '.' NAME - OP_SETUP_LEFT_REC, OP_RULE, R_DOTTED_NAME, OP_TOKEN, DOT, OP_NAME, - OP_RETURN_LEFT_REC, A_DOTTED_NAME_0, + OP_RETURN, A_DOTTED_NAME_0, // NAME OP_NAME, - OP_RETURN_LEFT_REC, A_DOTTED_NAME_1, + OP_RETURN, A_DOTTED_NAME_1, }, }, {"if_stmt", R_IF_STMT, - 0, + 0, // memo + 0, // leftrec {0, 12, -1}, { // 'if' named_expression ':' block elif_stmt @@ -1508,7 +1536,8 @@ static Rule all_rules[] = { }, {"elif_stmt", R_ELIF_STMT, - 0, + 0, // memo + 0, // leftrec {0, 12, -1}, { // 'elif' named_expression ':' block elif_stmt @@ -1532,7 +1561,8 @@ static Rule all_rules[] = { }, {"else_block", R_ELSE_BLOCK, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'else' ':' block @@ -1545,7 +1575,8 @@ static Rule all_rules[] = { }, {"while_stmt", R_WHILE_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'while' named_expression ':' block else_block? @@ -1561,7 +1592,8 @@ static Rule all_rules[] = { }, {"for_stmt", R_FOR_STMT, - 0, + 0, // memo + 0, // leftrec {0, 20, -1}, { // 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? @@ -1595,7 +1627,8 @@ static Rule all_rules[] = { }, {"with_stmt", R_WITH_STMT, - 0, + 0, // memo + 0, // leftrec {0, 17, 30, 49, -1}, { // 'with' '(' ','.with_item+ ','? ')' ':' block @@ -1644,7 +1677,8 @@ static Rule all_rules[] = { }, {"with_item", R_WITH_ITEM, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // expression ['as' target] @@ -1657,7 +1691,8 @@ static Rule all_rules[] = { }, {"try_stmt", R_TRY_STMT, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // 'try' ':' block finally_block @@ -1682,7 +1717,8 @@ static Rule all_rules[] = { }, {"except_block", R_EXCEPT_BLOCK, - 0, + 0, // memo + 0, // leftrec {0, 13, -1}, { // 'except' expression ['as' NAME] ':' block @@ -1704,7 +1740,8 @@ static Rule all_rules[] = { }, {"finally_block", R_FINALLY_BLOCK, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'finally' ':' block @@ -1717,7 +1754,8 @@ static Rule all_rules[] = { }, {"return_stmt", R_RETURN_STMT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'return' star_expressions? @@ -1730,7 +1768,8 @@ static Rule all_rules[] = { }, {"raise_stmt", R_RAISE_STMT, - 0, + 0, // memo + 0, // leftrec {0, 9, -1}, { // 'raise' expression ['from' expression] @@ -1748,7 +1787,8 @@ static Rule all_rules[] = { }, {"function_def", R_FUNCTION_DEF, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // decorators function_def_raw @@ -1764,7 +1804,8 @@ static Rule all_rules[] = { }, {"function_def_raw", R_FUNCTION_DEF_RAW, - 0, + 0, // memo + 0, // leftrec {0, 22, -1}, { // 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block @@ -1802,7 +1843,8 @@ static Rule all_rules[] = { }, {"func_type_comment", R_FUNC_TYPE_COMMENT, - 0, + 0, // memo + 0, // leftrec {0, 10, 14, -1}, { // NEWLINE TYPE_COMMENT &(NEWLINE INDENT) @@ -1825,7 +1867,8 @@ static Rule all_rules[] = { }, {"params", R_PARAMS, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // invalid_parameters @@ -1840,7 +1883,8 @@ static Rule all_rules[] = { }, {"parameters", R_PARAMETERS, - 0, + 0, // memo + 0, // leftrec {0, 11, 20, 29, 36, -1}, { // slash_no_default param_no_default* param_with_default* star_etc? @@ -1879,7 +1923,8 @@ static Rule all_rules[] = { }, {"slash_no_default", R_SLASH_NO_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // param_no_default+ '/' ',' @@ -1900,7 +1945,8 @@ static Rule all_rules[] = { }, {"slash_with_default", R_SLASH_WITH_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // param_no_default* param_with_default+ '/' ',' @@ -1923,7 +1969,8 @@ static Rule all_rules[] = { }, {"star_etc", R_STAR_ETC, - 0, + 0, // memo + 0, // leftrec {0, 11, 22, 26, -1}, { // '*' param_no_default param_maybe_default* kwds? @@ -1954,7 +2001,8 @@ static Rule all_rules[] = { }, {"kwds", R_KWDS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '**' param_no_default @@ -1966,7 +2014,8 @@ static Rule all_rules[] = { }, {"param_no_default", R_PARAM_NO_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 9, -1}, { // param ',' TYPE_COMMENT? @@ -1989,7 +2038,8 @@ static Rule all_rules[] = { }, {"param_with_default", R_PARAM_WITH_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 11, -1}, { // param default ',' TYPE_COMMENT? @@ -2014,7 +2064,8 @@ static Rule all_rules[] = { }, {"param_maybe_default", R_PARAM_MAYBE_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 12, -1}, { // param default? ',' TYPE_COMMENT? @@ -2041,7 +2092,8 @@ static Rule all_rules[] = { }, {"param", R_PARAM, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // NAME annotation? @@ -2054,7 +2106,8 @@ static Rule all_rules[] = { }, {"annotation", R_ANNOTATION, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ':' expression @@ -2066,7 +2119,8 @@ static Rule all_rules[] = { }, {"default", R_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '=' expression @@ -2078,7 +2132,8 @@ static Rule all_rules[] = { }, {"decorators", R_DECORATORS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // (('@' named_expression NEWLINE))+ @@ -2089,7 +2144,8 @@ static Rule all_rules[] = { }, {"class_def", R_CLASS_DEF, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // decorators class_def_raw @@ -2105,7 +2161,8 @@ static Rule all_rules[] = { }, {"class_def_raw", R_CLASS_DEF_RAW, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'class' NAME ['(' arguments? ')'] ':' block @@ -2122,6 +2179,7 @@ static Rule all_rules[] = { {"block", R_BLOCK, 1, // memo + 0, // leftrec {0, 10, 14, -1}, { // NEWLINE INDENT statements DEDENT @@ -2143,7 +2201,8 @@ static Rule all_rules[] = { }, {"expressions_list", R_EXPRESSIONS_LIST, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.star_expression+ ','? @@ -2156,7 +2215,8 @@ static Rule all_rules[] = { }, {"star_expressions", R_STAR_EXPRESSIONS, - 0, + 0, // memo + 0, // leftrec {0, 9, 15, -1}, { // star_expression ((',' star_expression))+ ','? @@ -2180,6 +2240,7 @@ static Rule all_rules[] = { {"star_expression", R_STAR_EXPRESSION, 1, // memo + 0, // leftrec {0, 6, -1}, { // '*' bitwise_or @@ -2195,7 +2256,8 @@ static Rule all_rules[] = { }, {"star_named_expressions", R_STAR_NAMED_EXPRESSIONS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.star_named_expression+ ','? @@ -2208,7 +2270,8 @@ static Rule all_rules[] = { }, {"star_named_expression", R_STAR_NAMED_EXPRESSION, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // '*' bitwise_or @@ -2224,7 +2287,8 @@ static Rule all_rules[] = { }, {"named_expression", R_NAMED_EXPRESSION, - 0, + 0, // memo + 0, // leftrec {0, 7, 15, -1}, { // NAME ':=' expression @@ -2248,7 +2312,8 @@ static Rule all_rules[] = { }, {"annotated_rhs", R_ANNOTATED_RHS, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -2263,7 +2328,8 @@ static Rule all_rules[] = { }, {"expressions", R_EXPRESSIONS, - 0, + 0, // memo + 0, // leftrec {0, 9, 15, -1}, { // expression ((',' expression))+ ','? @@ -2287,6 +2353,7 @@ static Rule all_rules[] = { {"expression", R_EXPRESSION, 1, // memo + 0, // leftrec {0, 12, 16, -1}, { // disjunction 'if' disjunction 'else' expression @@ -2309,7 +2376,8 @@ static Rule all_rules[] = { }, {"lambdef", R_LAMBDEF, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'lambda' lambda_parameters? ':' expression @@ -2324,7 +2392,8 @@ static Rule all_rules[] = { }, {"lambda_parameters", R_LAMBDA_PARAMETERS, - 0, + 0, // memo + 0, // leftrec {0, 11, 20, 29, 36, -1}, { // lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc? @@ -2363,7 +2432,8 @@ static Rule all_rules[] = { }, {"lambda_slash_no_default", R_LAMBDA_SLASH_NO_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // lambda_param_no_default+ '/' ',' @@ -2384,7 +2454,8 @@ static Rule all_rules[] = { }, {"lambda_slash_with_default", R_LAMBDA_SLASH_WITH_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // lambda_param_no_default* lambda_param_with_default+ '/' ',' @@ -2407,7 +2478,8 @@ static Rule all_rules[] = { }, {"lambda_star_etc", R_LAMBDA_STAR_ETC, - 0, + 0, // memo + 0, // leftrec {0, 11, 22, 26, -1}, { // '*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds? @@ -2438,7 +2510,8 @@ static Rule all_rules[] = { }, {"lambda_kwds", R_LAMBDA_KWDS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '**' lambda_param_no_default @@ -2450,7 +2523,8 @@ static Rule all_rules[] = { }, {"lambda_param_no_default", R_LAMBDA_PARAM_NO_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // lambda_param ',' @@ -2469,7 +2543,8 @@ static Rule all_rules[] = { }, {"lambda_param_with_default", R_LAMBDA_PARAM_WITH_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // lambda_param default ',' @@ -2490,7 +2565,8 @@ static Rule all_rules[] = { }, {"lambda_param_maybe_default", R_LAMBDA_PARAM_MAYBE_DEFAULT, - 0, + 0, // memo + 0, // leftrec {0, 9, -1}, { // lambda_param default? ',' @@ -2513,7 +2589,8 @@ static Rule all_rules[] = { }, {"lambda_param", R_LAMBDA_PARAM, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // NAME @@ -2525,6 +2602,7 @@ static Rule all_rules[] = { {"disjunction", R_DISJUNCTION, 1, // memo + 0, // leftrec {0, 6, -1}, { // conjunction (('or' conjunction))+ @@ -2541,6 +2619,7 @@ static Rule all_rules[] = { {"conjunction", R_CONJUNCTION, 1, // memo + 0, // leftrec {0, 6, -1}, { // inversion (('and' inversion))+ @@ -2557,6 +2636,7 @@ static Rule all_rules[] = { {"inversion", R_INVERSION, 1, // memo + 0, // leftrec {0, 6, -1}, { // 'not' inversion @@ -2572,7 +2652,8 @@ static Rule all_rules[] = { }, {"comparison", R_COMPARISON, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // bitwise_or compare_op_bitwise_or_pair+ @@ -2588,7 +2669,8 @@ static Rule all_rules[] = { }, {"compare_op_bitwise_or_pair", R_COMPARE_OP_BITWISE_OR_PAIR, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, 12, 16, 20, 24, 28, 32, 36, -1}, { // eq_bitwise_or @@ -2635,7 +2717,8 @@ static Rule all_rules[] = { }, {"eq_bitwise_or", R_EQ_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '==' bitwise_or @@ -2647,7 +2730,8 @@ static Rule all_rules[] = { }, {"noteq_bitwise_or", R_NOTEQ_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ('!=') bitwise_or @@ -2659,7 +2743,8 @@ static Rule all_rules[] = { }, {"lte_bitwise_or", R_LTE_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '<=' bitwise_or @@ -2671,7 +2756,8 @@ static Rule all_rules[] = { }, {"lt_bitwise_or", R_LT_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '<' bitwise_or @@ -2683,7 +2769,8 @@ static Rule all_rules[] = { }, {"gte_bitwise_or", R_GTE_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '>=' bitwise_or @@ -2695,7 +2782,8 @@ static Rule all_rules[] = { }, {"gt_bitwise_or", R_GT_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '>' bitwise_or @@ -2707,7 +2795,8 @@ static Rule all_rules[] = { }, {"notin_bitwise_or", R_NOTIN_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'not' 'in' bitwise_or @@ -2720,7 +2809,8 @@ static Rule all_rules[] = { }, {"in_bitwise_or", R_IN_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'in' bitwise_or @@ -2732,7 +2822,8 @@ static Rule all_rules[] = { }, {"isnot_bitwise_or", R_ISNOT_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'is' 'not' bitwise_or @@ -2745,7 +2836,8 @@ static Rule all_rules[] = { }, {"is_bitwise_or", R_IS_BITWISE_OR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'is' bitwise_or @@ -2757,151 +2849,152 @@ static Rule all_rules[] = { }, {"bitwise_or", R_BITWISE_OR, + 0, // memo 1, // leftrec - {0, 9, -1}, + {0, 8, -1}, { // bitwise_or '|' bitwise_xor - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_OR, OP_TOKEN, VBAR, OP_RULE, R_BITWISE_XOR, - OP_RETURN_LEFT_REC, A_BITWISE_OR_0, + OP_RETURN, A_BITWISE_OR_0, // bitwise_xor OP_RULE, R_BITWISE_XOR, - OP_RETURN_LEFT_REC, A_BITWISE_OR_1, + OP_RETURN, A_BITWISE_OR_1, }, }, {"bitwise_xor", R_BITWISE_XOR, + 0, // memo 1, // leftrec - {0, 9, -1}, + {0, 8, -1}, { // bitwise_xor '^' bitwise_and - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_XOR, OP_TOKEN, CIRCUMFLEX, OP_RULE, R_BITWISE_AND, - OP_RETURN_LEFT_REC, A_BITWISE_XOR_0, + OP_RETURN, A_BITWISE_XOR_0, // bitwise_and OP_RULE, R_BITWISE_AND, - OP_RETURN_LEFT_REC, A_BITWISE_XOR_1, + OP_RETURN, A_BITWISE_XOR_1, }, }, {"bitwise_and", R_BITWISE_AND, + 0, // memo 1, // leftrec - {0, 9, -1}, + {0, 8, -1}, { // bitwise_and '&' shift_expr - OP_SETUP_LEFT_REC, OP_RULE, R_BITWISE_AND, OP_TOKEN, AMPER, OP_RULE, R_SHIFT_EXPR, - OP_RETURN_LEFT_REC, A_BITWISE_AND_0, + OP_RETURN, A_BITWISE_AND_0, // shift_expr OP_RULE, R_SHIFT_EXPR, - OP_RETURN_LEFT_REC, A_BITWISE_AND_1, + OP_RETURN, A_BITWISE_AND_1, }, }, {"shift_expr", R_SHIFT_EXPR, + 0, // memo 1, // leftrec - {0, 9, 17, -1}, + {0, 8, 16, -1}, { // shift_expr '<<' sum - OP_SETUP_LEFT_REC, OP_RULE, R_SHIFT_EXPR, OP_TOKEN, LEFTSHIFT, OP_RULE, R_SUM, - OP_RETURN_LEFT_REC, A_SHIFT_EXPR_0, + OP_RETURN, A_SHIFT_EXPR_0, // shift_expr '>>' sum OP_RULE, R_SHIFT_EXPR, OP_TOKEN, RIGHTSHIFT, OP_RULE, R_SUM, - OP_RETURN_LEFT_REC, A_SHIFT_EXPR_1, + OP_RETURN, A_SHIFT_EXPR_1, // sum OP_RULE, R_SUM, - OP_RETURN_LEFT_REC, A_SHIFT_EXPR_2, + OP_RETURN, A_SHIFT_EXPR_2, }, }, {"sum", R_SUM, + 0, // memo 1, // leftrec - {0, 9, 17, -1}, + {0, 8, 16, -1}, { // sum '+' term - OP_SETUP_LEFT_REC, OP_RULE, R_SUM, OP_TOKEN, PLUS, OP_RULE, R_TERM, - OP_RETURN_LEFT_REC, A_SUM_0, + OP_RETURN, A_SUM_0, // sum '-' term OP_RULE, R_SUM, OP_TOKEN, MINUS, OP_RULE, R_TERM, - OP_RETURN_LEFT_REC, A_SUM_1, + OP_RETURN, A_SUM_1, // term OP_RULE, R_TERM, - OP_RETURN_LEFT_REC, A_SUM_2, + OP_RETURN, A_SUM_2, }, }, {"term", R_TERM, + 0, // memo 1, // leftrec - {0, 9, 17, 25, 33, 41, -1}, + {0, 8, 16, 24, 32, 40, -1}, { // term '*' factor - OP_SETUP_LEFT_REC, OP_RULE, R_TERM, OP_TOKEN, STAR, OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_0, + OP_RETURN, A_TERM_0, // term '/' factor OP_RULE, R_TERM, OP_TOKEN, SLASH, OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_1, + OP_RETURN, A_TERM_1, // term '//' factor OP_RULE, R_TERM, OP_TOKEN, DOUBLESLASH, OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_2, + OP_RETURN, A_TERM_2, // term '%' factor OP_RULE, R_TERM, OP_TOKEN, PERCENT, OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_3, + OP_RETURN, A_TERM_3, // term '@' factor OP_RULE, R_TERM, OP_TOKEN, AT, OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_4, + OP_RETURN, A_TERM_4, // factor OP_RULE, R_FACTOR, - OP_RETURN_LEFT_REC, A_TERM_5, + OP_RETURN, A_TERM_5, }, }, {"factor", R_FACTOR, 1, // memo + 0, // leftrec {0, 6, 12, 18, -1}, { // '+' factor @@ -2927,7 +3020,8 @@ static Rule all_rules[] = { }, {"power", R_POWER, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // await_primary '**' factor @@ -2945,6 +3039,7 @@ static Rule all_rules[] = { {"await_primary", R_AWAIT_PRIMARY, 1, // memo + 0, // leftrec {0, 6, -1}, { // AWAIT primary @@ -2960,20 +3055,20 @@ static Rule all_rules[] = { }, {"primary", R_PRIMARY, + 0, // memo 1, // leftrec - {0, 8, 14, 25, 35, -1}, + {0, 7, 13, 24, 34, -1}, { // primary '.' NAME - OP_SETUP_LEFT_REC, OP_RULE, R_PRIMARY, OP_TOKEN, DOT, OP_NAME, - OP_RETURN_LEFT_REC, A_PRIMARY_0, + OP_RETURN, A_PRIMARY_0, // primary genexp OP_RULE, R_PRIMARY, OP_RULE, R_GENEXP, - OP_RETURN_LEFT_REC, A_PRIMARY_1, + OP_RETURN, A_PRIMARY_1, // primary '(' arguments? ')' OP_RULE, R_PRIMARY, @@ -2981,24 +3076,25 @@ static Rule all_rules[] = { OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, - OP_RETURN_LEFT_REC, A_PRIMARY_2, + OP_RETURN, A_PRIMARY_2, // primary '[' slices ']' OP_RULE, R_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, - OP_RETURN_LEFT_REC, A_PRIMARY_3, + OP_RETURN, A_PRIMARY_3, // atom OP_RULE, R_ATOM, - OP_RETURN_LEFT_REC, A_PRIMARY_4, + OP_RETURN, A_PRIMARY_4, }, }, {"slices", R_SLICES, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // slice !',' @@ -3018,7 +3114,8 @@ static Rule all_rules[] = { }, {"slice", R_SLICE, - 0, + 0, // memo + 0, // leftrec {0, 13, -1}, { // expression? ':' expression? [':' expression?] @@ -3039,7 +3136,8 @@ static Rule all_rules[] = { }, {"atom", R_ATOM, - 0, + 0, // memo + 0, // leftrec {0, 3, 7, 11, 15, 19, 26, 29, 37, 45, 53, -1}, { // NAME @@ -3103,6 +3201,7 @@ static Rule all_rules[] = { {"strings", R_STRINGS, 1, // memo + 0, // leftrec {0, -1}, { // STRING+ @@ -3113,7 +3212,8 @@ static Rule all_rules[] = { }, {"list", R_LIST, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '[' star_named_expressions? ']' @@ -3127,7 +3227,8 @@ static Rule all_rules[] = { }, {"listcomp", R_LISTCOMP, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // '[' named_expression for_if_clauses ']' @@ -3145,7 +3246,8 @@ static Rule all_rules[] = { }, {"tuple", R_TUPLE, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '(' [star_named_expression ',' star_named_expressions?] ')' @@ -3159,7 +3261,8 @@ static Rule all_rules[] = { }, {"group", R_GROUP, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '(' (yield_expr | named_expression) ')' @@ -3172,7 +3275,8 @@ static Rule all_rules[] = { }, {"genexp", R_GENEXP, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // '(' expression for_if_clauses ')' @@ -3190,7 +3294,8 @@ static Rule all_rules[] = { }, {"set", R_SET, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '{' expressions_list '}' @@ -3203,7 +3308,8 @@ static Rule all_rules[] = { }, {"setcomp", R_SETCOMP, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // '{' expression for_if_clauses '}' @@ -3221,7 +3327,8 @@ static Rule all_rules[] = { }, {"dict", R_DICT, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '{' double_starred_kvpairs? '}' @@ -3235,7 +3342,8 @@ static Rule all_rules[] = { }, {"dictcomp", R_DICTCOMP, - 0, + 0, // memo + 0, // leftrec {0, 10, -1}, { // '{' kvpair for_if_clauses '}' @@ -3253,7 +3361,8 @@ static Rule all_rules[] = { }, {"double_starred_kvpairs", R_DOUBLE_STARRED_KVPAIRS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.double_starred_kvpair+ ','? @@ -3266,7 +3375,8 @@ static Rule all_rules[] = { }, {"double_starred_kvpair", R_DOUBLE_STARRED_KVPAIR, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // '**' bitwise_or @@ -3282,7 +3392,8 @@ static Rule all_rules[] = { }, {"kvpair", R_KVPAIR, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // expression ':' expression @@ -3295,7 +3406,8 @@ static Rule all_rules[] = { }, {"for_if_clauses", R_FOR_IF_CLAUSES, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // for_if_clause+ @@ -3306,7 +3418,8 @@ static Rule all_rules[] = { }, {"for_if_clause", R_FOR_IF_CLAUSE, - 0, + 0, // memo + 0, // leftrec {0, 14, -1}, { // ASYNC 'for' star_targets 'in' disjunction (('if' disjunction))* @@ -3330,7 +3443,8 @@ static Rule all_rules[] = { }, {"yield_expr", R_YIELD_EXPR, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // 'yield' 'from' expression @@ -3350,6 +3464,7 @@ static Rule all_rules[] = { {"arguments", R_ARGUMENTS, 1, // memo + 0, // leftrec {0, 11, -1}, { // args ','? &')' @@ -3369,7 +3484,8 @@ static Rule all_rules[] = { }, {"args", R_ARGS, - 0, + 0, // memo + 0, // leftrec {0, 7, 11, -1}, { // starred_expression [',' args] @@ -3392,7 +3508,8 @@ static Rule all_rules[] = { }, {"kwargs", R_KWARGS, - 0, + 0, // memo + 0, // leftrec {0, 8, 12, -1}, { // ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ @@ -3413,7 +3530,8 @@ static Rule all_rules[] = { }, {"starred_expression", R_STARRED_EXPRESSION, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '*' expression @@ -3425,7 +3543,8 @@ static Rule all_rules[] = { }, {"kwarg_or_starred", R_KWARG_OR_STARRED, - 0, + 0, // memo + 0, // leftrec {0, 7, 11, -1}, { // NAME '=' expression @@ -3446,7 +3565,8 @@ static Rule all_rules[] = { }, {"kwarg_or_double_starred", R_KWARG_OR_DOUBLE_STARRED, - 0, + 0, // memo + 0, // leftrec {0, 7, 13, -1}, { // NAME '=' expression @@ -3468,7 +3588,8 @@ static Rule all_rules[] = { }, {"star_targets", R_STAR_TARGETS, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // star_target !',' @@ -3489,7 +3610,8 @@ static Rule all_rules[] = { }, {"star_targets_seq", R_STAR_TARGETS_SEQ, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.star_target+ ','? @@ -3503,6 +3625,7 @@ static Rule all_rules[] = { {"star_target", R_STAR_TARGET, 1, // memo + 0, // leftrec {0, 6, 17, 31, -1}, { // '*' (!'*' star_target) @@ -3537,7 +3660,8 @@ static Rule all_rules[] = { }, {"star_atom", R_STAR_ATOM, - 0, + 0, // memo + 0, // leftrec {0, 3, 11, 20, -1}, { // NAME @@ -3568,7 +3692,8 @@ static Rule all_rules[] = { }, {"single_target", R_SINGLE_TARGET, - 0, + 0, // memo + 0, // leftrec {0, 4, 7, -1}, { // single_subscript_attribute_target @@ -3589,7 +3714,8 @@ static Rule all_rules[] = { }, {"single_subscript_attribute_target", R_SINGLE_SUBSCRIPT_ATTRIBUTE_TARGET, - 0, + 0, // memo + 0, // leftrec {0, 11, -1}, { // t_primary '.' NAME !t_lookahead @@ -3615,7 +3741,8 @@ static Rule all_rules[] = { }, {"del_targets", R_DEL_TARGETS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.del_target+ ','? @@ -3629,6 +3756,7 @@ static Rule all_rules[] = { {"del_target", R_DEL_TARGET, 1, // memo + 0, // leftrec {0, 11, 25, -1}, { // t_primary '.' NAME &del_target_end @@ -3658,7 +3786,8 @@ static Rule all_rules[] = { }, {"del_t_atom", R_DEL_T_ATOM, - 0, + 0, // memo + 0, // leftrec {0, 7, 15, 24, 33, -1}, { // NAME &del_target_end @@ -3696,7 +3825,8 @@ static Rule all_rules[] = { }, {"del_target_end", R_DEL_TARGET_END, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, 12, 16, -1}, { // ')' @@ -3723,7 +3853,8 @@ static Rule all_rules[] = { }, {"targets", R_TARGETS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ','.target+ ','? @@ -3737,6 +3868,7 @@ static Rule all_rules[] = { {"target", R_TARGET, 1, // memo + 0, // leftrec {0, 11, 25, -1}, { // t_primary '.' NAME !t_lookahead @@ -3766,18 +3898,18 @@ static Rule all_rules[] = { }, {"t_primary", R_T_PRIMARY, + 0, // memo 1, // leftrec - {0, 12, 26, 36, 51, -1}, + {0, 11, 25, 35, 50, -1}, { // t_primary '.' NAME &t_lookahead - OP_SETUP_LEFT_REC, OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, - OP_RETURN_LEFT_REC, A_T_PRIMARY_0, + OP_RETURN, A_T_PRIMARY_0, // t_primary '[' slices ']' &t_lookahead OP_RULE, R_T_PRIMARY, @@ -3787,7 +3919,7 @@ static Rule all_rules[] = { OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, - OP_RETURN_LEFT_REC, A_T_PRIMARY_1, + OP_RETURN, A_T_PRIMARY_1, // t_primary genexp &t_lookahead OP_RULE, R_T_PRIMARY, @@ -3795,7 +3927,7 @@ static Rule all_rules[] = { OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, - OP_RETURN_LEFT_REC, A_T_PRIMARY_2, + OP_RETURN, A_T_PRIMARY_2, // t_primary '(' arguments? ')' &t_lookahead OP_RULE, R_T_PRIMARY, @@ -3806,20 +3938,21 @@ static Rule all_rules[] = { OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, - OP_RETURN_LEFT_REC, A_T_PRIMARY_3, + OP_RETURN, A_T_PRIMARY_3, // atom &t_lookahead OP_RULE, R_ATOM, OP_SAVE_MARK, OP_RULE, R_T_LOOKAHEAD, OP_POS_LOOKAHEAD, - OP_RETURN_LEFT_REC, A_T_PRIMARY_4, + OP_RETURN, A_T_PRIMARY_4, }, }, {"t_lookahead", R_T_LOOKAHEAD, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, -1}, { // '(' @@ -3838,7 +3971,8 @@ static Rule all_rules[] = { }, {"t_atom", R_T_ATOM, - 0, + 0, // memo + 0, // leftrec {0, 3, 11, 20, -1}, { // NAME @@ -3869,7 +4003,8 @@ static Rule all_rules[] = { }, {"incorrect_arguments", R_INCORRECT_ARGUMENTS, - 0, + 0, // memo + 0, // leftrec {0, 8, 19, 25, 35, -1}, { // args ',' '*' @@ -3908,7 +4043,8 @@ static Rule all_rules[] = { }, {"invalid_kwarg", R_INVALID_KWARG, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // expression '=' @@ -3920,7 +4056,8 @@ static Rule all_rules[] = { }, {"invalid_named_expression", R_INVALID_NAMED_EXPRESSION, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // expression ':=' expression @@ -3933,7 +4070,8 @@ static Rule all_rules[] = { }, {"invalid_assignment", R_INVALID_ASSIGNMENT, - 0, + 0, // memo + 0, // leftrec {0, 6, 12, 22, 33, 41, -1}, { // list ':' @@ -3977,7 +4115,8 @@ static Rule all_rules[] = { }, {"invalid_block", R_INVALID_BLOCK, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // NEWLINE !INDENT @@ -3991,7 +4130,8 @@ static Rule all_rules[] = { }, {"invalid_comprehension", R_INVALID_COMPREHENSION, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ('[' | '(' | '{') starred_expression for_if_clauses @@ -4004,7 +4144,8 @@ static Rule all_rules[] = { }, {"invalid_dict_comprehension", R_INVALID_DICT_COMPREHENSION, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '{' '**' bitwise_or for_if_clauses '}' @@ -4019,7 +4160,8 @@ static Rule all_rules[] = { }, {"invalid_parameters", R_INVALID_PARAMETERS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // param_no_default* (slash_with_default | param_with_default+) param_no_default @@ -4032,7 +4174,8 @@ static Rule all_rules[] = { }, {"invalid_star_etc", R_INVALID_STAR_ETC, - 0, + 0, // memo + 0, // leftrec {0, 6, -1}, { // '*' (')' | ',' (')' | '**')) @@ -4050,7 +4193,8 @@ static Rule all_rules[] = { }, {"invalid_lambda_star_etc", R_INVALID_LAMBDA_STAR_ETC, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '*' (':' | ',' (':' | '**')) @@ -4062,7 +4206,8 @@ static Rule all_rules[] = { }, {"invalid_double_type_comments", R_INVALID_DOUBLE_TYPE_COMMENTS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT @@ -4077,7 +4222,8 @@ static Rule all_rules[] = { }, {"invalid_del_target", R_INVALID_DEL_TARGET, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // star_expression &del_target_end @@ -4091,7 +4237,8 @@ static Rule all_rules[] = { }, {"invalid_import_from_targets", R_INVALID_IMPORT_FROM_TARGETS, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // import_from_as_names ',' @@ -4104,6 +4251,7 @@ static Rule all_rules[] = { {"root", R_ROOT, 0, + 0, {0, 3, -1}, { // @@ -4117,7 +4265,8 @@ static Rule all_rules[] = { }, {"_loop0_1", R__LOOP0_1, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // NEWLINE @@ -4131,7 +4280,8 @@ static Rule all_rules[] = { }, {"_loop0_2", R__LOOP0_2, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // NEWLINE @@ -4145,7 +4295,8 @@ static Rule all_rules[] = { }, {"_gather_3", R__GATHER_3, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // expression ',' @@ -4161,7 +4312,8 @@ static Rule all_rules[] = { }, {"_gather_4", R__GATHER_4, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // expression ',' @@ -4177,7 +4329,8 @@ static Rule all_rules[] = { }, {"_gather_5", R__GATHER_5, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // expression ',' @@ -4193,7 +4346,8 @@ static Rule all_rules[] = { }, {"_gather_6", R__GATHER_6, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // expression ',' @@ -4209,7 +4363,8 @@ static Rule all_rules[] = { }, {"_loop1_7", R__LOOP1_7, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // statement @@ -4223,7 +4378,8 @@ static Rule all_rules[] = { }, {"_gather_8", R__GATHER_8, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // small_stmt ';' @@ -4239,7 +4395,8 @@ static Rule all_rules[] = { }, {"_tmp_9", R__TMP_9, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // 'import' @@ -4254,7 +4411,8 @@ static Rule all_rules[] = { }, {"_tmp_10", R__TMP_10, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, -1}, { // 'def' @@ -4273,7 +4431,8 @@ static Rule all_rules[] = { }, {"_tmp_11", R__TMP_11, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // 'class' @@ -4288,7 +4447,8 @@ static Rule all_rules[] = { }, {"_tmp_12", R__TMP_12, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // 'with' @@ -4303,7 +4463,8 @@ static Rule all_rules[] = { }, {"_tmp_13", R__TMP_13, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // 'for' @@ -4318,7 +4479,8 @@ static Rule all_rules[] = { }, {"_tmp_14", R__TMP_14, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '=' annotated_rhs @@ -4330,7 +4492,8 @@ static Rule all_rules[] = { }, {"_tmp_15", R__TMP_15, - 0, + 0, // memo + 0, // leftrec {0, 8, -1}, { // '(' single_target ')' @@ -4347,7 +4510,8 @@ static Rule all_rules[] = { }, {"_tmp_16", R__TMP_16, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '=' annotated_rhs @@ -4359,7 +4523,8 @@ static Rule all_rules[] = { }, {"_loop1_17", R__LOOP1_17, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // (star_targets '=') @@ -4373,7 +4538,8 @@ static Rule all_rules[] = { }, {"_tmp_18", R__TMP_18, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -4388,7 +4554,8 @@ static Rule all_rules[] = { }, {"_tmp_19", R__TMP_19, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -4403,7 +4570,8 @@ static Rule all_rules[] = { }, {"_gather_20", R__GATHER_20, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // NAME ',' @@ -4419,7 +4587,8 @@ static Rule all_rules[] = { }, {"_gather_21", R__GATHER_21, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // NAME ',' @@ -4435,7 +4604,8 @@ static Rule all_rules[] = { }, {"_tmp_22", R__TMP_22, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' expression @@ -4447,7 +4617,8 @@ static Rule all_rules[] = { }, {"_loop0_23", R__LOOP0_23, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('.' | '...') @@ -4461,7 +4632,8 @@ static Rule all_rules[] = { }, {"_loop1_24", R__LOOP1_24, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('.' | '...') @@ -4475,7 +4647,8 @@ static Rule all_rules[] = { }, {"_gather_25", R__GATHER_25, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // import_from_as_name ',' @@ -4491,7 +4664,8 @@ static Rule all_rules[] = { }, {"_tmp_26", R__TMP_26, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'as' NAME @@ -4503,7 +4677,8 @@ static Rule all_rules[] = { }, {"_gather_27", R__GATHER_27, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // dotted_as_name ',' @@ -4519,7 +4694,8 @@ static Rule all_rules[] = { }, {"_tmp_28", R__TMP_28, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'as' NAME @@ -4531,7 +4707,8 @@ static Rule all_rules[] = { }, {"_gather_29", R__GATHER_29, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // with_item ',' @@ -4547,7 +4724,8 @@ static Rule all_rules[] = { }, {"_gather_30", R__GATHER_30, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // with_item ',' @@ -4563,7 +4741,8 @@ static Rule all_rules[] = { }, {"_gather_31", R__GATHER_31, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // with_item ',' @@ -4579,7 +4758,8 @@ static Rule all_rules[] = { }, {"_gather_32", R__GATHER_32, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // with_item ',' @@ -4595,7 +4775,8 @@ static Rule all_rules[] = { }, {"_tmp_33", R__TMP_33, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'as' target @@ -4607,7 +4788,8 @@ static Rule all_rules[] = { }, {"_loop1_34", R__LOOP1_34, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // except_block @@ -4621,7 +4803,8 @@ static Rule all_rules[] = { }, {"_tmp_35", R__TMP_35, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'as' NAME @@ -4633,7 +4816,8 @@ static Rule all_rules[] = { }, {"_tmp_36", R__TMP_36, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'from' expression @@ -4645,7 +4829,8 @@ static Rule all_rules[] = { }, {"_tmp_37", R__TMP_37, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '->' expression @@ -4657,7 +4842,8 @@ static Rule all_rules[] = { }, {"_tmp_38", R__TMP_38, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '->' expression @@ -4669,7 +4855,8 @@ static Rule all_rules[] = { }, {"_tmp_39", R__TMP_39, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // NEWLINE INDENT @@ -4681,7 +4868,8 @@ static Rule all_rules[] = { }, {"_loop0_40", R__LOOP0_40, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4695,7 +4883,8 @@ static Rule all_rules[] = { }, {"_loop0_41", R__LOOP0_41, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4709,7 +4898,8 @@ static Rule all_rules[] = { }, {"_loop0_42", R__LOOP0_42, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4723,7 +4913,8 @@ static Rule all_rules[] = { }, {"_loop1_43", R__LOOP1_43, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4737,7 +4928,8 @@ static Rule all_rules[] = { }, {"_loop0_44", R__LOOP0_44, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4751,7 +4943,8 @@ static Rule all_rules[] = { }, {"_loop1_45", R__LOOP1_45, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4765,7 +4958,8 @@ static Rule all_rules[] = { }, {"_loop1_46", R__LOOP1_46, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4779,7 +4973,8 @@ static Rule all_rules[] = { }, {"_loop1_47", R__LOOP1_47, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4793,7 +4988,8 @@ static Rule all_rules[] = { }, {"_loop0_48", R__LOOP0_48, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4807,7 +5003,8 @@ static Rule all_rules[] = { }, {"_loop1_49", R__LOOP1_49, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4821,7 +5018,8 @@ static Rule all_rules[] = { }, {"_loop0_50", R__LOOP0_50, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -4835,7 +5033,8 @@ static Rule all_rules[] = { }, {"_loop1_51", R__LOOP1_51, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -4849,7 +5048,8 @@ static Rule all_rules[] = { }, {"_loop0_52", R__LOOP0_52, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_maybe_default @@ -4863,7 +5063,8 @@ static Rule all_rules[] = { }, {"_loop1_53", R__LOOP1_53, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_maybe_default @@ -4877,7 +5078,8 @@ static Rule all_rules[] = { }, {"_loop1_54", R__LOOP1_54, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('@' named_expression NEWLINE) @@ -4891,7 +5093,8 @@ static Rule all_rules[] = { }, {"_tmp_55", R__TMP_55, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '(' arguments? ')' @@ -4905,7 +5108,8 @@ static Rule all_rules[] = { }, {"_gather_56", R__GATHER_56, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // star_expression ',' @@ -4921,7 +5125,8 @@ static Rule all_rules[] = { }, {"_loop1_57", R__LOOP1_57, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // (',' star_expression) @@ -4935,7 +5140,8 @@ static Rule all_rules[] = { }, {"_gather_58", R__GATHER_58, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // star_named_expression ',' @@ -4951,7 +5157,8 @@ static Rule all_rules[] = { }, {"_loop1_59", R__LOOP1_59, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // (',' expression) @@ -4965,7 +5172,8 @@ static Rule all_rules[] = { }, {"_loop0_60", R__LOOP0_60, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -4979,7 +5187,8 @@ static Rule all_rules[] = { }, {"_loop0_61", R__LOOP0_61, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -4993,7 +5202,8 @@ static Rule all_rules[] = { }, {"_loop0_62", R__LOOP0_62, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -5007,7 +5217,8 @@ static Rule all_rules[] = { }, {"_loop1_63", R__LOOP1_63, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -5021,7 +5232,8 @@ static Rule all_rules[] = { }, {"_loop0_64", R__LOOP0_64, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -5035,7 +5247,8 @@ static Rule all_rules[] = { }, {"_loop1_65", R__LOOP1_65, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -5049,7 +5262,8 @@ static Rule all_rules[] = { }, {"_loop1_66", R__LOOP1_66, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -5063,7 +5277,8 @@ static Rule all_rules[] = { }, {"_loop1_67", R__LOOP1_67, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -5077,7 +5292,8 @@ static Rule all_rules[] = { }, {"_loop0_68", R__LOOP0_68, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -5091,7 +5307,8 @@ static Rule all_rules[] = { }, {"_loop1_69", R__LOOP1_69, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -5105,7 +5322,8 @@ static Rule all_rules[] = { }, {"_loop0_70", R__LOOP0_70, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_no_default @@ -5119,7 +5337,8 @@ static Rule all_rules[] = { }, {"_loop1_71", R__LOOP1_71, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_with_default @@ -5133,7 +5352,8 @@ static Rule all_rules[] = { }, {"_loop0_72", R__LOOP0_72, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_maybe_default @@ -5147,7 +5367,8 @@ static Rule all_rules[] = { }, {"_loop1_73", R__LOOP1_73, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // lambda_param_maybe_default @@ -5161,7 +5382,8 @@ static Rule all_rules[] = { }, {"_loop1_74", R__LOOP1_74, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('or' conjunction) @@ -5175,7 +5397,8 @@ static Rule all_rules[] = { }, {"_loop1_75", R__LOOP1_75, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('and' inversion) @@ -5189,7 +5412,8 @@ static Rule all_rules[] = { }, {"_loop1_76", R__LOOP1_76, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // compare_op_bitwise_or_pair @@ -5203,7 +5427,8 @@ static Rule all_rules[] = { }, {"_tmp_77", R__TMP_77, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '!=' @@ -5214,7 +5439,8 @@ static Rule all_rules[] = { }, {"_gather_78", R__GATHER_78, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // slice ',' @@ -5230,7 +5456,8 @@ static Rule all_rules[] = { }, {"_tmp_79", R__TMP_79, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ':' expression? @@ -5243,7 +5470,8 @@ static Rule all_rules[] = { }, {"_tmp_80", R__TMP_80, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, -1}, { // tuple @@ -5262,7 +5490,8 @@ static Rule all_rules[] = { }, {"_tmp_81", R__TMP_81, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // list @@ -5277,7 +5506,8 @@ static Rule all_rules[] = { }, {"_tmp_82", R__TMP_82, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, 12, -1}, { // dict @@ -5300,7 +5530,8 @@ static Rule all_rules[] = { }, {"_loop1_83", R__LOOP1_83, - 0, + 0, // memo + 0, // leftrec {0, 2, -1}, { // STRING @@ -5314,7 +5545,8 @@ static Rule all_rules[] = { }, {"_tmp_84", R__TMP_84, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // star_named_expression ',' star_named_expressions? @@ -5328,7 +5560,8 @@ static Rule all_rules[] = { }, {"_tmp_85", R__TMP_85, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -5343,7 +5576,8 @@ static Rule all_rules[] = { }, {"_gather_86", R__GATHER_86, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // double_starred_kvpair ',' @@ -5359,7 +5593,8 @@ static Rule all_rules[] = { }, {"_loop1_87", R__LOOP1_87, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // for_if_clause @@ -5373,7 +5608,8 @@ static Rule all_rules[] = { }, {"_loop0_88", R__LOOP0_88, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('if' disjunction) @@ -5387,7 +5623,8 @@ static Rule all_rules[] = { }, {"_loop0_89", R__LOOP0_89, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // ('if' disjunction) @@ -5401,7 +5638,8 @@ static Rule all_rules[] = { }, {"_tmp_90", R__TMP_90, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' args @@ -5413,7 +5651,8 @@ static Rule all_rules[] = { }, {"_tmp_91", R__TMP_91, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' args @@ -5425,7 +5664,8 @@ static Rule all_rules[] = { }, {"_gather_92", R__GATHER_92, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // kwarg_or_starred ',' @@ -5441,7 +5681,8 @@ static Rule all_rules[] = { }, {"_gather_93", R__GATHER_93, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // kwarg_or_double_starred ',' @@ -5457,7 +5698,8 @@ static Rule all_rules[] = { }, {"_gather_94", R__GATHER_94, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // kwarg_or_starred ',' @@ -5473,7 +5715,8 @@ static Rule all_rules[] = { }, {"_gather_95", R__GATHER_95, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // kwarg_or_double_starred ',' @@ -5489,7 +5732,8 @@ static Rule all_rules[] = { }, {"_loop0_96", R__LOOP0_96, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // (',' star_target) @@ -5503,7 +5747,8 @@ static Rule all_rules[] = { }, {"_gather_97", R__GATHER_97, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // star_target ',' @@ -5519,7 +5764,8 @@ static Rule all_rules[] = { }, {"_tmp_98", R__TMP_98, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // !'*' star_target @@ -5533,7 +5779,8 @@ static Rule all_rules[] = { }, {"_gather_99", R__GATHER_99, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // del_target ',' @@ -5549,7 +5796,8 @@ static Rule all_rules[] = { }, {"_gather_100", R__GATHER_100, - 0, + 0, // memo + 0, // leftrec {0, 5, -1}, { // target ',' @@ -5565,7 +5813,8 @@ static Rule all_rules[] = { }, {"_tmp_101", R__TMP_101, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // args @@ -5581,7 +5830,8 @@ static Rule all_rules[] = { }, {"_loop0_102", R__LOOP0_102, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // star_named_expressions @@ -5595,7 +5845,8 @@ static Rule all_rules[] = { }, {"_tmp_103", R__TMP_103, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '=' annotated_rhs @@ -5607,7 +5858,8 @@ static Rule all_rules[] = { }, {"_tmp_104", R__TMP_104, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -5622,7 +5874,8 @@ static Rule all_rules[] = { }, {"_tmp_105", R__TMP_105, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // yield_expr @@ -5637,7 +5890,8 @@ static Rule all_rules[] = { }, {"_tmp_106", R__TMP_106, - 0, + 0, // memo + 0, // leftrec {0, 4, 8, -1}, { // '[' @@ -5656,7 +5910,8 @@ static Rule all_rules[] = { }, {"_loop0_107", R__LOOP0_107, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_no_default @@ -5670,7 +5925,8 @@ static Rule all_rules[] = { }, {"_tmp_108", R__TMP_108, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // slash_with_default @@ -5685,7 +5941,8 @@ static Rule all_rules[] = { }, {"_tmp_109", R__TMP_109, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // ')' @@ -5701,7 +5958,8 @@ static Rule all_rules[] = { }, {"_tmp_110", R__TMP_110, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // ':' @@ -5717,7 +5975,8 @@ static Rule all_rules[] = { }, {"_tmp_111", R__TMP_111, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // star_targets '=' @@ -5729,7 +5988,8 @@ static Rule all_rules[] = { }, {"_tmp_112", R__TMP_112, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // '.' @@ -5744,7 +6004,8 @@ static Rule all_rules[] = { }, {"_tmp_113", R__TMP_113, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // '.' @@ -5759,7 +6020,8 @@ static Rule all_rules[] = { }, {"_tmp_114", R__TMP_114, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // '@' named_expression NEWLINE @@ -5772,7 +6034,8 @@ static Rule all_rules[] = { }, {"_tmp_115", R__TMP_115, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' star_expression @@ -5784,7 +6047,8 @@ static Rule all_rules[] = { }, {"_tmp_116", R__TMP_116, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' expression @@ -5796,7 +6060,8 @@ static Rule all_rules[] = { }, {"_tmp_117", R__TMP_117, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'or' conjunction @@ -5808,7 +6073,8 @@ static Rule all_rules[] = { }, {"_tmp_118", R__TMP_118, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'and' inversion @@ -5820,7 +6086,8 @@ static Rule all_rules[] = { }, {"_tmp_119", R__TMP_119, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'if' disjunction @@ -5832,7 +6099,8 @@ static Rule all_rules[] = { }, {"_tmp_120", R__TMP_120, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // 'if' disjunction @@ -5844,7 +6112,8 @@ static Rule all_rules[] = { }, {"_tmp_121", R__TMP_121, - 0, + 0, // memo + 0, // leftrec {0, -1}, { // ',' star_target @@ -5856,7 +6125,8 @@ static Rule all_rules[] = { }, {"_loop1_122", R__LOOP1_122, - 0, + 0, // memo + 0, // leftrec {0, 3, -1}, { // param_with_default @@ -5870,7 +6140,8 @@ static Rule all_rules[] = { }, {"_tmp_123", R__TMP_123, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // ')' @@ -5885,7 +6156,8 @@ static Rule all_rules[] = { }, {"_tmp_124", R__TMP_124, - 0, + 0, // memo + 0, // leftrec {0, 4, -1}, { // ':' diff --git a/Tools/peg_generator/pegen/vm_generator.py b/Tools/peg_generator/pegen/vm_generator.py index 3a8e7536d8dbd3..40a724e4aeea9c 100644 --- a/Tools/peg_generator/pegen/vm_generator.py +++ b/Tools/peg_generator/pegen/vm_generator.py @@ -351,6 +351,7 @@ def visit_RootRule(self, node: RootRule) -> None: self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") self.print(f" 0,") + self.print(f" 0,") first_alt = Alt([]) second_alt = Alt([]) @@ -371,14 +372,8 @@ def visit_Rule(self, node: Rule) -> None: rhs = node.flatten() self.print(f'{{"{node.name}",') self.print(f" R_{node.name.upper()},") - if node.memo or node.left_recursive: - if node.left_recursive: - tag = "leftrec" - else: - tag = "memo" - self.print(f" 1, // {tag}") - else: - self.print(f" 0,") + self.print(f" {int(node.memo)}, // memo") + self.print(f" {int(node.left_recursive)}, // leftrec") self.current_rule = node # TODO: make this a context manager self.visit( rhs, @@ -448,8 +443,6 @@ def handle_default_rhs( ) -> None: for index, alt in enumerate(node.alts): with self.set_opcode_buffer(opcodes_by_alt[alt]): - if is_leftrec and index == 0: - self.add_opcode("OP_SETUP_LEFT_REC") self.visit(alt, is_loop=False, is_loop1=False, is_gather=False) assert not ( alt.action and (is_loop or is_gather) @@ -461,8 +454,6 @@ def handle_default_rhs( self.add_opcode("OP_LOOP_COLLECT_DELIMITED") else: opcode = "OP_RETURN" - if is_leftrec: - opcode += "_LEFT_REC" self.add_opcode(opcode, f"A_{self.current_rule.name.upper()}_{index}") def visit_Rhs( From 2394b96a38a561584a2138a857b2660169b0a3f0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 15 Aug 2020 19:55:55 -0700 Subject: [PATCH 63/67] Remove leftover conflict markers --- Parser/pegen.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 6ae9e0a05ab4be..efe22539051d76 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1046,16 +1046,10 @@ compute_parser_flags(PyCompilerFlags *flags) if (flags->cf_flags & PyCF_TYPE_COMMENTS) { parser_flags |= PyPARSE_TYPE_COMMENTS; } -<<<<<<< HEAD:Parser/pegen/pegen.c - if (flags->cf_flags &PyCF_VMPARSER) { + if (flags->cf_flags & PyCF_VMPARSER) { parser_flags |= PyPARSE_VMPARSER; } - if (flags->cf_feature_version < 7) { -||||||| merged common ancestors:Parser/pegen/pegen.c - if (flags->cf_feature_version < 7) { -======= if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) { ->>>>>>> upstream/master:Parser/pegen.c parser_flags |= PyPARSE_ASYNC_HACKS; } return parser_flags; From be344996dce9934816ffe1be65e7a96c6e06d4af Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 15 Aug 2020 20:15:13 -0700 Subject: [PATCH 64/67] Fix deps for vm.o --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index d3b639b0f7e722..b7568e22d23ff2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1125,7 +1125,7 @@ PYTHON_HEADERS= \ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) -$(srcdir)/Parser/pegen/vm.o: $(srcdir)/Parser/pegen/vm.h $(srcdir)/Parser/pegen/vmparse.h +$(srcdir)/Parser/vm.o: $(srcdir)/Parser/vm.h $(srcdir)/Parser/vmparse.h ###################################################################### From b9394e41a09a0bd024c78071ed02f3077802dba8 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 15 Aug 2020 20:15:30 -0700 Subject: [PATCH 65/67] Fix includes for vm.c --- Parser/vm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/vm.c b/Parser/vm.c index b3f5066659c94f..0dff69b808d57b 100644 --- a/Parser/vm.c +++ b/Parser/vm.c @@ -1,9 +1,9 @@ #include #include -#include "../tokenizer.h" +#include "tokenizer.h" #include "pegen.h" -#include "parse_string.h" +#include "string_parser.h" #include "vm.h" #include "vmparse.h" // Generated parser tables From b64113bcf460f4a7b71d539d54c0ca66a3f43b34 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 15 Aug 2020 20:15:45 -0700 Subject: [PATCH 66/67] Regenerated vmparse.h --- Parser/vmparse.h | 1701 ++++++++++++++++++++++++++-------------------- 1 file changed, 963 insertions(+), 738 deletions(-) diff --git a/Parser/vmparse.h b/Parser/vmparse.h index 31910e3e7acb6c..a36374255aa22a 100644 --- a/Parser/vmparse.h +++ b/Parser/vmparse.h @@ -1,22 +1,22 @@ -static const int n_keyword_lists = 15; +static const int n_keyword_lists = 9; static KeywordToken *reserved_keywords[] = { NULL, NULL, (KeywordToken[]) { {"if", 510}, {"in", 518}, - {"is", 526}, - {"as", 531}, - {"or", 532}, + {"as", 520}, + {"is", 527}, + {"or", 531}, {NULL, -1}, }, (KeywordToken[]) { {"del", 503}, {"try", 511}, {"for", 517}, - {"def", 522}, - {"not", 525}, - {"and", 533}, + {"def", 523}, + {"not", 526}, + {"and", 532}, {NULL, -1}, }, (KeywordToken[]) { @@ -25,8 +25,8 @@ static KeywordToken *reserved_keywords[] = { {"elif", 515}, {"else", 516}, {"with", 519}, - {"True", 527}, - {"None", 529}, + {"True", 528}, + {"None", 530}, {NULL, -1}, }, (KeywordToken[]) { @@ -34,8 +34,8 @@ static KeywordToken *reserved_keywords[] = { {"yield", 504}, {"break", 506}, {"while", 512}, - {"class", 523}, - {"False", 528}, + {"class", 524}, + {"False", 529}, {NULL, -1}, }, (KeywordToken[]) { @@ -43,12 +43,12 @@ static KeywordToken *reserved_keywords[] = { {"assert", 505}, {"global", 508}, {"import", 513}, - {"except", 520}, - {"lambda", 524}, + {"except", 521}, + {"lambda", 525}, {NULL, -1}, }, (KeywordToken[]) { - {"finally", 521}, + {"finally", 522}, {NULL, -1}, }, (KeywordToken[]) { @@ -56,15 +56,6 @@ static KeywordToken *reserved_keywords[] = { {"nonlocal", 509}, {NULL, -1}, }, - NULL, - NULL, - NULL, - NULL, - NULL, - (KeywordToken[]) { - {"__new_parser__", 530}, - {NULL, -1}, - }, }; static const char *soft_keywords[] = { @@ -140,6 +131,7 @@ enum { R_EXPRESSIONS, R_EXPRESSION, R_LAMBDEF, + R_LAMBDA_PARAMS, R_LAMBDA_PARAMETERS, R_LAMBDA_SLASH_NO_DEFAULT, R_LAMBDA_SLASH_WITH_DEFAULT, @@ -208,7 +200,6 @@ enum { R_DEL_TARGETS, R_DEL_TARGET, R_DEL_T_ATOM, - R_DEL_TARGET_END, R_TARGETS, R_TARGET, R_T_PRIMARY, @@ -218,14 +209,19 @@ enum { R_INVALID_KWARG, R_INVALID_NAMED_EXPRESSION, R_INVALID_ASSIGNMENT, + R_INVALID_ANN_ASSIGN_TARGET, + R_INVALID_DEL_STMT, R_INVALID_BLOCK, R_INVALID_COMPREHENSION, R_INVALID_DICT_COMPREHENSION, R_INVALID_PARAMETERS, + R_INVALID_LAMBDA_PARAMETERS, R_INVALID_STAR_ETC, R_INVALID_LAMBDA_STAR_ETC, R_INVALID_DOUBLE_TYPE_COMMENTS, - R_INVALID_DEL_TARGET, + R_INVALID_WITH_ITEM, + R_INVALID_FOR_TARGET, + R_INVALID_GROUP, R_INVALID_IMPORT_FROM_TARGETS, R_ROOT, R__LOOP0_1, @@ -250,94 +246,94 @@ enum { R__GATHER_20, R__GATHER_21, R__TMP_22, - R__LOOP0_23, - R__LOOP1_24, - R__GATHER_25, - R__TMP_26, - R__GATHER_27, - R__TMP_28, - R__GATHER_29, + R__TMP_23, + R__LOOP0_24, + R__LOOP1_25, + R__GATHER_26, + R__TMP_27, + R__GATHER_28, + R__TMP_29, R__GATHER_30, R__GATHER_31, R__GATHER_32, - R__TMP_33, - R__LOOP1_34, - R__TMP_35, + R__GATHER_33, + R__TMP_34, + R__LOOP1_35, R__TMP_36, R__TMP_37, R__TMP_38, R__TMP_39, - R__LOOP0_40, + R__TMP_40, R__LOOP0_41, R__LOOP0_42, - R__LOOP1_43, - R__LOOP0_44, - R__LOOP1_45, + R__LOOP0_43, + R__LOOP1_44, + R__LOOP0_45, R__LOOP1_46, R__LOOP1_47, - R__LOOP0_48, - R__LOOP1_49, - R__LOOP0_50, - R__LOOP1_51, - R__LOOP0_52, - R__LOOP1_53, + R__LOOP1_48, + R__LOOP0_49, + R__LOOP1_50, + R__LOOP0_51, + R__LOOP1_52, + R__LOOP0_53, R__LOOP1_54, - R__TMP_55, - R__GATHER_56, - R__LOOP1_57, - R__GATHER_58, - R__LOOP1_59, - R__LOOP0_60, + R__LOOP1_55, + R__TMP_56, + R__GATHER_57, + R__LOOP1_58, + R__GATHER_59, + R__LOOP1_60, R__LOOP0_61, R__LOOP0_62, - R__LOOP1_63, - R__LOOP0_64, - R__LOOP1_65, + R__LOOP0_63, + R__LOOP1_64, + R__LOOP0_65, R__LOOP1_66, R__LOOP1_67, - R__LOOP0_68, - R__LOOP1_69, - R__LOOP0_70, - R__LOOP1_71, - R__LOOP0_72, - R__LOOP1_73, + R__LOOP1_68, + R__LOOP0_69, + R__LOOP1_70, + R__LOOP0_71, + R__LOOP1_72, + R__LOOP0_73, R__LOOP1_74, R__LOOP1_75, R__LOOP1_76, - R__TMP_77, - R__GATHER_78, - R__TMP_79, + R__LOOP1_77, + R__TMP_78, + R__GATHER_79, R__TMP_80, R__TMP_81, R__TMP_82, - R__LOOP1_83, - R__TMP_84, + R__TMP_83, + R__LOOP1_84, R__TMP_85, - R__GATHER_86, - R__LOOP1_87, - R__LOOP0_88, + R__TMP_86, + R__GATHER_87, + R__LOOP1_88, R__LOOP0_89, - R__TMP_90, + R__LOOP0_90, R__TMP_91, - R__GATHER_92, + R__TMP_92, R__GATHER_93, R__GATHER_94, R__GATHER_95, - R__LOOP0_96, - R__GATHER_97, - R__TMP_98, - R__GATHER_99, + R__GATHER_96, + R__LOOP0_97, + R__GATHER_98, + R__TMP_99, R__GATHER_100, - R__TMP_101, - R__LOOP0_102, - R__TMP_103, - R__TMP_104, - R__TMP_105, + R__GATHER_101, + R__TMP_102, + R__LOOP0_103, + R__LOOP0_104, + R__LOOP0_105, R__TMP_106, - R__LOOP0_107, - R__TMP_108, + R__TMP_107, + R__LOOP0_108, R__TMP_109, - R__TMP_110, + R__LOOP0_110, R__TMP_111, R__TMP_112, R__TMP_113, @@ -349,9 +345,15 @@ enum { R__TMP_119, R__TMP_120, R__TMP_121, - R__LOOP1_122, + R__TMP_122, R__TMP_123, R__TMP_124, + R__TMP_125, + R__TMP_126, + R__LOOP1_127, + R__LOOP1_128, + R__TMP_129, + R__TMP_130, }; enum { @@ -419,6 +421,7 @@ enum { A_YIELD_STMT_0, A_ASSERT_STMT_0, A_DEL_STMT_0, + A_DEL_STMT_1, A_IMPORT_STMT_0, A_IMPORT_STMT_1, A_IMPORT_NAME_0, @@ -442,11 +445,14 @@ enum { A_WHILE_STMT_0, A_FOR_STMT_0, A_FOR_STMT_1, + A_FOR_STMT_2, A_WITH_STMT_0, A_WITH_STMT_1, A_WITH_STMT_2, A_WITH_STMT_3, A_WITH_ITEM_0, + A_WITH_ITEM_1, + A_WITH_ITEM_2, A_TRY_STMT_0, A_TRY_STMT_1, A_EXCEPT_BLOCK_0, @@ -515,6 +521,8 @@ enum { A_EXPRESSION_1, A_EXPRESSION_2, A_LAMBDEF_0, + A_LAMBDA_PARAMS_0, + A_LAMBDA_PARAMS_1, A_LAMBDA_PARAMETERS_0, A_LAMBDA_PARAMETERS_1, A_LAMBDA_PARAMETERS_2, @@ -609,13 +617,13 @@ enum { A_ATOM_7, A_ATOM_8, A_ATOM_9, - A_ATOM_10, A_STRINGS_0, A_LIST_0, A_LISTCOMP_0, A_LISTCOMP_1, A_TUPLE_0, A_GROUP_0, + A_GROUP_1, A_GENEXP_0, A_GENEXP_1, A_SET_0, @@ -631,6 +639,7 @@ enum { A_FOR_IF_CLAUSES_0, A_FOR_IF_CLAUSE_0, A_FOR_IF_CLAUSE_1, + A_FOR_IF_CLAUSE_2, A_YIELD_EXPR_0, A_YIELD_EXPR_1, A_ARGUMENTS_0, @@ -672,12 +681,6 @@ enum { A_DEL_T_ATOM_1, A_DEL_T_ATOM_2, A_DEL_T_ATOM_3, - A_DEL_T_ATOM_4, - A_DEL_TARGET_END_0, - A_DEL_TARGET_END_1, - A_DEL_TARGET_END_2, - A_DEL_TARGET_END_3, - A_DEL_TARGET_END_4, A_TARGETS_0, A_TARGET_0, A_TARGET_1, @@ -707,15 +710,22 @@ enum { A_INVALID_ASSIGNMENT_3, A_INVALID_ASSIGNMENT_4, A_INVALID_ASSIGNMENT_5, + A_INVALID_ANN_ASSIGN_TARGET_0, + A_INVALID_ANN_ASSIGN_TARGET_1, + A_INVALID_ANN_ASSIGN_TARGET_2, + A_INVALID_DEL_STMT_0, A_INVALID_BLOCK_0, A_INVALID_COMPREHENSION_0, A_INVALID_DICT_COMPREHENSION_0, A_INVALID_PARAMETERS_0, + A_INVALID_LAMBDA_PARAMETERS_0, A_INVALID_STAR_ETC_0, A_INVALID_STAR_ETC_1, A_INVALID_LAMBDA_STAR_ETC_0, A_INVALID_DOUBLE_TYPE_COMMENTS_0, - A_INVALID_DEL_TARGET_0, + A_INVALID_WITH_ITEM_0, + A_INVALID_FOR_TARGET_0, + A_INVALID_GROUP_0, A_INVALID_IMPORT_FROM_TARGETS_0, A__GATHER_3_0, A__GATHER_3_1, @@ -751,99 +761,104 @@ enum { A__GATHER_21_0, A__GATHER_21_1, A__TMP_22_0, - A__GATHER_25_0, - A__GATHER_25_1, - A__TMP_26_0, - A__GATHER_27_0, - A__GATHER_27_1, - A__TMP_28_0, - A__GATHER_29_0, - A__GATHER_29_1, + A__TMP_23_0, + A__TMP_23_1, + A__GATHER_26_0, + A__GATHER_26_1, + A__TMP_27_0, + A__GATHER_28_0, + A__GATHER_28_1, + A__TMP_29_0, A__GATHER_30_0, A__GATHER_30_1, A__GATHER_31_0, A__GATHER_31_1, A__GATHER_32_0, A__GATHER_32_1, - A__TMP_33_0, - A__TMP_35_0, + A__GATHER_33_0, + A__GATHER_33_1, + A__TMP_34_0, + A__TMP_34_1, + A__TMP_34_2, A__TMP_36_0, A__TMP_37_0, A__TMP_38_0, A__TMP_39_0, - A__TMP_55_0, - A__GATHER_56_0, - A__GATHER_56_1, - A__GATHER_58_0, - A__GATHER_58_1, - A__TMP_77_0, - A__GATHER_78_0, - A__GATHER_78_1, - A__TMP_79_0, + A__TMP_40_0, + A__TMP_56_0, + A__GATHER_57_0, + A__GATHER_57_1, + A__GATHER_59_0, + A__GATHER_59_1, + A__TMP_78_0, + A__GATHER_79_0, + A__GATHER_79_1, A__TMP_80_0, - A__TMP_80_1, - A__TMP_80_2, A__TMP_81_0, A__TMP_81_1, + A__TMP_81_2, A__TMP_82_0, A__TMP_82_1, - A__TMP_82_2, - A__TMP_82_3, - A__TMP_84_0, + A__TMP_83_0, + A__TMP_83_1, + A__TMP_83_2, + A__TMP_83_3, A__TMP_85_0, - A__TMP_85_1, - A__GATHER_86_0, - A__GATHER_86_1, - A__TMP_90_0, + A__TMP_86_0, + A__TMP_86_1, + A__GATHER_87_0, + A__GATHER_87_1, A__TMP_91_0, - A__GATHER_92_0, - A__GATHER_92_1, + A__TMP_92_0, A__GATHER_93_0, A__GATHER_93_1, A__GATHER_94_0, A__GATHER_94_1, A__GATHER_95_0, A__GATHER_95_1, - A__GATHER_97_0, - A__GATHER_97_1, - A__TMP_98_0, - A__GATHER_99_0, - A__GATHER_99_1, + A__GATHER_96_0, + A__GATHER_96_1, + A__GATHER_98_0, + A__GATHER_98_1, + A__TMP_99_0, A__GATHER_100_0, A__GATHER_100_1, - A__TMP_101_0, - A__TMP_101_1, - A__TMP_103_0, - A__TMP_104_0, - A__TMP_104_1, - A__TMP_105_0, - A__TMP_105_1, + A__GATHER_101_0, + A__GATHER_101_1, + A__TMP_102_0, + A__TMP_102_1, A__TMP_106_0, A__TMP_106_1, - A__TMP_106_2, - A__TMP_108_0, - A__TMP_108_1, + A__TMP_107_0, + A__TMP_107_1, + A__TMP_107_2, A__TMP_109_0, A__TMP_109_1, - A__TMP_110_0, - A__TMP_110_1, A__TMP_111_0, + A__TMP_111_1, A__TMP_112_0, A__TMP_112_1, A__TMP_113_0, A__TMP_113_1, A__TMP_114_0, A__TMP_115_0, + A__TMP_115_1, A__TMP_116_0, + A__TMP_116_1, A__TMP_117_0, A__TMP_118_0, A__TMP_119_0, A__TMP_120_0, A__TMP_121_0, + A__TMP_122_0, A__TMP_123_0, - A__TMP_123_1, A__TMP_124_0, - A__TMP_124_1, + A__TMP_125_0, + A__TMP_126_0, + A__TMP_129_0, + A__TMP_129_1, + A__TMP_130_0, + A__TMP_130_1, }; static Rule all_rules[] = { @@ -1193,7 +1208,7 @@ static Rule all_rules[] = { R_ASSIGNMENT, 0, // memo 0, // leftrec - {0, 10, 21, 30, 38, -1}, + {0, 10, 21, 34, 43, -1}, { // NAME ':' expression ['=' annotated_rhs] OP_NAME, @@ -1211,16 +1226,20 @@ static Rule all_rules[] = { OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_1, - // ((star_targets '='))+ (yield_expr | star_expressions) TYPE_COMMENT? + // ((star_targets '='))+ (yield_expr | star_expressions) !'=' TYPE_COMMENT? OP_RULE, R__LOOP1_17, OP_RULE, R__TMP_18, + OP_SAVE_MARK, + OP_TOKEN, EQUAL, + OP_NEG_LOOKAHEAD, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, OP_RETURN, A_ASSIGNMENT_2, - // single_target augassign (yield_expr | star_expressions) + // single_target augassign ~ (yield_expr | star_expressions) OP_RULE, R_SINGLE_TARGET, OP_RULE, R_AUGASSIGN, + OP_CUT, OP_RULE, R__TMP_19, OP_RETURN, A_ASSIGNMENT_3, @@ -1347,13 +1366,20 @@ static Rule all_rules[] = { R_DEL_STMT, 0, // memo 0, // leftrec - {0, -1}, + {0, 10, -1}, { - // 'del' del_targets + // 'del' del_targets &(';' | NEWLINE) OP_TOKEN, 503, OP_RULE, R_DEL_TARGETS, + OP_SAVE_MARK, + OP_RULE, R__TMP_23, + OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_STMT_0, + // invalid_del_stmt + OP_RULE, R_INVALID_DEL_STMT, + OP_RETURN, A_DEL_STMT_1, + }, }, {"import_stmt", @@ -1393,7 +1419,7 @@ static Rule all_rules[] = { { // 'from' (('.' | '...'))* dotted_name 'import' import_from_targets OP_TOKEN, 514, - OP_RULE, R__LOOP0_23, + OP_RULE, R__LOOP0_24, OP_RULE, R_DOTTED_NAME, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, @@ -1401,7 +1427,7 @@ static Rule all_rules[] = { // 'from' (('.' | '...'))+ 'import' import_from_targets OP_TOKEN, 514, - OP_RULE, R__LOOP1_24, + OP_RULE, R__LOOP1_25, OP_TOKEN, 513, OP_RULE, R_IMPORT_FROM_TARGETS, OP_RETURN, A_IMPORT_FROM_1, @@ -1446,7 +1472,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.import_from_as_name+ - OP_RULE, R__GATHER_25, + OP_RULE, R__GATHER_26, OP_RETURN, A_IMPORT_FROM_AS_NAMES_0, }, @@ -1459,7 +1485,7 @@ static Rule all_rules[] = { { // NAME ['as' NAME] OP_NAME, - OP_RULE, R__TMP_26, + OP_RULE, R__TMP_27, OP_OPTIONAL, OP_RETURN, A_IMPORT_FROM_AS_NAME_0, @@ -1472,7 +1498,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.dotted_as_name+ - OP_RULE, R__GATHER_27, + OP_RULE, R__GATHER_28, OP_RETURN, A_DOTTED_AS_NAMES_0, }, @@ -1485,7 +1511,7 @@ static Rule all_rules[] = { { // dotted_name ['as' NAME] OP_RULE, R_DOTTED_NAME, - OP_RULE, R__TMP_28, + OP_RULE, R__TMP_29, OP_OPTIONAL, OP_RETURN, A_DOTTED_AS_NAME_0, @@ -1594,12 +1620,13 @@ static Rule all_rules[] = { R_FOR_STMT, 0, // memo 0, // leftrec - {0, 20, -1}, + {0, 21, 44, -1}, { - // 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? + // 'for' star_targets 'in' ~ star_expressions ':' TYPE_COMMENT? block else_block? OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, + OP_CUT, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, @@ -1609,11 +1636,12 @@ static Rule all_rules[] = { OP_OPTIONAL, OP_RETURN, A_FOR_STMT_0, - // ASYNC 'for' star_targets 'in' star_expressions ':' TYPE_COMMENT? block else_block? + // ASYNC 'for' star_targets 'in' ~ star_expressions ':' TYPE_COMMENT? block else_block? OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, + OP_CUT, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, @@ -1623,6 +1651,10 @@ static Rule all_rules[] = { OP_OPTIONAL, OP_RETURN, A_FOR_STMT_1, + // invalid_for_target + OP_RULE, R_INVALID_FOR_TARGET, + OP_RETURN, A_FOR_STMT_2, + }, }, {"with_stmt", @@ -1634,7 +1666,7 @@ static Rule all_rules[] = { // 'with' '(' ','.with_item+ ','? ')' ':' block OP_TOKEN, 519, OP_TOKEN, LPAR, - OP_RULE, R__GATHER_29, + OP_RULE, R__GATHER_30, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, @@ -1644,7 +1676,7 @@ static Rule all_rules[] = { // 'with' ','.with_item+ ':' TYPE_COMMENT? block OP_TOKEN, 519, - OP_RULE, R__GATHER_30, + OP_RULE, R__GATHER_31, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, @@ -1655,7 +1687,7 @@ static Rule all_rules[] = { OP_TOKEN, ASYNC, OP_TOKEN, 519, OP_TOKEN, LPAR, - OP_RULE, R__GATHER_31, + OP_RULE, R__GATHER_32, OP_TOKEN, COMMA, OP_OPTIONAL, OP_TOKEN, RPAR, @@ -1666,7 +1698,7 @@ static Rule all_rules[] = { // ASYNC 'with' ','.with_item+ ':' TYPE_COMMENT? block OP_TOKEN, ASYNC, OP_TOKEN, 519, - OP_RULE, R__GATHER_32, + OP_RULE, R__GATHER_33, OP_TOKEN, COLON, OP_TOKEN, TYPE_COMMENT, OP_OPTIONAL, @@ -1679,14 +1711,25 @@ static Rule all_rules[] = { R_WITH_ITEM, 0, // memo 0, // leftrec - {0, -1}, + {0, 12, 16, -1}, { - // expression ['as' target] + // expression 'as' target &(',' | ')' | ':') OP_RULE, R_EXPRESSION, - OP_RULE, R__TMP_33, - OP_OPTIONAL, + OP_TOKEN, 520, + OP_RULE, R_TARGET, + OP_SAVE_MARK, + OP_RULE, R__TMP_34, + OP_POS_LOOKAHEAD, OP_RETURN, A_WITH_ITEM_0, + // invalid_with_item + OP_RULE, R_INVALID_WITH_ITEM, + OP_RETURN, A_WITH_ITEM_1, + + // expression + OP_RULE, R_EXPRESSION, + OP_RETURN, A_WITH_ITEM_2, + }, }, {"try_stmt", @@ -1706,7 +1749,7 @@ static Rule all_rules[] = { OP_TOKEN, 511, OP_TOKEN, COLON, OP_RULE, R_BLOCK, - OP_RULE, R__LOOP1_34, + OP_RULE, R__LOOP1_35, OP_RULE, R_ELSE_BLOCK, OP_OPTIONAL, OP_RULE, R_FINALLY_BLOCK, @@ -1722,16 +1765,16 @@ static Rule all_rules[] = { {0, 13, -1}, { // 'except' expression ['as' NAME] ':' block - OP_TOKEN, 520, + OP_TOKEN, 521, OP_RULE, R_EXPRESSION, - OP_RULE, R__TMP_35, + OP_RULE, R__TMP_36, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_0, // 'except' ':' block - OP_TOKEN, 520, + OP_TOKEN, 521, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_EXCEPT_BLOCK_1, @@ -1745,7 +1788,7 @@ static Rule all_rules[] = { {0, -1}, { // 'finally' ':' block - OP_TOKEN, 521, + OP_TOKEN, 522, OP_TOKEN, COLON, OP_RULE, R_BLOCK, OP_RETURN, A_FINALLY_BLOCK_0, @@ -1775,7 +1818,7 @@ static Rule all_rules[] = { // 'raise' expression ['from' expression] OP_TOKEN, 501, OP_RULE, R_EXPRESSION, - OP_RULE, R__TMP_36, + OP_RULE, R__TMP_37, OP_OPTIONAL, OP_RETURN, A_RAISE_STMT_0, @@ -1809,13 +1852,13 @@ static Rule all_rules[] = { {0, 22, -1}, { // 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block - OP_TOKEN, 522, + OP_TOKEN, 523, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, - OP_RULE, R__TMP_37, + OP_RULE, R__TMP_38, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, @@ -1825,13 +1868,13 @@ static Rule all_rules[] = { // ASYNC 'def' NAME '(' params? ')' ['->' expression] ':' func_type_comment? block OP_TOKEN, ASYNC, - OP_TOKEN, 522, + OP_TOKEN, 523, OP_NAME, OP_TOKEN, LPAR, OP_RULE, R_PARAMS, OP_OPTIONAL, OP_TOKEN, RPAR, - OP_RULE, R__TMP_38, + OP_RULE, R__TMP_39, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_FUNC_TYPE_COMMENT, @@ -1851,7 +1894,7 @@ static Rule all_rules[] = { OP_TOKEN, NEWLINE, OP_TOKEN, TYPE_COMMENT, OP_SAVE_MARK, - OP_RULE, R__TMP_39, + OP_RULE, R__TMP_40, OP_POS_LOOKAHEAD, OP_RETURN, A_FUNC_TYPE_COMMENT_0, @@ -1889,28 +1932,28 @@ static Rule all_rules[] = { { // slash_no_default param_no_default* param_with_default* star_etc? OP_RULE, R_SLASH_NO_DEFAULT, - OP_RULE, R__LOOP0_40, OP_RULE, R__LOOP0_41, + OP_RULE, R__LOOP0_42, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_0, // slash_with_default param_with_default* star_etc? OP_RULE, R_SLASH_WITH_DEFAULT, - OP_RULE, R__LOOP0_42, + OP_RULE, R__LOOP0_43, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_1, // param_no_default+ param_with_default* star_etc? - OP_RULE, R__LOOP1_43, - OP_RULE, R__LOOP0_44, + OP_RULE, R__LOOP1_44, + OP_RULE, R__LOOP0_45, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_2, // param_with_default+ star_etc? - OP_RULE, R__LOOP1_45, + OP_RULE, R__LOOP1_46, OP_RULE, R_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_PARAMETERS_3, @@ -1928,13 +1971,13 @@ static Rule all_rules[] = { {0, 8, -1}, { // param_no_default+ '/' ',' - OP_RULE, R__LOOP1_46, + OP_RULE, R__LOOP1_47, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_NO_DEFAULT_0, // param_no_default+ '/' &')' - OP_RULE, R__LOOP1_47, + OP_RULE, R__LOOP1_48, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, @@ -1950,15 +1993,15 @@ static Rule all_rules[] = { {0, 10, -1}, { // param_no_default* param_with_default+ '/' ',' - OP_RULE, R__LOOP0_48, - OP_RULE, R__LOOP1_49, + OP_RULE, R__LOOP0_49, + OP_RULE, R__LOOP1_50, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_SLASH_WITH_DEFAULT_0, // param_no_default* param_with_default+ '/' &')' - OP_RULE, R__LOOP0_50, - OP_RULE, R__LOOP1_51, + OP_RULE, R__LOOP0_51, + OP_RULE, R__LOOP1_52, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, RPAR, @@ -1976,7 +2019,7 @@ static Rule all_rules[] = { // '*' param_no_default param_maybe_default* kwds? OP_TOKEN, STAR, OP_RULE, R_PARAM_NO_DEFAULT, - OP_RULE, R__LOOP0_52, + OP_RULE, R__LOOP0_53, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_0, @@ -1984,7 +2027,7 @@ static Rule all_rules[] = { // '*' ',' param_maybe_default+ kwds? OP_TOKEN, STAR, OP_TOKEN, COMMA, - OP_RULE, R__LOOP1_53, + OP_RULE, R__LOOP1_54, OP_RULE, R_KWDS, OP_OPTIONAL, OP_RETURN, A_STAR_ETC_1, @@ -2137,7 +2180,7 @@ static Rule all_rules[] = { {0, -1}, { // (('@' named_expression NEWLINE))+ - OP_RULE, R__LOOP1_54, + OP_RULE, R__LOOP1_55, OP_RETURN, A_DECORATORS_0, }, @@ -2166,9 +2209,9 @@ static Rule all_rules[] = { {0, -1}, { // 'class' NAME ['(' arguments? ')'] ':' block - OP_TOKEN, 523, + OP_TOKEN, 524, OP_NAME, - OP_RULE, R__TMP_55, + OP_RULE, R__TMP_56, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_BLOCK, @@ -2206,7 +2249,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.star_expression+ ','? - OP_RULE, R__GATHER_56, + OP_RULE, R__GATHER_57, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_LIST_0, @@ -2221,7 +2264,7 @@ static Rule all_rules[] = { { // star_expression ((',' star_expression))+ ','? OP_RULE, R_STAR_EXPRESSION, - OP_RULE, R__LOOP1_57, + OP_RULE, R__LOOP1_58, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_EXPRESSIONS_0, @@ -2261,7 +2304,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.star_named_expression+ ','? - OP_RULE, R__GATHER_58, + OP_RULE, R__GATHER_59, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_NAMED_EXPRESSIONS_0, @@ -2289,11 +2332,12 @@ static Rule all_rules[] = { R_NAMED_EXPRESSION, 0, // memo 0, // leftrec - {0, 7, 15, -1}, + {0, 8, 16, -1}, { - // NAME ':=' expression + // NAME ':=' ~ expression OP_NAME, OP_TOKEN, COLONEQUAL, + OP_CUT, OP_RULE, R_EXPRESSION, OP_RETURN, A_NAMED_EXPRESSION_0, @@ -2334,7 +2378,7 @@ static Rule all_rules[] = { { // expression ((',' expression))+ ','? OP_RULE, R_EXPRESSION, - OP_RULE, R__LOOP1_59, + OP_RULE, R__LOOP1_60, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_EXPRESSIONS_0, @@ -2380,9 +2424,9 @@ static Rule all_rules[] = { 0, // leftrec {0, -1}, { - // 'lambda' lambda_parameters? ':' expression - OP_TOKEN, 524, - OP_RULE, R_LAMBDA_PARAMETERS, + // 'lambda' lambda_params? ':' expression + OP_TOKEN, 525, + OP_RULE, R_LAMBDA_PARAMS, OP_OPTIONAL, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, @@ -2390,6 +2434,22 @@ static Rule all_rules[] = { }, }, + {"lambda_params", + R_LAMBDA_PARAMS, + 0, // memo + 0, // leftrec + {0, 4, -1}, + { + // invalid_lambda_parameters + OP_RULE, R_INVALID_LAMBDA_PARAMETERS, + OP_RETURN, A_LAMBDA_PARAMS_0, + + // lambda_parameters + OP_RULE, R_LAMBDA_PARAMETERS, + OP_RETURN, A_LAMBDA_PARAMS_1, + + }, + }, {"lambda_parameters", R_LAMBDA_PARAMETERS, 0, // memo @@ -2398,28 +2458,28 @@ static Rule all_rules[] = { { // lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc? OP_RULE, R_LAMBDA_SLASH_NO_DEFAULT, - OP_RULE, R__LOOP0_60, OP_RULE, R__LOOP0_61, + OP_RULE, R__LOOP0_62, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_0, // lambda_slash_with_default lambda_param_with_default* lambda_star_etc? OP_RULE, R_LAMBDA_SLASH_WITH_DEFAULT, - OP_RULE, R__LOOP0_62, + OP_RULE, R__LOOP0_63, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_1, // lambda_param_no_default+ lambda_param_with_default* lambda_star_etc? - OP_RULE, R__LOOP1_63, - OP_RULE, R__LOOP0_64, + OP_RULE, R__LOOP1_64, + OP_RULE, R__LOOP0_65, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_2, // lambda_param_with_default+ lambda_star_etc? - OP_RULE, R__LOOP1_65, + OP_RULE, R__LOOP1_66, OP_RULE, R_LAMBDA_STAR_ETC, OP_OPTIONAL, OP_RETURN, A_LAMBDA_PARAMETERS_3, @@ -2437,13 +2497,13 @@ static Rule all_rules[] = { {0, 8, -1}, { // lambda_param_no_default+ '/' ',' - OP_RULE, R__LOOP1_66, + OP_RULE, R__LOOP1_67, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_NO_DEFAULT_0, // lambda_param_no_default+ '/' &':' - OP_RULE, R__LOOP1_67, + OP_RULE, R__LOOP1_68, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, @@ -2459,15 +2519,15 @@ static Rule all_rules[] = { {0, 10, -1}, { // lambda_param_no_default* lambda_param_with_default+ '/' ',' - OP_RULE, R__LOOP0_68, - OP_RULE, R__LOOP1_69, + OP_RULE, R__LOOP0_69, + OP_RULE, R__LOOP1_70, OP_TOKEN, SLASH, OP_TOKEN, COMMA, OP_RETURN, A_LAMBDA_SLASH_WITH_DEFAULT_0, // lambda_param_no_default* lambda_param_with_default+ '/' &':' - OP_RULE, R__LOOP0_70, - OP_RULE, R__LOOP1_71, + OP_RULE, R__LOOP0_71, + OP_RULE, R__LOOP1_72, OP_TOKEN, SLASH, OP_SAVE_MARK, OP_TOKEN, COLON, @@ -2485,7 +2545,7 @@ static Rule all_rules[] = { // '*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds? OP_TOKEN, STAR, OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, - OP_RULE, R__LOOP0_72, + OP_RULE, R__LOOP0_73, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_0, @@ -2493,7 +2553,7 @@ static Rule all_rules[] = { // '*' ',' lambda_param_maybe_default+ lambda_kwds? OP_TOKEN, STAR, OP_TOKEN, COMMA, - OP_RULE, R__LOOP1_73, + OP_RULE, R__LOOP1_74, OP_RULE, R_LAMBDA_KWDS, OP_OPTIONAL, OP_RETURN, A_LAMBDA_STAR_ETC_1, @@ -2607,7 +2667,7 @@ static Rule all_rules[] = { { // conjunction (('or' conjunction))+ OP_RULE, R_CONJUNCTION, - OP_RULE, R__LOOP1_74, + OP_RULE, R__LOOP1_75, OP_RETURN, A_DISJUNCTION_0, // conjunction @@ -2624,7 +2684,7 @@ static Rule all_rules[] = { { // inversion (('and' inversion))+ OP_RULE, R_INVERSION, - OP_RULE, R__LOOP1_75, + OP_RULE, R__LOOP1_76, OP_RETURN, A_CONJUNCTION_0, // inversion @@ -2640,7 +2700,7 @@ static Rule all_rules[] = { {0, 6, -1}, { // 'not' inversion - OP_TOKEN, 525, + OP_TOKEN, 526, OP_RULE, R_INVERSION, OP_RETURN, A_INVERSION_0, @@ -2658,7 +2718,7 @@ static Rule all_rules[] = { { // bitwise_or compare_op_bitwise_or_pair+ OP_RULE, R_BITWISE_OR, - OP_RULE, R__LOOP1_76, + OP_RULE, R__LOOP1_77, OP_RETURN, A_COMPARISON_0, // bitwise_or @@ -2735,7 +2795,7 @@ static Rule all_rules[] = { {0, -1}, { // ('!=') bitwise_or - OP_RULE, R__TMP_77, + OP_RULE, R__TMP_78, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTEQ_BITWISE_OR_0, @@ -2800,7 +2860,7 @@ static Rule all_rules[] = { {0, -1}, { // 'not' 'in' bitwise_or - OP_TOKEN, 525, + OP_TOKEN, 526, OP_TOKEN, 518, OP_RULE, R_BITWISE_OR, OP_RETURN, A_NOTIN_BITWISE_OR_0, @@ -2827,8 +2887,8 @@ static Rule all_rules[] = { {0, -1}, { // 'is' 'not' bitwise_or + OP_TOKEN, 527, OP_TOKEN, 526, - OP_TOKEN, 525, OP_RULE, R_BITWISE_OR, OP_RETURN, A_ISNOT_BITWISE_OR_0, @@ -2841,7 +2901,7 @@ static Rule all_rules[] = { {0, -1}, { // 'is' bitwise_or - OP_TOKEN, 526, + OP_TOKEN, 527, OP_RULE, R_BITWISE_OR, OP_RETURN, A_IS_BITWISE_OR_0, @@ -3105,7 +3165,7 @@ static Rule all_rules[] = { OP_RETURN, A_SLICES_0, // ','.slice+ ','? - OP_RULE, R__GATHER_78, + OP_RULE, R__GATHER_79, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_SLICES_1, @@ -3124,7 +3184,7 @@ static Rule all_rules[] = { OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, - OP_RULE, R__TMP_79, + OP_RULE, R__TMP_80, OP_OPTIONAL, OP_RETURN, A_SLICE_0, @@ -3138,63 +3198,59 @@ static Rule all_rules[] = { R_ATOM, 0, // memo 0, // leftrec - {0, 3, 7, 11, 15, 19, 26, 29, 37, 45, 53, -1}, + {0, 3, 7, 11, 15, 22, 25, 33, 41, 49, -1}, { // NAME OP_NAME, OP_RETURN, A_ATOM_0, // 'True' - OP_TOKEN, 527, + OP_TOKEN, 528, OP_RETURN, A_ATOM_1, // 'False' - OP_TOKEN, 528, + OP_TOKEN, 529, OP_RETURN, A_ATOM_2, // 'None' - OP_TOKEN, 529, - OP_RETURN, A_ATOM_3, - - // '__new_parser__' OP_TOKEN, 530, - OP_RETURN, A_ATOM_4, + OP_RETURN, A_ATOM_3, // &STRING strings OP_SAVE_MARK, OP_STRING, OP_POS_LOOKAHEAD, OP_RULE, R_STRINGS, - OP_RETURN, A_ATOM_5, + OP_RETURN, A_ATOM_4, // NUMBER OP_NUMBER, - OP_RETURN, A_ATOM_6, + OP_RETURN, A_ATOM_5, // &'(' (tuple | group | genexp) OP_SAVE_MARK, OP_TOKEN, LPAR, OP_POS_LOOKAHEAD, - OP_RULE, R__TMP_80, - OP_RETURN, A_ATOM_7, + OP_RULE, R__TMP_81, + OP_RETURN, A_ATOM_6, // &'[' (list | listcomp) OP_SAVE_MARK, OP_TOKEN, LSQB, OP_POS_LOOKAHEAD, - OP_RULE, R__TMP_81, - OP_RETURN, A_ATOM_8, + OP_RULE, R__TMP_82, + OP_RETURN, A_ATOM_7, // &'{' (dict | set | dictcomp | setcomp) OP_SAVE_MARK, OP_TOKEN, LBRACE, OP_POS_LOOKAHEAD, - OP_RULE, R__TMP_82, - OP_RETURN, A_ATOM_9, + OP_RULE, R__TMP_83, + OP_RETURN, A_ATOM_8, // '...' OP_TOKEN, ELLIPSIS, - OP_RETURN, A_ATOM_10, + OP_RETURN, A_ATOM_9, }, }, @@ -3205,7 +3261,7 @@ static Rule all_rules[] = { {0, -1}, { // STRING+ - OP_RULE, R__LOOP1_83, + OP_RULE, R__LOOP1_84, OP_RETURN, A_STRINGS_0, }, @@ -3229,11 +3285,12 @@ static Rule all_rules[] = { R_LISTCOMP, 0, // memo 0, // leftrec - {0, 10, -1}, + {0, 11, -1}, { - // '[' named_expression for_if_clauses ']' + // '[' named_expression ~ for_if_clauses ']' OP_TOKEN, LSQB, OP_RULE, R_NAMED_EXPRESSION, + OP_CUT, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RSQB, OP_RETURN, A_LISTCOMP_0, @@ -3252,7 +3309,7 @@ static Rule all_rules[] = { { // '(' [star_named_expression ',' star_named_expressions?] ')' OP_TOKEN, LPAR, - OP_RULE, R__TMP_84, + OP_RULE, R__TMP_85, OP_OPTIONAL, OP_TOKEN, RPAR, OP_RETURN, A_TUPLE_0, @@ -3263,25 +3320,30 @@ static Rule all_rules[] = { R_GROUP, 0, // memo 0, // leftrec - {0, -1}, + {0, 8, -1}, { // '(' (yield_expr | named_expression) ')' OP_TOKEN, LPAR, - OP_RULE, R__TMP_85, + OP_RULE, R__TMP_86, OP_TOKEN, RPAR, OP_RETURN, A_GROUP_0, + // invalid_group + OP_RULE, R_INVALID_GROUP, + OP_RETURN, A_GROUP_1, + }, }, {"genexp", R_GENEXP, 0, // memo 0, // leftrec - {0, 10, -1}, + {0, 11, -1}, { - // '(' expression for_if_clauses ')' + // '(' expression ~ for_if_clauses ')' OP_TOKEN, LPAR, OP_RULE, R_EXPRESSION, + OP_CUT, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RPAR, OP_RETURN, A_GENEXP_0, @@ -3310,11 +3372,12 @@ static Rule all_rules[] = { R_SETCOMP, 0, // memo 0, // leftrec - {0, 10, -1}, + {0, 11, -1}, { - // '{' expression for_if_clauses '}' + // '{' expression ~ for_if_clauses '}' OP_TOKEN, LBRACE, OP_RULE, R_EXPRESSION, + OP_CUT, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, RBRACE, OP_RETURN, A_SETCOMP_0, @@ -3366,7 +3429,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.double_starred_kvpair+ ','? - OP_RULE, R__GATHER_86, + OP_RULE, R__GATHER_87, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DOUBLE_STARRED_KVPAIRS_0, @@ -3411,7 +3474,7 @@ static Rule all_rules[] = { {0, -1}, { // for_if_clause+ - OP_RULE, R__LOOP1_87, + OP_RULE, R__LOOP1_88, OP_RETURN, A_FOR_IF_CLAUSES_0, }, @@ -3420,25 +3483,31 @@ static Rule all_rules[] = { R_FOR_IF_CLAUSE, 0, // memo 0, // leftrec - {0, 14, -1}, + {0, 15, 28, -1}, { - // ASYNC 'for' star_targets 'in' disjunction (('if' disjunction))* + // ASYNC 'for' star_targets 'in' ~ disjunction (('if' disjunction))* OP_TOKEN, ASYNC, OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, + OP_CUT, OP_RULE, R_DISJUNCTION, - OP_RULE, R__LOOP0_88, + OP_RULE, R__LOOP0_89, OP_RETURN, A_FOR_IF_CLAUSE_0, - // 'for' star_targets 'in' disjunction (('if' disjunction))* + // 'for' star_targets 'in' ~ disjunction (('if' disjunction))* OP_TOKEN, 517, OP_RULE, R_STAR_TARGETS, OP_TOKEN, 518, + OP_CUT, OP_RULE, R_DISJUNCTION, - OP_RULE, R__LOOP0_89, + OP_RULE, R__LOOP0_90, OP_RETURN, A_FOR_IF_CLAUSE_1, + // invalid_for_target + OP_RULE, R_INVALID_FOR_TARGET, + OP_RETURN, A_FOR_IF_CLAUSE_2, + }, }, {"yield_expr", @@ -3490,7 +3559,7 @@ static Rule all_rules[] = { { // starred_expression [',' args] OP_RULE, R_STARRED_EXPRESSION, - OP_RULE, R__TMP_90, + OP_RULE, R__TMP_91, OP_OPTIONAL, OP_RETURN, A_ARGS_0, @@ -3500,7 +3569,7 @@ static Rule all_rules[] = { // named_expression [',' args] OP_RULE, R_NAMED_EXPRESSION, - OP_RULE, R__TMP_91, + OP_RULE, R__TMP_92, OP_OPTIONAL, OP_RETURN, A_ARGS_2, @@ -3513,17 +3582,17 @@ static Rule all_rules[] = { {0, 8, 12, -1}, { // ','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+ - OP_RULE, R__GATHER_92, - OP_TOKEN, COMMA, OP_RULE, R__GATHER_93, + OP_TOKEN, COMMA, + OP_RULE, R__GATHER_94, OP_RETURN, A_KWARGS_0, // ','.kwarg_or_starred+ - OP_RULE, R__GATHER_94, + OP_RULE, R__GATHER_95, OP_RETURN, A_KWARGS_1, // ','.kwarg_or_double_starred+ - OP_RULE, R__GATHER_95, + OP_RULE, R__GATHER_96, OP_RETURN, A_KWARGS_2, }, @@ -3601,7 +3670,7 @@ static Rule all_rules[] = { // star_target ((',' star_target))* ','? OP_RULE, R_STAR_TARGET, - OP_RULE, R__LOOP0_96, + OP_RULE, R__LOOP0_97, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_1, @@ -3615,7 +3684,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.star_target+ ','? - OP_RULE, R__GATHER_97, + OP_RULE, R__GATHER_98, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_STAR_TARGETS_SEQ_0, @@ -3630,7 +3699,7 @@ static Rule all_rules[] = { { // '*' (!'*' star_target) OP_TOKEN, STAR, - OP_RULE, R__TMP_98, + OP_RULE, R__TMP_99, OP_RETURN, A_STAR_TARGET_0, // t_primary '.' NAME !t_lookahead @@ -3746,7 +3815,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.del_target+ ','? - OP_RULE, R__GATHER_99, + OP_RULE, R__GATHER_100, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_DEL_TARGETS_0, @@ -3759,23 +3828,23 @@ static Rule all_rules[] = { 0, // leftrec {0, 11, 25, -1}, { - // t_primary '.' NAME &del_target_end + // t_primary '.' NAME !t_lookahead OP_RULE, R_T_PRIMARY, OP_TOKEN, DOT, OP_NAME, OP_SAVE_MARK, - OP_RULE, R_DEL_TARGET_END, - OP_POS_LOOKAHEAD, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_0, - // t_primary '[' slices ']' &del_target_end + // t_primary '[' slices ']' !t_lookahead OP_RULE, R_T_PRIMARY, OP_TOKEN, LSQB, OP_RULE, R_SLICES, OP_TOKEN, RSQB, OP_SAVE_MARK, - OP_RULE, R_DEL_TARGET_END, - OP_POS_LOOKAHEAD, + OP_RULE, R_T_LOOKAHEAD, + OP_NEG_LOOKAHEAD, OP_RETURN, A_DEL_TARGET_1, // del_t_atom @@ -3788,13 +3857,10 @@ static Rule all_rules[] = { R_DEL_T_ATOM, 0, // memo 0, // leftrec - {0, 7, 15, 24, 33, -1}, + {0, 3, 11, 20, -1}, { - // NAME &del_target_end + // NAME OP_NAME, - OP_SAVE_MARK, - OP_RULE, R_DEL_TARGET_END, - OP_POS_LOOKAHEAD, OP_RETURN, A_DEL_T_ATOM_0, // '(' del_target ')' @@ -3817,38 +3883,6 @@ static Rule all_rules[] = { OP_TOKEN, RSQB, OP_RETURN, A_DEL_T_ATOM_3, - // invalid_del_target - OP_RULE, R_INVALID_DEL_TARGET, - OP_RETURN, A_DEL_T_ATOM_4, - - }, - }, - {"del_target_end", - R_DEL_TARGET_END, - 0, // memo - 0, // leftrec - {0, 4, 8, 12, 16, -1}, - { - // ')' - OP_TOKEN, RPAR, - OP_RETURN, A_DEL_TARGET_END_0, - - // ']' - OP_TOKEN, RSQB, - OP_RETURN, A_DEL_TARGET_END_1, - - // ',' - OP_TOKEN, COMMA, - OP_RETURN, A_DEL_TARGET_END_2, - - // ';' - OP_TOKEN, SEMI, - OP_RETURN, A_DEL_TARGET_END_3, - - // NEWLINE - OP_TOKEN, NEWLINE, - OP_RETURN, A_DEL_TARGET_END_4, - }, }, {"targets", @@ -3858,7 +3892,7 @@ static Rule all_rules[] = { {0, -1}, { // ','.target+ ','? - OP_RULE, R__GATHER_100, + OP_RULE, R__GATHER_101, OP_TOKEN, COMMA, OP_OPTIONAL, OP_RETURN, A_TARGETS_0, @@ -4017,7 +4051,7 @@ static Rule all_rules[] = { OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_TOKEN, COMMA, - OP_RULE, R__TMP_101, + OP_RULE, R__TMP_102, OP_OPTIONAL, OP_RETURN, A_INCORRECT_ARGUMENTS_1, @@ -4072,47 +4106,83 @@ static Rule all_rules[] = { R_INVALID_ASSIGNMENT, 0, // memo 0, // leftrec - {0, 6, 12, 22, 33, 41, -1}, + {0, 8, 20, 28, 36, 44, -1}, { - // list ':' - OP_RULE, R_LIST, + // invalid_ann_assign_target ':' expression + OP_RULE, R_INVALID_ANN_ASSIGN_TARGET, OP_TOKEN, COLON, + OP_RULE, R_EXPRESSION, OP_RETURN, A_INVALID_ASSIGNMENT_0, - // tuple ':' - OP_RULE, R_TUPLE, - OP_TOKEN, COLON, - OP_RETURN, A_INVALID_ASSIGNMENT_1, - - // star_named_expression ',' star_named_expressions* ':' + // star_named_expression ',' star_named_expressions* ':' expression OP_RULE, R_STAR_NAMED_EXPRESSION, OP_TOKEN, COMMA, - OP_RULE, R__LOOP0_102, + OP_RULE, R__LOOP0_103, OP_TOKEN, COLON, - OP_RETURN, A_INVALID_ASSIGNMENT_2, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_INVALID_ASSIGNMENT_1, - // expression ':' expression ['=' annotated_rhs] + // expression ':' expression OP_RULE, R_EXPRESSION, OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, - OP_RULE, R__TMP_103, - OP_OPTIONAL, - OP_RETURN, A_INVALID_ASSIGNMENT_3, + OP_RETURN, A_INVALID_ASSIGNMENT_2, - // star_expressions '=' (yield_expr | star_expressions) + // ((star_targets '='))* star_expressions '=' + OP_RULE, R__LOOP0_104, OP_RULE, R_STAR_EXPRESSIONS, OP_TOKEN, EQUAL, - OP_RULE, R__TMP_104, + OP_RETURN, A_INVALID_ASSIGNMENT_3, + + // ((star_targets '='))* yield_expr '=' + OP_RULE, R__LOOP0_105, + OP_RULE, R_YIELD_EXPR, + OP_TOKEN, EQUAL, OP_RETURN, A_INVALID_ASSIGNMENT_4, // star_expressions augassign (yield_expr | star_expressions) OP_RULE, R_STAR_EXPRESSIONS, OP_RULE, R_AUGASSIGN, - OP_RULE, R__TMP_105, + OP_RULE, R__TMP_106, OP_RETURN, A_INVALID_ASSIGNMENT_5, }, }, + {"invalid_ann_assign_target", + R_INVALID_ANN_ASSIGN_TARGET, + 0, // memo + 0, // leftrec + {0, 4, 8, -1}, + { + // list + OP_RULE, R_LIST, + OP_RETURN, A_INVALID_ANN_ASSIGN_TARGET_0, + + // tuple + OP_RULE, R_TUPLE, + OP_RETURN, A_INVALID_ANN_ASSIGN_TARGET_1, + + // '(' invalid_ann_assign_target ')' + OP_TOKEN, LPAR, + OP_RULE, R_INVALID_ANN_ASSIGN_TARGET, + OP_TOKEN, RPAR, + OP_RETURN, A_INVALID_ANN_ASSIGN_TARGET_2, + + }, + }, + {"invalid_del_stmt", + R_INVALID_DEL_STMT, + 0, // memo + 0, // leftrec + {0, -1}, + { + // 'del' star_expressions + OP_TOKEN, 503, + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A_INVALID_DEL_STMT_0, + + }, + }, {"invalid_block", R_INVALID_BLOCK, 0, // memo @@ -4135,7 +4205,7 @@ static Rule all_rules[] = { {0, -1}, { // ('[' | '(' | '{') starred_expression for_if_clauses - OP_RULE, R__TMP_106, + OP_RULE, R__TMP_107, OP_RULE, R_STARRED_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, OP_RETURN, A_INVALID_COMPREHENSION_0, @@ -4165,13 +4235,27 @@ static Rule all_rules[] = { {0, -1}, { // param_no_default* (slash_with_default | param_with_default+) param_no_default - OP_RULE, R__LOOP0_107, - OP_RULE, R__TMP_108, + OP_RULE, R__LOOP0_108, + OP_RULE, R__TMP_109, OP_RULE, R_PARAM_NO_DEFAULT, OP_RETURN, A_INVALID_PARAMETERS_0, }, }, + {"invalid_lambda_parameters", + R_INVALID_LAMBDA_PARAMETERS, + 0, // memo + 0, // leftrec + {0, -1}, + { + // lambda_param_no_default* (lambda_slash_with_default | lambda_param_with_default+) lambda_param_no_default + OP_RULE, R__LOOP0_110, + OP_RULE, R__TMP_111, + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_RETURN, A_INVALID_LAMBDA_PARAMETERS_0, + + }, + }, {"invalid_star_etc", R_INVALID_STAR_ETC, 0, // memo @@ -4180,7 +4264,7 @@ static Rule all_rules[] = { { // '*' (')' | ',' (')' | '**')) OP_TOKEN, STAR, - OP_RULE, R__TMP_109, + OP_RULE, R__TMP_112, OP_RETURN, A_INVALID_STAR_ETC_0, // '*' ',' TYPE_COMMENT @@ -4199,7 +4283,7 @@ static Rule all_rules[] = { { // '*' (':' | ',' (':' | '**')) OP_TOKEN, STAR, - OP_RULE, R__TMP_110, + OP_RULE, R__TMP_113, OP_RETURN, A_INVALID_LAMBDA_STAR_ETC_0, }, @@ -4220,41 +4304,69 @@ static Rule all_rules[] = { }, }, - {"invalid_del_target", - R_INVALID_DEL_TARGET, + {"invalid_with_item", + R_INVALID_WITH_ITEM, 0, // memo 0, // leftrec {0, -1}, { - // star_expression &del_target_end - OP_RULE, R_STAR_EXPRESSION, - OP_SAVE_MARK, - OP_RULE, R_DEL_TARGET_END, - OP_POS_LOOKAHEAD, - OP_RETURN, A_INVALID_DEL_TARGET_0, + // expression 'as' expression + OP_RULE, R_EXPRESSION, + OP_TOKEN, 520, + OP_RULE, R_EXPRESSION, + OP_RETURN, A_INVALID_WITH_ITEM_0, }, }, - {"invalid_import_from_targets", - R_INVALID_IMPORT_FROM_TARGETS, + {"invalid_for_target", + R_INVALID_FOR_TARGET, 0, // memo 0, // leftrec {0, -1}, { - // import_from_as_names ',' - OP_RULE, R_IMPORT_FROM_AS_NAMES, - OP_TOKEN, COMMA, - OP_RETURN, A_INVALID_IMPORT_FROM_TARGETS_0, + // ASYNC? 'for' star_expressions + OP_TOKEN, ASYNC, + OP_OPTIONAL, + OP_TOKEN, 517, + OP_RULE, R_STAR_EXPRESSIONS, + OP_RETURN, A_INVALID_FOR_TARGET_0, }, }, - {"root", - R_ROOT, - 0, - 0, - {0, 3, -1}, - { - // + {"invalid_group", + R_INVALID_GROUP, + 0, // memo + 0, // leftrec + {0, -1}, + { + // '(' starred_expression ')' + OP_TOKEN, LPAR, + OP_RULE, R_STARRED_EXPRESSION, + OP_TOKEN, RPAR, + OP_RETURN, A_INVALID_GROUP_0, + + }, + }, + {"invalid_import_from_targets", + R_INVALID_IMPORT_FROM_TARGETS, + 0, // memo + 0, // leftrec + {0, -1}, + { + // import_from_as_names ',' + OP_RULE, R_IMPORT_FROM_AS_NAMES, + OP_TOKEN, COMMA, + OP_RETURN, A_INVALID_IMPORT_FROM_TARGETS_0, + + }, + }, + {"root", + R_ROOT, + 0, + 0, + {0, 3, -1}, + { + // OP_RULE, R_FILE, OP_SUCCESS, @@ -4416,7 +4528,7 @@ static Rule all_rules[] = { {0, 4, 8, -1}, { // 'def' - OP_TOKEN, 522, + OP_TOKEN, 523, OP_RETURN, A__TMP_10_0, // '@' @@ -4436,7 +4548,7 @@ static Rule all_rules[] = { {0, 4, -1}, { // 'class' - OP_TOKEN, 523, + OP_TOKEN, 524, OP_RETURN, A__TMP_11_0, // '@' @@ -4528,7 +4640,7 @@ static Rule all_rules[] = { {0, 3, -1}, { // (star_targets '=') - OP_RULE, R__TMP_111, + OP_RULE, R__TMP_114, OP_LOOP_ITERATE, // @@ -4615,14 +4727,30 @@ static Rule all_rules[] = { }, }, - {"_loop0_23", - R__LOOP0_23, + {"_tmp_23", + R__TMP_23, + 0, // memo + 0, // leftrec + {0, 4, -1}, + { + // ';' + OP_TOKEN, SEMI, + OP_RETURN, A__TMP_23_0, + + // NEWLINE + OP_TOKEN, NEWLINE, + OP_RETURN, A__TMP_23_1, + + }, + }, + {"_loop0_24", + R__LOOP0_24, 0, // memo 0, // leftrec {0, 3, -1}, { // ('.' | '...') - OP_RULE, R__TMP_112, + OP_RULE, R__TMP_115, OP_LOOP_ITERATE, // @@ -4630,14 +4758,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_24", - R__LOOP1_24, + {"_loop1_25", + R__LOOP1_25, 0, // memo 0, // leftrec {0, 3, -1}, { // ('.' | '...') - OP_RULE, R__TMP_113, + OP_RULE, R__TMP_116, OP_LOOP_ITERATE, // @@ -4645,8 +4773,8 @@ static Rule all_rules[] = { }, }, - {"_gather_25", - R__GATHER_25, + {"_gather_26", + R__GATHER_26, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4662,21 +4790,21 @@ static Rule all_rules[] = { }, }, - {"_tmp_26", - R__TMP_26, + {"_tmp_27", + R__TMP_27, 0, // memo 0, // leftrec {0, -1}, { // 'as' NAME - OP_TOKEN, 531, + OP_TOKEN, 520, OP_NAME, - OP_RETURN, A__TMP_26_0, + OP_RETURN, A__TMP_27_0, }, }, - {"_gather_27", - R__GATHER_27, + {"_gather_28", + R__GATHER_28, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4692,21 +4820,21 @@ static Rule all_rules[] = { }, }, - {"_tmp_28", - R__TMP_28, + {"_tmp_29", + R__TMP_29, 0, // memo 0, // leftrec {0, -1}, { // 'as' NAME - OP_TOKEN, 531, + OP_TOKEN, 520, OP_NAME, - OP_RETURN, A__TMP_28_0, + OP_RETURN, A__TMP_29_0, }, }, - {"_gather_29", - R__GATHER_29, + {"_gather_30", + R__GATHER_30, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4722,8 +4850,8 @@ static Rule all_rules[] = { }, }, - {"_gather_30", - R__GATHER_30, + {"_gather_31", + R__GATHER_31, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4739,8 +4867,8 @@ static Rule all_rules[] = { }, }, - {"_gather_31", - R__GATHER_31, + {"_gather_32", + R__GATHER_32, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4756,8 +4884,8 @@ static Rule all_rules[] = { }, }, - {"_gather_32", - R__GATHER_32, + {"_gather_33", + R__GATHER_33, 0, // memo 0, // leftrec {0, 5, -1}, @@ -4773,21 +4901,28 @@ static Rule all_rules[] = { }, }, - {"_tmp_33", - R__TMP_33, + {"_tmp_34", + R__TMP_34, 0, // memo 0, // leftrec - {0, -1}, + {0, 4, 8, -1}, { - // 'as' target - OP_TOKEN, 531, - OP_RULE, R_TARGET, - OP_RETURN, A__TMP_33_0, + // ',' + OP_TOKEN, COMMA, + OP_RETURN, A__TMP_34_0, + + // ')' + OP_TOKEN, RPAR, + OP_RETURN, A__TMP_34_1, + + // ':' + OP_TOKEN, COLON, + OP_RETURN, A__TMP_34_2, }, }, - {"_loop1_34", - R__LOOP1_34, + {"_loop1_35", + R__LOOP1_35, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4801,21 +4936,21 @@ static Rule all_rules[] = { }, }, - {"_tmp_35", - R__TMP_35, + {"_tmp_36", + R__TMP_36, 0, // memo 0, // leftrec {0, -1}, { // 'as' NAME - OP_TOKEN, 531, + OP_TOKEN, 520, OP_NAME, - OP_RETURN, A__TMP_35_0, + OP_RETURN, A__TMP_36_0, }, }, - {"_tmp_36", - R__TMP_36, + {"_tmp_37", + R__TMP_37, 0, // memo 0, // leftrec {0, -1}, @@ -4823,12 +4958,12 @@ static Rule all_rules[] = { // 'from' expression OP_TOKEN, 514, OP_RULE, R_EXPRESSION, - OP_RETURN, A__TMP_36_0, + OP_RETURN, A__TMP_37_0, }, }, - {"_tmp_37", - R__TMP_37, + {"_tmp_38", + R__TMP_38, 0, // memo 0, // leftrec {0, -1}, @@ -4836,12 +4971,12 @@ static Rule all_rules[] = { // '->' expression OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, - OP_RETURN, A__TMP_37_0, + OP_RETURN, A__TMP_38_0, }, }, - {"_tmp_38", - R__TMP_38, + {"_tmp_39", + R__TMP_39, 0, // memo 0, // leftrec {0, -1}, @@ -4849,12 +4984,12 @@ static Rule all_rules[] = { // '->' expression OP_TOKEN, RARROW, OP_RULE, R_EXPRESSION, - OP_RETURN, A__TMP_38_0, + OP_RETURN, A__TMP_39_0, }, }, - {"_tmp_39", - R__TMP_39, + {"_tmp_40", + R__TMP_40, 0, // memo 0, // leftrec {0, -1}, @@ -4862,12 +4997,12 @@ static Rule all_rules[] = { // NEWLINE INDENT OP_TOKEN, NEWLINE, OP_TOKEN, INDENT, - OP_RETURN, A__TMP_39_0, + OP_RETURN, A__TMP_40_0, }, }, - {"_loop0_40", - R__LOOP0_40, + {"_loop0_41", + R__LOOP0_41, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4881,8 +5016,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_41", - R__LOOP0_41, + {"_loop0_42", + R__LOOP0_42, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4896,8 +5031,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_42", - R__LOOP0_42, + {"_loop0_43", + R__LOOP0_43, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4911,8 +5046,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_43", - R__LOOP1_43, + {"_loop1_44", + R__LOOP1_44, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4926,8 +5061,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_44", - R__LOOP0_44, + {"_loop0_45", + R__LOOP0_45, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4941,8 +5076,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_45", - R__LOOP1_45, + {"_loop1_46", + R__LOOP1_46, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4956,8 +5091,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_46", - R__LOOP1_46, + {"_loop1_47", + R__LOOP1_47, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4971,8 +5106,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_47", - R__LOOP1_47, + {"_loop1_48", + R__LOOP1_48, 0, // memo 0, // leftrec {0, 3, -1}, @@ -4986,8 +5121,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_48", - R__LOOP0_48, + {"_loop0_49", + R__LOOP0_49, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5001,8 +5136,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_49", - R__LOOP1_49, + {"_loop1_50", + R__LOOP1_50, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5016,8 +5151,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_50", - R__LOOP0_50, + {"_loop0_51", + R__LOOP0_51, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5031,8 +5166,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_51", - R__LOOP1_51, + {"_loop1_52", + R__LOOP1_52, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5046,8 +5181,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_52", - R__LOOP0_52, + {"_loop0_53", + R__LOOP0_53, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5061,8 +5196,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_53", - R__LOOP1_53, + {"_loop1_54", + R__LOOP1_54, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5076,14 +5211,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_54", - R__LOOP1_54, + {"_loop1_55", + R__LOOP1_55, 0, // memo 0, // leftrec {0, 3, -1}, { // ('@' named_expression NEWLINE) - OP_RULE, R__TMP_114, + OP_RULE, R__TMP_117, OP_LOOP_ITERATE, // @@ -5091,8 +5226,8 @@ static Rule all_rules[] = { }, }, - {"_tmp_55", - R__TMP_55, + {"_tmp_56", + R__TMP_56, 0, // memo 0, // leftrec {0, -1}, @@ -5102,12 +5237,12 @@ static Rule all_rules[] = { OP_RULE, R_ARGUMENTS, OP_OPTIONAL, OP_TOKEN, RPAR, - OP_RETURN, A__TMP_55_0, + OP_RETURN, A__TMP_56_0, }, }, - {"_gather_56", - R__GATHER_56, + {"_gather_57", + R__GATHER_57, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5123,14 +5258,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_57", - R__LOOP1_57, + {"_loop1_58", + R__LOOP1_58, 0, // memo 0, // leftrec {0, 3, -1}, { // (',' star_expression) - OP_RULE, R__TMP_115, + OP_RULE, R__TMP_118, OP_LOOP_ITERATE, // @@ -5138,8 +5273,8 @@ static Rule all_rules[] = { }, }, - {"_gather_58", - R__GATHER_58, + {"_gather_59", + R__GATHER_59, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5155,14 +5290,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_59", - R__LOOP1_59, + {"_loop1_60", + R__LOOP1_60, 0, // memo 0, // leftrec {0, 3, -1}, { // (',' expression) - OP_RULE, R__TMP_116, + OP_RULE, R__TMP_119, OP_LOOP_ITERATE, // @@ -5170,8 +5305,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_60", - R__LOOP0_60, + {"_loop0_61", + R__LOOP0_61, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5185,8 +5320,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_61", - R__LOOP0_61, + {"_loop0_62", + R__LOOP0_62, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5200,8 +5335,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_62", - R__LOOP0_62, + {"_loop0_63", + R__LOOP0_63, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5215,8 +5350,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_63", - R__LOOP1_63, + {"_loop1_64", + R__LOOP1_64, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5230,8 +5365,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_64", - R__LOOP0_64, + {"_loop0_65", + R__LOOP0_65, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5245,8 +5380,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_65", - R__LOOP1_65, + {"_loop1_66", + R__LOOP1_66, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5260,8 +5395,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_66", - R__LOOP1_66, + {"_loop1_67", + R__LOOP1_67, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5275,8 +5410,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_67", - R__LOOP1_67, + {"_loop1_68", + R__LOOP1_68, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5290,8 +5425,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_68", - R__LOOP0_68, + {"_loop0_69", + R__LOOP0_69, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5305,8 +5440,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_69", - R__LOOP1_69, + {"_loop1_70", + R__LOOP1_70, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5320,8 +5455,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_70", - R__LOOP0_70, + {"_loop0_71", + R__LOOP0_71, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5335,8 +5470,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_71", - R__LOOP1_71, + {"_loop1_72", + R__LOOP1_72, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5350,8 +5485,8 @@ static Rule all_rules[] = { }, }, - {"_loop0_72", - R__LOOP0_72, + {"_loop0_73", + R__LOOP0_73, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5365,8 +5500,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_73", - R__LOOP1_73, + {"_loop1_74", + R__LOOP1_74, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5380,14 +5515,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_74", - R__LOOP1_74, + {"_loop1_75", + R__LOOP1_75, 0, // memo 0, // leftrec {0, 3, -1}, { // ('or' conjunction) - OP_RULE, R__TMP_117, + OP_RULE, R__TMP_120, OP_LOOP_ITERATE, // @@ -5395,14 +5530,14 @@ static Rule all_rules[] = { }, }, - {"_loop1_75", - R__LOOP1_75, + {"_loop1_76", + R__LOOP1_76, 0, // memo 0, // leftrec {0, 3, -1}, { // ('and' inversion) - OP_RULE, R__TMP_118, + OP_RULE, R__TMP_121, OP_LOOP_ITERATE, // @@ -5410,8 +5545,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_76", - R__LOOP1_76, + {"_loop1_77", + R__LOOP1_77, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5425,20 +5560,20 @@ static Rule all_rules[] = { }, }, - {"_tmp_77", - R__TMP_77, + {"_tmp_78", + R__TMP_78, 0, // memo 0, // leftrec {0, -1}, { // '!=' OP_TOKEN, NOTEQUAL, - OP_RETURN, A__TMP_77_0, + OP_RETURN, A__TMP_78_0, }, }, - {"_gather_78", - R__GATHER_78, + {"_gather_79", + R__GATHER_79, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5454,8 +5589,8 @@ static Rule all_rules[] = { }, }, - {"_tmp_79", - R__TMP_79, + {"_tmp_80", + R__TMP_80, 0, // memo 0, // leftrec {0, -1}, @@ -5464,72 +5599,72 @@ static Rule all_rules[] = { OP_TOKEN, COLON, OP_RULE, R_EXPRESSION, OP_OPTIONAL, - OP_RETURN, A__TMP_79_0, + OP_RETURN, A__TMP_80_0, }, }, - {"_tmp_80", - R__TMP_80, + {"_tmp_81", + R__TMP_81, 0, // memo 0, // leftrec {0, 4, 8, -1}, { // tuple OP_RULE, R_TUPLE, - OP_RETURN, A__TMP_80_0, + OP_RETURN, A__TMP_81_0, // group OP_RULE, R_GROUP, - OP_RETURN, A__TMP_80_1, + OP_RETURN, A__TMP_81_1, // genexp OP_RULE, R_GENEXP, - OP_RETURN, A__TMP_80_2, + OP_RETURN, A__TMP_81_2, }, }, - {"_tmp_81", - R__TMP_81, + {"_tmp_82", + R__TMP_82, 0, // memo 0, // leftrec {0, 4, -1}, { // list OP_RULE, R_LIST, - OP_RETURN, A__TMP_81_0, + OP_RETURN, A__TMP_82_0, // listcomp OP_RULE, R_LISTCOMP, - OP_RETURN, A__TMP_81_1, + OP_RETURN, A__TMP_82_1, }, }, - {"_tmp_82", - R__TMP_82, + {"_tmp_83", + R__TMP_83, 0, // memo 0, // leftrec {0, 4, 8, 12, -1}, { // dict OP_RULE, R_DICT, - OP_RETURN, A__TMP_82_0, + OP_RETURN, A__TMP_83_0, // set OP_RULE, R_SET, - OP_RETURN, A__TMP_82_1, + OP_RETURN, A__TMP_83_1, // dictcomp OP_RULE, R_DICTCOMP, - OP_RETURN, A__TMP_82_2, + OP_RETURN, A__TMP_83_2, // setcomp OP_RULE, R_SETCOMP, - OP_RETURN, A__TMP_82_3, + OP_RETURN, A__TMP_83_3, }, }, - {"_loop1_83", - R__LOOP1_83, + {"_loop1_84", + R__LOOP1_84, 0, // memo 0, // leftrec {0, 2, -1}, @@ -5543,8 +5678,8 @@ static Rule all_rules[] = { }, }, - {"_tmp_84", - R__TMP_84, + {"_tmp_85", + R__TMP_85, 0, // memo 0, // leftrec {0, -1}, @@ -5554,28 +5689,28 @@ static Rule all_rules[] = { OP_TOKEN, COMMA, OP_RULE, R_STAR_NAMED_EXPRESSIONS, OP_OPTIONAL, - OP_RETURN, A__TMP_84_0, + OP_RETURN, A__TMP_85_0, }, }, - {"_tmp_85", - R__TMP_85, + {"_tmp_86", + R__TMP_86, 0, // memo 0, // leftrec {0, 4, -1}, { // yield_expr OP_RULE, R_YIELD_EXPR, - OP_RETURN, A__TMP_85_0, + OP_RETURN, A__TMP_86_0, // named_expression OP_RULE, R_NAMED_EXPRESSION, - OP_RETURN, A__TMP_85_1, + OP_RETURN, A__TMP_86_1, }, }, - {"_gather_86", - R__GATHER_86, + {"_gather_87", + R__GATHER_87, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5591,8 +5726,8 @@ static Rule all_rules[] = { }, }, - {"_loop1_87", - R__LOOP1_87, + {"_loop1_88", + R__LOOP1_88, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5606,14 +5741,14 @@ static Rule all_rules[] = { }, }, - {"_loop0_88", - R__LOOP0_88, + {"_loop0_89", + R__LOOP0_89, 0, // memo 0, // leftrec {0, 3, -1}, { // ('if' disjunction) - OP_RULE, R__TMP_119, + OP_RULE, R__TMP_122, OP_LOOP_ITERATE, // @@ -5621,14 +5756,14 @@ static Rule all_rules[] = { }, }, - {"_loop0_89", - R__LOOP0_89, + {"_loop0_90", + R__LOOP0_90, 0, // memo 0, // leftrec {0, 3, -1}, { // ('if' disjunction) - OP_RULE, R__TMP_120, + OP_RULE, R__TMP_123, OP_LOOP_ITERATE, // @@ -5636,8 +5771,8 @@ static Rule all_rules[] = { }, }, - {"_tmp_90", - R__TMP_90, + {"_tmp_91", + R__TMP_91, 0, // memo 0, // leftrec {0, -1}, @@ -5645,12 +5780,12 @@ static Rule all_rules[] = { // ',' args OP_TOKEN, COMMA, OP_RULE, R_ARGS, - OP_RETURN, A__TMP_90_0, + OP_RETURN, A__TMP_91_0, }, }, - {"_tmp_91", - R__TMP_91, + {"_tmp_92", + R__TMP_92, 0, // memo 0, // leftrec {0, -1}, @@ -5658,12 +5793,12 @@ static Rule all_rules[] = { // ',' args OP_TOKEN, COMMA, OP_RULE, R_ARGS, - OP_RETURN, A__TMP_91_0, + OP_RETURN, A__TMP_92_0, }, }, - {"_gather_92", - R__GATHER_92, + {"_gather_93", + R__GATHER_93, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5679,8 +5814,8 @@ static Rule all_rules[] = { }, }, - {"_gather_93", - R__GATHER_93, + {"_gather_94", + R__GATHER_94, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5696,8 +5831,8 @@ static Rule all_rules[] = { }, }, - {"_gather_94", - R__GATHER_94, + {"_gather_95", + R__GATHER_95, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5713,8 +5848,8 @@ static Rule all_rules[] = { }, }, - {"_gather_95", - R__GATHER_95, + {"_gather_96", + R__GATHER_96, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5730,14 +5865,14 @@ static Rule all_rules[] = { }, }, - {"_loop0_96", - R__LOOP0_96, + {"_loop0_97", + R__LOOP0_97, 0, // memo 0, // leftrec {0, 3, -1}, { // (',' star_target) - OP_RULE, R__TMP_121, + OP_RULE, R__TMP_124, OP_LOOP_ITERATE, // @@ -5745,8 +5880,8 @@ static Rule all_rules[] = { }, }, - {"_gather_97", - R__GATHER_97, + {"_gather_98", + R__GATHER_98, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5762,8 +5897,8 @@ static Rule all_rules[] = { }, }, - {"_tmp_98", - R__TMP_98, + {"_tmp_99", + R__TMP_99, 0, // memo 0, // leftrec {0, -1}, @@ -5773,12 +5908,12 @@ static Rule all_rules[] = { OP_TOKEN, STAR, OP_NEG_LOOKAHEAD, OP_RULE, R_STAR_TARGET, - OP_RETURN, A__TMP_98_0, + OP_RETURN, A__TMP_99_0, }, }, - {"_gather_99", - R__GATHER_99, + {"_gather_100", + R__GATHER_100, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5794,8 +5929,8 @@ static Rule all_rules[] = { }, }, - {"_gather_100", - R__GATHER_100, + {"_gather_101", + R__GATHER_101, 0, // memo 0, // leftrec {0, 5, -1}, @@ -5811,25 +5946,25 @@ static Rule all_rules[] = { }, }, - {"_tmp_101", - R__TMP_101, + {"_tmp_102", + R__TMP_102, 0, // memo 0, // leftrec {0, 4, -1}, { // args OP_RULE, R_ARGS, - OP_RETURN, A__TMP_101_0, + OP_RETURN, A__TMP_102_0, // expression for_if_clauses OP_RULE, R_EXPRESSION, OP_RULE, R_FOR_IF_CLAUSES, - OP_RETURN, A__TMP_101_1, + OP_RETURN, A__TMP_102_1, }, }, - {"_loop0_102", - R__LOOP0_102, + {"_loop0_103", + R__LOOP0_103, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5843,73 +5978,74 @@ static Rule all_rules[] = { }, }, - {"_tmp_103", - R__TMP_103, + {"_loop0_104", + R__LOOP0_104, 0, // memo 0, // leftrec - {0, -1}, + {0, 3, -1}, { - // '=' annotated_rhs - OP_TOKEN, EQUAL, - OP_RULE, R_ANNOTATED_RHS, - OP_RETURN, A__TMP_103_0, + // (star_targets '=') + OP_RULE, R__TMP_125, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, }, }, - {"_tmp_104", - R__TMP_104, + {"_loop0_105", + R__LOOP0_105, 0, // memo 0, // leftrec - {0, 4, -1}, + {0, 3, -1}, { - // yield_expr - OP_RULE, R_YIELD_EXPR, - OP_RETURN, A__TMP_104_0, + // (star_targets '=') + OP_RULE, R__TMP_126, + OP_LOOP_ITERATE, - // star_expressions - OP_RULE, R_STAR_EXPRESSIONS, - OP_RETURN, A__TMP_104_1, + // + OP_LOOP_COLLECT, }, }, - {"_tmp_105", - R__TMP_105, + {"_tmp_106", + R__TMP_106, 0, // memo 0, // leftrec {0, 4, -1}, { // yield_expr OP_RULE, R_YIELD_EXPR, - OP_RETURN, A__TMP_105_0, + OP_RETURN, A__TMP_106_0, // star_expressions OP_RULE, R_STAR_EXPRESSIONS, - OP_RETURN, A__TMP_105_1, + OP_RETURN, A__TMP_106_1, }, }, - {"_tmp_106", - R__TMP_106, + {"_tmp_107", + R__TMP_107, 0, // memo 0, // leftrec {0, 4, 8, -1}, { // '[' OP_TOKEN, LSQB, - OP_RETURN, A__TMP_106_0, + OP_RETURN, A__TMP_107_0, // '(' OP_TOKEN, LPAR, - OP_RETURN, A__TMP_106_1, + OP_RETURN, A__TMP_107_1, // '{' OP_TOKEN, LBRACE, - OP_RETURN, A__TMP_106_2, + OP_RETURN, A__TMP_107_2, }, }, - {"_loop0_107", - R__LOOP0_107, + {"_loop0_108", + R__LOOP0_108, 0, // memo 0, // leftrec {0, 3, -1}, @@ -5923,58 +6059,89 @@ static Rule all_rules[] = { }, }, - {"_tmp_108", - R__TMP_108, + {"_tmp_109", + R__TMP_109, 0, // memo 0, // leftrec {0, 4, -1}, { // slash_with_default OP_RULE, R_SLASH_WITH_DEFAULT, - OP_RETURN, A__TMP_108_0, + OP_RETURN, A__TMP_109_0, // param_with_default+ - OP_RULE, R__LOOP1_122, - OP_RETURN, A__TMP_108_1, + OP_RULE, R__LOOP1_127, + OP_RETURN, A__TMP_109_1, }, }, - {"_tmp_109", - R__TMP_109, + {"_loop0_110", + R__LOOP0_110, + 0, // memo + 0, // leftrec + {0, 3, -1}, + { + // lambda_param_no_default + OP_RULE, R_LAMBDA_PARAM_NO_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT, + + }, + }, + {"_tmp_111", + R__TMP_111, + 0, // memo + 0, // leftrec + {0, 4, -1}, + { + // lambda_slash_with_default + OP_RULE, R_LAMBDA_SLASH_WITH_DEFAULT, + OP_RETURN, A__TMP_111_0, + + // lambda_param_with_default+ + OP_RULE, R__LOOP1_128, + OP_RETURN, A__TMP_111_1, + + }, + }, + {"_tmp_112", + R__TMP_112, 0, // memo 0, // leftrec {0, 4, -1}, { // ')' OP_TOKEN, RPAR, - OP_RETURN, A__TMP_109_0, + OP_RETURN, A__TMP_112_0, // ',' (')' | '**') OP_TOKEN, COMMA, - OP_RULE, R__TMP_123, - OP_RETURN, A__TMP_109_1, + OP_RULE, R__TMP_129, + OP_RETURN, A__TMP_112_1, }, }, - {"_tmp_110", - R__TMP_110, + {"_tmp_113", + R__TMP_113, 0, // memo 0, // leftrec {0, 4, -1}, { // ':' OP_TOKEN, COLON, - OP_RETURN, A__TMP_110_0, + OP_RETURN, A__TMP_113_0, // ',' (':' | '**') OP_TOKEN, COMMA, - OP_RULE, R__TMP_124, - OP_RETURN, A__TMP_110_1, + OP_RULE, R__TMP_130, + OP_RETURN, A__TMP_113_1, }, }, - {"_tmp_111", - R__TMP_111, + {"_tmp_114", + R__TMP_114, 0, // memo 0, // leftrec {0, -1}, @@ -5982,44 +6149,44 @@ static Rule all_rules[] = { // star_targets '=' OP_RULE, R_STAR_TARGETS, OP_TOKEN, EQUAL, - OP_RETURN, A__TMP_111_0, + OP_RETURN, A__TMP_114_0, }, }, - {"_tmp_112", - R__TMP_112, + {"_tmp_115", + R__TMP_115, 0, // memo 0, // leftrec {0, 4, -1}, { // '.' OP_TOKEN, DOT, - OP_RETURN, A__TMP_112_0, + OP_RETURN, A__TMP_115_0, // '...' OP_TOKEN, ELLIPSIS, - OP_RETURN, A__TMP_112_1, + OP_RETURN, A__TMP_115_1, }, }, - {"_tmp_113", - R__TMP_113, + {"_tmp_116", + R__TMP_116, 0, // memo 0, // leftrec {0, 4, -1}, { // '.' OP_TOKEN, DOT, - OP_RETURN, A__TMP_113_0, + OP_RETURN, A__TMP_116_0, // '...' OP_TOKEN, ELLIPSIS, - OP_RETURN, A__TMP_113_1, + OP_RETURN, A__TMP_116_1, }, }, - {"_tmp_114", - R__TMP_114, + {"_tmp_117", + R__TMP_117, 0, // memo 0, // leftrec {0, -1}, @@ -6028,12 +6195,12 @@ static Rule all_rules[] = { OP_TOKEN, AT, OP_RULE, R_NAMED_EXPRESSION, OP_TOKEN, NEWLINE, - OP_RETURN, A__TMP_114_0, + OP_RETURN, A__TMP_117_0, }, }, - {"_tmp_115", - R__TMP_115, + {"_tmp_118", + R__TMP_118, 0, // memo 0, // leftrec {0, -1}, @@ -6041,12 +6208,12 @@ static Rule all_rules[] = { // ',' star_expression OP_TOKEN, COMMA, OP_RULE, R_STAR_EXPRESSION, - OP_RETURN, A__TMP_115_0, + OP_RETURN, A__TMP_118_0, }, }, - {"_tmp_116", - R__TMP_116, + {"_tmp_119", + R__TMP_119, 0, // memo 0, // leftrec {0, -1}, @@ -6054,38 +6221,38 @@ static Rule all_rules[] = { // ',' expression OP_TOKEN, COMMA, OP_RULE, R_EXPRESSION, - OP_RETURN, A__TMP_116_0, + OP_RETURN, A__TMP_119_0, }, }, - {"_tmp_117", - R__TMP_117, + {"_tmp_120", + R__TMP_120, 0, // memo 0, // leftrec {0, -1}, { // 'or' conjunction - OP_TOKEN, 532, + OP_TOKEN, 531, OP_RULE, R_CONJUNCTION, - OP_RETURN, A__TMP_117_0, + OP_RETURN, A__TMP_120_0, }, }, - {"_tmp_118", - R__TMP_118, + {"_tmp_121", + R__TMP_121, 0, // memo 0, // leftrec {0, -1}, { // 'and' inversion - OP_TOKEN, 533, + OP_TOKEN, 532, OP_RULE, R_INVERSION, - OP_RETURN, A__TMP_118_0, + OP_RETURN, A__TMP_121_0, }, }, - {"_tmp_119", - R__TMP_119, + {"_tmp_122", + R__TMP_122, 0, // memo 0, // leftrec {0, -1}, @@ -6093,12 +6260,12 @@ static Rule all_rules[] = { // 'if' disjunction OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, - OP_RETURN, A__TMP_119_0, + OP_RETURN, A__TMP_122_0, }, }, - {"_tmp_120", - R__TMP_120, + {"_tmp_123", + R__TMP_123, 0, // memo 0, // leftrec {0, -1}, @@ -6106,12 +6273,12 @@ static Rule all_rules[] = { // 'if' disjunction OP_TOKEN, 510, OP_RULE, R_DISJUNCTION, - OP_RETURN, A__TMP_120_0, + OP_RETURN, A__TMP_123_0, }, }, - {"_tmp_121", - R__TMP_121, + {"_tmp_124", + R__TMP_124, 0, // memo 0, // leftrec {0, -1}, @@ -6119,12 +6286,38 @@ static Rule all_rules[] = { // ',' star_target OP_TOKEN, COMMA, OP_RULE, R_STAR_TARGET, - OP_RETURN, A__TMP_121_0, + OP_RETURN, A__TMP_124_0, + + }, + }, + {"_tmp_125", + R__TMP_125, + 0, // memo + 0, // leftrec + {0, -1}, + { + // star_targets '=' + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, EQUAL, + OP_RETURN, A__TMP_125_0, + + }, + }, + {"_tmp_126", + R__TMP_126, + 0, // memo + 0, // leftrec + {0, -1}, + { + // star_targets '=' + OP_RULE, R_STAR_TARGETS, + OP_TOKEN, EQUAL, + OP_RETURN, A__TMP_126_0, }, }, - {"_loop1_122", - R__LOOP1_122, + {"_loop1_127", + R__LOOP1_127, 0, // memo 0, // leftrec {0, 3, -1}, @@ -6138,35 +6331,50 @@ static Rule all_rules[] = { }, }, - {"_tmp_123", - R__TMP_123, + {"_loop1_128", + R__LOOP1_128, + 0, // memo + 0, // leftrec + {0, 3, -1}, + { + // lambda_param_with_default + OP_RULE, R_LAMBDA_PARAM_WITH_DEFAULT, + OP_LOOP_ITERATE, + + // + OP_LOOP_COLLECT_NONEMPTY, + + }, + }, + {"_tmp_129", + R__TMP_129, 0, // memo 0, // leftrec {0, 4, -1}, { // ')' OP_TOKEN, RPAR, - OP_RETURN, A__TMP_123_0, + OP_RETURN, A__TMP_129_0, // '**' OP_TOKEN, DOUBLESTAR, - OP_RETURN, A__TMP_123_1, + OP_RETURN, A__TMP_129_1, }, }, - {"_tmp_124", - R__TMP_124, + {"_tmp_130", + R__TMP_130, 0, // memo 0, // leftrec {0, 4, -1}, { // ':' OP_TOKEN, COLON, - OP_RETURN, A__TMP_124_0, + OP_RETURN, A__TMP_130_0, // '**' OP_TOKEN, DOUBLESTAR, - OP_RETURN, A__TMP_124_1, + OP_RETURN, A__TMP_130_1, }, }, @@ -6227,6 +6435,8 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_DEL_TARGET_2: case A_TARGET_2: case A_T_PRIMARY_4: + case A_INVALID_ANN_ASSIGN_TARGET_0: + case A_INVALID_ANN_ASSIGN_TARGET_1: case A__GATHER_3_0: case A__GATHER_3_1: case A__GATHER_4_0: @@ -6244,36 +6454,36 @@ call_action(Parser *p, Frame *_f, int _iaction) case A__GATHER_20_1: case A__GATHER_21_0: case A__GATHER_21_1: - case A__GATHER_56_0: - case A__GATHER_56_1: - case A__GATHER_58_0: - case A__GATHER_58_1: - case A__GATHER_78_0: - case A__GATHER_78_1: - case A__TMP_80_0: - case A__TMP_80_1: - case A__TMP_80_2: + case A__GATHER_57_0: + case A__GATHER_57_1: + case A__GATHER_59_0: + case A__GATHER_59_1: + case A__GATHER_79_0: + case A__GATHER_79_1: case A__TMP_81_0: case A__TMP_81_1: + case A__TMP_81_2: case A__TMP_82_0: case A__TMP_82_1: - case A__TMP_82_2: - case A__TMP_82_3: - case A__TMP_85_0: - case A__TMP_85_1: - case A__GATHER_97_0: - case A__GATHER_97_1: - case A__GATHER_99_0: - case A__GATHER_99_1: + case A__TMP_83_0: + case A__TMP_83_1: + case A__TMP_83_2: + case A__TMP_83_3: + case A__TMP_86_0: + case A__TMP_86_1: + case A__GATHER_98_0: + case A__GATHER_98_1: case A__GATHER_100_0: case A__GATHER_100_1: - case A__TMP_101_0: - case A__TMP_101_1: - case A__TMP_104_0: - case A__TMP_104_1: - case A__TMP_105_0: - case A__TMP_105_1: - case A__TMP_111_0: + case A__GATHER_101_0: + case A__GATHER_101_1: + case A__TMP_102_0: + case A__TMP_102_1: + case A__TMP_106_0: + case A__TMP_106_1: + case A__TMP_114_0: + case A__TMP_125_0: + case A__TMP_126_0: return ((expr_ty)_f->vals[0]); case A_TYPE_EXPRESSIONS_0: return _PyPegen_seq_append_to_end ( p , CHECK ( _PyPegen_seq_append_to_end ( p , _f->vals[0] , ((expr_ty)_f->vals[3]) ) ) , ((expr_ty)_f->vals[6]) ); @@ -6303,9 +6513,12 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_COMPOUND_STMT_5: case A_COMPOUND_STMT_6: case A_ASSIGNMENT_4: + case A_DEL_STMT_1: case A_IMPORT_FROM_TARGETS_3: case A_IMPORT_FROM_AS_NAMES_0: case A_DOTTED_AS_NAMES_0: + case A_FOR_STMT_2: + case A_WITH_ITEM_1: case A_FUNC_TYPE_COMMENT_1: case A_FUNC_TYPE_COMMENT_2: case A_PARAMS_0: @@ -6317,20 +6530,23 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_EXPRESSIONS_LIST_0: case A_STAR_NAMED_EXPRESSIONS_0: case A_NAMED_EXPRESSION_2: + case A_LAMBDA_PARAMS_0: case A_LAMBDA_SLASH_NO_DEFAULT_0: case A_LAMBDA_SLASH_NO_DEFAULT_1: case A_LAMBDA_STAR_ETC_3: + case A_ATOM_4: case A_ATOM_5: case A_ATOM_6: case A_ATOM_7: case A_ATOM_8: - case A_ATOM_9: case A_LISTCOMP_1: + case A_GROUP_1: case A_GENEXP_1: case A_SETCOMP_1: case A_DICTCOMP_1: case A_DOUBLE_STARRED_KVPAIRS_0: case A_FOR_IF_CLAUSES_0: + case A_FOR_IF_CLAUSE_2: case A_ARGUMENTS_1: case A_KWARGS_1: case A_KWARGS_2: @@ -6338,15 +6554,15 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_KWARG_OR_DOUBLE_STARRED_2: case A_STAR_TARGETS_SEQ_0: case A_DEL_TARGETS_0: - case A_DEL_T_ATOM_4: - case A_DEL_TARGET_END_4: case A_TARGETS_0: case A__TMP_10_2: case A__TMP_12_1: case A__TMP_13_1: - case A__TMP_39_0: - case A__TMP_98_0: - case A__TMP_108_1: + case A__TMP_23_1: + case A__TMP_40_0: + case A__TMP_99_0: + case A__TMP_109_1: + case A__TMP_111_1: return _f->vals[0]; case A_STATEMENTS_0: return _PyPegen_seq_flatten ( p , _f->vals[0] ); @@ -6450,9 +6666,9 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_WHILE_STMT_0: return _Py_While ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[3]) , _f->vals[4] , EXTRA ); case A_FOR_STMT_0: - return _Py_For ( ((expr_ty)_f->vals[1]) , ((expr_ty)_f->vals[3]) , ((asdl_seq*)_f->vals[6]) , _f->vals[7] , NEW_TYPE_COMMENT ( p , _f->vals[5] ) , EXTRA ); + return _Py_For ( ((expr_ty)_f->vals[1]) , _f->vals[3] , _f->vals[6] , ((asdl_seq*)_f->vals[7]) , NEW_TYPE_COMMENT ( p , ((Token *)_f->vals[5]) ) , EXTRA ); case A_FOR_STMT_1: - return CHECK_VERSION ( 5 , "Async for loops are" , _Py_AsyncFor ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[4]) , ((asdl_seq*)_f->vals[7]) , _f->vals[8] , NEW_TYPE_COMMENT ( p , _f->vals[6] ) , EXTRA ) ); + return CHECK_VERSION ( 5 , "Async for loops are" , _Py_AsyncFor ( ((expr_ty)_f->vals[2]) , _f->vals[4] , _f->vals[7] , ((asdl_seq*)_f->vals[8]) , NEW_TYPE_COMMENT ( p , ((Token *)_f->vals[6]) ) , EXTRA ) ); case A_WITH_STMT_0: return _Py_With ( _f->vals[2] , ((asdl_seq*)_f->vals[6]) , NULL , EXTRA ); case A_WITH_STMT_1: @@ -6462,7 +6678,9 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_WITH_STMT_3: return CHECK_VERSION ( 5 , "Async with statements are" , _Py_AsyncWith ( _f->vals[2] , ((asdl_seq*)_f->vals[5]) , NEW_TYPE_COMMENT ( p , _f->vals[4] ) , EXTRA ) ); case A_WITH_ITEM_0: - return _Py_withitem ( ((expr_ty)_f->vals[0]) , _f->vals[1] , p -> arena ); + return _Py_withitem ( ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) , p -> arena ); + case A_WITH_ITEM_2: + return _Py_withitem ( ((expr_ty)_f->vals[0]) , NULL , p -> arena ); case A_TRY_STMT_0: return _Py_Try ( ((asdl_seq*)_f->vals[2]) , NULL , NULL , ((asdl_seq*)_f->vals[3]) , EXTRA ); case A_TRY_STMT_1: @@ -6485,10 +6703,11 @@ call_action(Parser *p, Frame *_f, int _iaction) return CHECK_VERSION ( 5 , "Async functions are" , _Py_AsyncFunctionDef ( ((expr_ty)_f->vals[2]) -> v . Name . id , ( _f->vals[4] ) ? _f->vals[4] : CHECK ( _PyPegen_empty_arguments ( p ) ) , ((asdl_seq*)_f->vals[9]) , NULL , _f->vals[6] , NEW_TYPE_COMMENT ( p , _f->vals[8] ) , EXTRA ) ); case A_FUNC_TYPE_COMMENT_0: case A_GROUP_0: - case A__TMP_55_0: - case A__TMP_79_0: + case A__TMP_56_0: + case A__TMP_80_0: return _f->vals[1]; case A_PARAMS_1: + case A_LAMBDA_PARAMS_1: return ((arguments_ty)_f->vals[0]); case A_PARAMETERS_0: case A_LAMBDA_PARAMETERS_0: @@ -6539,27 +6758,27 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_ANNOTATION_0: case A_DEFAULT_0: case A_SINGLE_TARGET_2: + case A_INVALID_ANN_ASSIGN_TARGET_2: case A__TMP_14_0: case A__TMP_15_0: case A__TMP_16_0: case A__TMP_22_0: - case A__TMP_26_0: - case A__TMP_28_0: - case A__TMP_33_0: - case A__TMP_35_0: + case A__TMP_27_0: + case A__TMP_29_0: case A__TMP_36_0: case A__TMP_37_0: case A__TMP_38_0: - case A__TMP_90_0: + case A__TMP_39_0: case A__TMP_91_0: - case A__TMP_114_0: - case A__TMP_115_0: - case A__TMP_116_0: + case A__TMP_92_0: case A__TMP_117_0: case A__TMP_118_0: case A__TMP_119_0: case A__TMP_120_0: case A__TMP_121_0: + case A__TMP_122_0: + case A__TMP_123_0: + case A__TMP_124_0: return ((expr_ty)_f->vals[1]); case A_CLASS_DEF_0: return _PyPegen_class_def_decorators ( p , ((asdl_seq*)_f->vals[0]) , ((stmt_ty)_f->vals[1]) ); @@ -6576,7 +6795,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_STARRED_EXPRESSION_0: return _Py_Starred ( ((expr_ty)_f->vals[1]) , Load , EXTRA ); case A_NAMED_EXPRESSION_0: - return _Py_NamedExpr ( CHECK ( _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Store ) ) , ((expr_ty)_f->vals[2]) , EXTRA ); + return _Py_NamedExpr ( CHECK ( _PyPegen_set_expr_context ( p , ((expr_ty)_f->vals[0]) , Store ) ) , _f->vals[2] , EXTRA ); case A_EXPRESSION_0: return _Py_IfExp ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[4]) , EXTRA ); case A_LAMBDEF_0: @@ -6687,24 +6906,22 @@ call_action(Parser *p, Frame *_f, int _iaction) return _Py_Constant ( Py_False , NULL , EXTRA ); case A_ATOM_3: return _Py_Constant ( Py_None , NULL , EXTRA ); - case A_ATOM_4: - return RAISE_SYNTAX_ERROR ( "You found it!" ); - case A_ATOM_10: + case A_ATOM_9: return _Py_Constant ( Py_Ellipsis , NULL , EXTRA ); case A_STRINGS_0: return _PyPegen_concatenate_strings ( p , _f->vals[0] ); case A_LIST_0: return _Py_List ( _f->vals[1] , Load , EXTRA ); case A_LISTCOMP_0: - return _Py_ListComp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + return _Py_ListComp ( ((expr_ty)_f->vals[1]) , _f->vals[2] , EXTRA ); case A_TUPLE_0: return _Py_Tuple ( _f->vals[1] , Load , EXTRA ); case A_GENEXP_0: - return _Py_GeneratorExp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + return _Py_GeneratorExp ( ((expr_ty)_f->vals[1]) , _f->vals[2] , EXTRA ); case A_SET_0: return _Py_Set ( ((asdl_seq*)_f->vals[1]) , EXTRA ); case A_SETCOMP_0: - return _Py_SetComp ( ((expr_ty)_f->vals[1]) , ((asdl_seq*)_f->vals[2]) , EXTRA ); + return _Py_SetComp ( ((expr_ty)_f->vals[1]) , _f->vals[2] , EXTRA ); case A_DICT_0: return _Py_Dict ( CHECK ( _PyPegen_get_keys ( p , _f->vals[1] ) ) , CHECK ( _PyPegen_get_values ( p , _f->vals[1] ) ) , EXTRA ); case A_DICTCOMP_0: @@ -6712,15 +6929,15 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_DOUBLE_STARRED_KVPAIR_0: return _PyPegen_key_value_pair ( p , NULL , ((expr_ty)_f->vals[1]) ); case A_DOUBLE_STARRED_KVPAIR_1: - case A__GATHER_86_0: - case A__GATHER_86_1: + case A__GATHER_87_0: + case A__GATHER_87_1: return ((KeyValuePair*)_f->vals[0]); case A_KVPAIR_0: return _PyPegen_key_value_pair ( p , ((expr_ty)_f->vals[0]) , ((expr_ty)_f->vals[2]) ); case A_FOR_IF_CLAUSE_0: - return CHECK_VERSION ( 6 , "Async comprehensions are" , _Py_comprehension ( ((expr_ty)_f->vals[2]) , ((expr_ty)_f->vals[4]) , _f->vals[5] , 1 , p -> arena ) ); + return CHECK_VERSION ( 6 , "Async comprehensions are" , _Py_comprehension ( ((expr_ty)_f->vals[2]) , _f->vals[4] , ((expr_ty)_f->vals[5]) , 1 , p -> arena ) ); case A_FOR_IF_CLAUSE_1: - return _Py_comprehension ( ((expr_ty)_f->vals[1]) , ((expr_ty)_f->vals[3]) , _f->vals[4] , 0 , p -> arena ); + return _Py_comprehension ( ((expr_ty)_f->vals[1]) , _f->vals[3] , ((expr_ty)_f->vals[4]) , 0 , p -> arena ); case A_YIELD_EXPR_0: return _Py_YieldFrom ( ((expr_ty)_f->vals[2]) , EXTRA ); case A_YIELD_EXPR_1: @@ -6776,10 +6993,6 @@ call_action(Parser *p, Frame *_f, int _iaction) return _Py_Tuple ( _f->vals[1] , Del , EXTRA ); case A_DEL_T_ATOM_3: return _Py_List ( _f->vals[1] , Del , EXTRA ); - case A_DEL_TARGET_END_0: - case A_DEL_TARGET_END_1: - case A_DEL_TARGET_END_2: - case A_DEL_TARGET_END_3: case A_T_LOOKAHEAD_0: case A_T_LOOKAHEAD_1: case A_T_LOOKAHEAD_2: @@ -6791,22 +7004,25 @@ call_action(Parser *p, Frame *_f, int _iaction) case A__TMP_11_1: case A__TMP_12_0: case A__TMP_13_0: - case A__TMP_103_0: - case A__TMP_106_0: - case A__TMP_106_1: - case A__TMP_106_2: - case A__TMP_109_0: - case A__TMP_109_1: - case A__TMP_110_0: - case A__TMP_110_1: + case A__TMP_23_0: + case A__TMP_34_0: + case A__TMP_34_1: + case A__TMP_34_2: + case A__TMP_107_0: + case A__TMP_107_1: + case A__TMP_107_2: case A__TMP_112_0: case A__TMP_112_1: case A__TMP_113_0: case A__TMP_113_1: - case A__TMP_123_0: - case A__TMP_123_1: - case A__TMP_124_0: - case A__TMP_124_1: + case A__TMP_115_0: + case A__TMP_115_1: + case A__TMP_116_0: + case A__TMP_116_1: + case A__TMP_129_0: + case A__TMP_129_1: + case A__TMP_130_0: + case A__TMP_130_1: return ((Token *)_f->vals[0]); case A_INCORRECT_ARGUMENTS_0: return RAISE_SYNTAX_ERROR ( "iterable argument unpacking follows keyword argument unpacking" ); @@ -6823,16 +7039,19 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_INVALID_NAMED_EXPRESSION_0: return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); case A_INVALID_ASSIGNMENT_0: - return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "only single target (not list) can be annotated" ); + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "only single target (not %s) can be annotated" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); case A_INVALID_ASSIGNMENT_1: - case A_INVALID_ASSIGNMENT_2: return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "only single target (not tuple) can be annotated" ); - case A_INVALID_ASSIGNMENT_3: + case A_INVALID_ASSIGNMENT_2: return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "illegal target for annotation" ); + case A_INVALID_ASSIGNMENT_3: + return RAISE_SYNTAX_ERROR_INVALID_TARGET ( STAR_TARGETS , ((expr_ty)_f->vals[1]) ); case A_INVALID_ASSIGNMENT_4: - return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( _PyPegen_get_invalid_target ( ((expr_ty)_f->vals[0]) ) , "cannot assign to %s" , _PyPegen_get_expr_name ( _PyPegen_get_invalid_target ( ((expr_ty)_f->vals[0]) ) ) ); + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[1]) , "assignment to yield expression not possible" ); case A_INVALID_ASSIGNMENT_5: return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "'%s' is an illegal expression for augmented assignment" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); + case A_INVALID_DEL_STMT_0: + return RAISE_SYNTAX_ERROR_INVALID_TARGET ( DEL_TARGETS , ((expr_ty)_f->vals[1]) ); case A_INVALID_BLOCK_0: return RAISE_INDENTATION_ERROR ( "expected an indented block" ); case A_INVALID_COMPREHENSION_0: @@ -6840,6 +7059,7 @@ call_action(Parser *p, Frame *_f, int _iaction) case A_INVALID_DICT_COMPREHENSION_0: return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((Token *)_f->vals[1]) , "dict unpacking cannot be used in dict comprehension" ); case A_INVALID_PARAMETERS_0: + case A_INVALID_LAMBDA_PARAMETERS_0: return RAISE_SYNTAX_ERROR ( "non-default argument follows default argument" ); case A_INVALID_STAR_ETC_0: case A_INVALID_LAMBDA_STAR_ETC_0: @@ -6848,38 +7068,43 @@ call_action(Parser *p, Frame *_f, int _iaction) return RAISE_SYNTAX_ERROR ( "bare * has associated type comment" ); case A_INVALID_DOUBLE_TYPE_COMMENTS_0: return RAISE_SYNTAX_ERROR ( "Cannot have two type comments on def" ); - case A_INVALID_DEL_TARGET_0: - return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[0]) , "cannot delete %s" , _PyPegen_get_expr_name ( ((expr_ty)_f->vals[0]) ) ); + case A_INVALID_WITH_ITEM_0: + return RAISE_SYNTAX_ERROR_INVALID_TARGET ( STAR_TARGETS , ((expr_ty)_f->vals[2]) ); + case A_INVALID_FOR_TARGET_0: + return RAISE_SYNTAX_ERROR_INVALID_TARGET ( FOR_TARGETS , ((expr_ty)_f->vals[2]) ); + case A_INVALID_GROUP_0: + return RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( ((expr_ty)_f->vals[1]) , "can't use starred expression here" ); case A_INVALID_IMPORT_FROM_TARGETS_0: return RAISE_SYNTAX_ERROR ( "trailing comma not allowed without surrounding parentheses" ); - case A__GATHER_25_0: - case A__GATHER_25_1: - case A__GATHER_27_0: - case A__GATHER_27_1: + case A__GATHER_26_0: + case A__GATHER_26_1: + case A__GATHER_28_0: + case A__GATHER_28_1: return ((alias_ty)_f->vals[0]); - case A__GATHER_29_0: - case A__GATHER_29_1: case A__GATHER_30_0: case A__GATHER_30_1: case A__GATHER_31_0: case A__GATHER_31_1: case A__GATHER_32_0: case A__GATHER_32_1: + case A__GATHER_33_0: + case A__GATHER_33_1: return ((withitem_ty)_f->vals[0]); - case A__TMP_77_0: + case A__TMP_78_0: return _PyPegen_check_barry_as_flufl ( p ) ? NULL : ((Token *)_f->vals[0]); - case A__TMP_84_0: + case A__TMP_85_0: return _PyPegen_seq_insert_in_front ( p , ((expr_ty)_f->vals[0]) , _f->vals[2] ); - case A__GATHER_92_0: - case A__GATHER_92_1: case A__GATHER_93_0: case A__GATHER_93_1: case A__GATHER_94_0: case A__GATHER_94_1: case A__GATHER_95_0: case A__GATHER_95_1: + case A__GATHER_96_0: + case A__GATHER_96_1: return ((KeywordOrStarred*)_f->vals[0]); - case A__TMP_108_0: + case A__TMP_109_0: + case A__TMP_111_0: return ((SlashWithDefault*)_f->vals[0]); default: assert(0); From a65604e708eee81c75e6c9263692f161ea0a6696 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 13 Aug 2022 19:10:25 +0100 Subject: [PATCH 67/67] bpo-40222: Mark exception table function in the dis module as private --- Lib/dis.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index d63f35a42b2467..a045d18241b465 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -394,7 +394,7 @@ def _get_name_info(name_index, get_name, **extrainfo): else: return UNKNOWN, '' -def parse_varint(iterator): +def _parse_varint(iterator): b = next(iterator) val = b & 63 while b&64: @@ -403,16 +403,16 @@ def parse_varint(iterator): val |= b&63 return val -def parse_exception_table(code): +def _parse_exception_table(code): iterator = iter(code.co_exceptiontable) entries = [] try: while True: - start = parse_varint(iterator)*2 - length = parse_varint(iterator)*2 + start = _parse_varint(iterator)*2 + length = _parse_varint(iterator)*2 end = start + length - target = parse_varint(iterator)*2 - dl = parse_varint(iterator) + target = _parse_varint(iterator)*2 + dl = _parse_varint(iterator) depth = dl >> 1 lasti = bool(dl&1) entries.append(_ExceptionTableEntry(start, end, target, depth, lasti)) @@ -527,7 +527,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) - exception_entries = parse_exception_table(co) + exception_entries = _parse_exception_table(co) _disassemble_bytes(_get_code_array(co, adaptive), lasti, co._varname_from_oparg, co.co_names, co.co_consts, linestarts, file=file, @@ -717,7 +717,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self._linestarts = dict(findlinestarts(co)) self._original_object = x self.current_offset = current_offset - self.exception_entries = parse_exception_table(co) + self.exception_entries = _parse_exception_table(co) self.show_caches = show_caches self.adaptive = adaptive