Skip to content

Commit

Permalink
Avoid reducing nesting level if lexer moves before closing parenthesis
Browse files Browse the repository at this point in the history
  • Loading branch information
dhruvmanila committed Jun 14, 2024
1 parent 03bd533 commit 4e5ccde
Show file tree
Hide file tree
Showing 3 changed files with 138 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,14 @@ def bar():

# The outer parenthesis is closed but the inner bracket isn't
if call(foo, [a, b)
def bar():
pass


# The parser tries to recover from an unclosed `]` when the current token is `)`. This
# test is to make sure it emits a `NonLogicalNewline` token after `c`.
if call(foo, [a,
b
)
def bar():
pass
25 changes: 24 additions & 1 deletion crates/ruff_python_parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1319,7 +1319,8 @@ impl<'src> Lexer<'src> {
return false;
}

// Reduce the nesting level because the parser recovered from an error inside list parsing.
// Reduce the nesting level because the parser recovered from an error inside list parsing
// i.e., it recovered from an unclosed parenthesis (`(`, `[`, or `{`).
self.nesting -= 1;

let current_position = self.current_range().start();
Expand All @@ -1338,7 +1339,29 @@ impl<'src> Lexer<'src> {
}
}

// The lexer should only be moved if there's a newline character which needs to be
// re-lexed.
if new_position != current_position && has_newline {
// Earlier we reduced the nesting level unconditionally. Now that we know the lexer's
// position is going to be moved back, the lexer needs to be put back into a
// parenthesized context if the current token was a closing parenthesis.
//
// ```py
// (a, [b,
// c
// )
// ```
//
// Here, the parser would request to re-lex the token when it's at `)` and can recover
// from an unclosed `[`. This method will move the lexer back to the newline character
// after `c` which means it goes back into parenthesized context.
if matches!(
self.current_kind,
TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace
) {
self.nesting += 1;
}

self.cursor = Cursor::new(self.source);
self.cursor.skip_bytes(new_position.to_usize());
self.state = State::Other;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/invalid/re_lex_logical_token.py
```
Module(
ModModule {
range: 0..611,
range: 0..824,
body: [
If(
StmtIf {
Expand Down Expand Up @@ -418,6 +418,89 @@ Module(
elif_else_clauses: [],
},
),
If(
StmtIf {
range: 772..824,
test: Call(
ExprCall {
range: 775..796,
func: Name(
ExprName {
range: 775..779,
id: "call",
ctx: Load,
},
),
arguments: Arguments {
range: 779..796,
args: [
Name(
ExprName {
range: 780..783,
id: "foo",
ctx: Load,
},
),
List(
ExprList {
range: 785..794,
elts: [
Name(
ExprName {
range: 786..787,
id: "a",
ctx: Load,
},
),
Name(
ExprName {
range: 793..794,
id: "b",
ctx: Load,
},
),
],
ctx: Load,
},
),
],
keywords: [],
},
},
),
body: [
FunctionDef(
StmtFunctionDef {
range: 801..824,
is_async: false,
decorator_list: [],
name: Identifier {
id: "bar",
range: 805..808,
},
type_params: None,
parameters: Parameters {
range: 808..810,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Pass(
StmtPass {
range: 820..824,
},
),
],
},
),
],
elif_else_clauses: [],
},
),
],
},
)
Expand Down Expand Up @@ -496,3 +579,24 @@ Module(
36 | def bar():
37 | pass
|


|
41 | # test is to make sure it emits a `NonLogicalNewline` token after `c`.
42 | if call(foo, [a,
43 | b
| ^ Syntax Error: Expected ']', found NonLogicalNewline
44 | )
45 | def bar():
46 | pass
|


|
42 | if call(foo, [a,
43 | b
44 | )
| ^ Syntax Error: Expected ':', found newline
45 | def bar():
46 | pass
|

0 comments on commit 4e5ccde

Please sign in to comment.