diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 5e6b5b3160633d..4f419c04eb1df4 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -1307,6 +1307,47 @@ impl<'src> Lexer<'src> { } } + /// Re-lex the current token in the context of a logical line. + /// + /// Returns a boolean indicating that whether the new current token is different than the + /// previous current token. + /// + /// This method is a no-op if the lexer isn't in a parenthesized context. + pub(crate) fn re_lex_logical_token(&mut self) -> bool { + if self.nesting == 0 { + return false; + } + + // Reduce the nesting level because the parser recovered from an error inside list parsing. + self.nesting -= 1; + + let current_position = self.current_range().start(); + let reverse_chars = self.source[..current_position.to_usize()].chars().rev(); + let mut new_position = current_position; + let mut has_newline = false; + + for ch in reverse_chars { + if is_python_whitespace(ch) { + new_position -= ch.text_len(); + } else if matches!(ch, '\n' | '\r') { + has_newline |= true; + new_position -= ch.text_len(); + } else { + break; + } + } + + if new_position != current_position && has_newline { + self.cursor = Cursor::new(self.source); + self.cursor.skip_bytes(new_position.to_usize()); + self.state = State::Other; + self.next_token(); + true + } else { + false + } + } + #[inline] fn token_range(&self) -> TextRange { let end = self.offset(); diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index d113ff992f7b3e..6f9a956cc5c6f0 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -558,7 +558,10 @@ impl<'src> Parser<'src> { } self.expect(TokenKind::Comma); - } else if recovery_context_kind.is_list_terminator(self) { + } else if recovery_context_kind + .list_terminator_kind(self) + .is_some_and(ListTerminatorKind::is_regular) + { break; } else { // Not a recognised element. Add an error and either skip the token or break @@ -570,6 +573,7 @@ impl<'src> Parser<'src> { // Run the error recovery: This also handles the case when an element is missing // between two commas: `a,,b` if self.is_enclosing_list_element_or_terminator() { + self.tokens.re_lex_logical_token(); break; } @@ -786,6 +790,12 @@ enum ListTerminatorKind { ErrorRecovery, } +impl ListTerminatorKind { + const fn is_regular(self) -> bool { + matches!(self, ListTerminatorKind::Regular) + } +} + #[derive(Copy, Clone, Debug)] enum RecoveryContextKind { /// When parsing a list of statements at the module level i.e., at the top level of a file. diff --git a/crates/ruff_python_parser/src/token_source.rs b/crates/ruff_python_parser/src/token_source.rs index a8a54e68f02c3b..17290b2f58c9f9 100644 --- a/crates/ruff_python_parser/src/token_source.rs +++ b/crates/ruff_python_parser/src/token_source.rs @@ -1,4 +1,4 @@ -use ruff_text_size::{TextRange, TextSize}; +use ruff_text_size::{Ranged, TextRange, TextSize}; use crate::lexer::{Lexer, LexerCheckpoint, LexicalError, Token, TokenFlags, TokenValue}; use crate::{Mode, TokenKind}; @@ -58,6 +58,23 @@ impl<'src> TokenSource<'src> { self.lexer.take_value() } + /// Calls the underying [`re_lex_logical_token`] method on the lexer and updates the token + /// vector accordingly. + /// + /// [`re_lex_logical_token`]: Lexer::re_lex_logical_token + pub(crate) fn re_lex_logical_token(&mut self) { + if self.lexer.re_lex_logical_token() { + let current_start = self.current_range().start(); + while self + .tokens + .last() + .is_some_and(|last| last.start() >= current_start) + { + self.tokens.pop(); + } + } + } + /// Returns the next non-trivia token without consuming it. /// /// Use [`peek2`] to get the next two tokens. diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_0.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_0.py.snap index 263334f753867a..938511f42d77de 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_0.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_0.py.snap @@ -75,7 +75,9 @@ Module( | +1 | call( + | ^ Syntax Error: Expected ')', found newline +2 | 3 | def foo(): 4 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_1.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_1.py.snap index e4e21a03bd4ad3..0a7e30184b2d2f 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_1.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_1.py.snap @@ -83,7 +83,9 @@ Module( | +1 | call(x + | ^ Syntax Error: Expected ')', found newline +2 | 3 | def foo(): 4 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_2.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_2.py.snap index f7ca1d97c7c78a..b394c9e948bddf 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_2.py.snap @@ -83,7 +83,9 @@ Module( | +1 | call(x, + | ^ Syntax Error: Expected ')', found newline +2 | 3 | def foo(): 4 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__missing_closing_brace_2.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__missing_closing_brace_2.py.snap index d60ca66d0a46f8..af465ac160437b 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__missing_closing_brace_2.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__missing_closing_brace_2.py.snap @@ -84,7 +84,9 @@ Module( | +1 | {x: 1, + | ^ Syntax Error: Expected '}', found newline +2 | 3 | def foo(): 4 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__missing_closing_bracket_3.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__missing_closing_bracket_3.py.snap index 3a0898a73853c3..a84426e59858ee 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__missing_closing_bracket_3.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__missing_closing_bracket_3.py.snap @@ -82,7 +82,11 @@ Module( | +2 | # token starts a statement. +3 | +4 | [1, 2 + | ^ Syntax Error: Expected ']', found newline +5 | 6 | def foo(): 7 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__missing_closing_paren_3.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__missing_closing_paren_3.py.snap index 4cd851ef5ce615..f487fb9df8c3f6 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__missing_closing_paren_3.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__missing_closing_paren_3.py.snap @@ -83,7 +83,11 @@ Module( | +2 | # token starts a statement. +3 | +4 | (1, 2 + | ^ Syntax Error: Expected ')', found newline +5 | 6 | def foo(): 7 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__missing_closing_curly_brace_3.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__missing_closing_curly_brace_3.py.snap index 128d499a9a74b8..3851df90fb0e68 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__missing_closing_curly_brace_3.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__missing_closing_curly_brace_3.py.snap @@ -81,7 +81,11 @@ Module( | +2 | # token starts a statement. +3 | +4 | {1, 2 + | ^ Syntax Error: Expected '}', found newline +5 | 6 | def foo(): 7 | pass - | Syntax Error: unexpected EOF while parsing | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap index 14302871b26dc5..66574d560f7fb5 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap @@ -140,9 +140,9 @@ Module( | 1 | from x import (a, b + | ^ Syntax Error: Expected ')', found newline 2 | 1 + 1 3 | from x import (a, b, - | ^^^^ Syntax Error: Simple statements must be separated by newlines or semicolons 4 | 2 + 2 | @@ -156,6 +156,9 @@ Module( | +1 | from x import (a, b +2 | 1 + 1 3 | from x import (a, b, + | ^ Syntax Error: Expected ')', found newline 4 | 2 + 2 | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@node_range_with_gaps.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@node_range_with_gaps.py.snap index 1456d1a7a8ede6..ceb160cfe38da2 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@node_range_with_gaps.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@node_range_with_gaps.py.snap @@ -103,7 +103,7 @@ Module( | 1 | def foo # comment 2 | def bar(): ... - | ^^^ Syntax Error: Expected ')', found 'def' + | ^^^ Syntax Error: Expected a parameter or the end of the parameter list 3 | def baz | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@with_items_parenthesized_missing_comma.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@with_items_parenthesized_missing_comma.py.snap index 68009deba08375..9e56d6c30648e0 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@with_items_parenthesized_missing_comma.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@with_items_parenthesized_missing_comma.py.snap @@ -332,9 +332,3 @@ Module( 5 | with (item1, item2: ... | ^ Syntax Error: Expected ',', found ':' | - - - | -4 | with (item1, item2 as f1 item3, item4): ... -5 | with (item1, item2: ... - |