Start detecting version-related syntax errors in the parser (#16090)

## Summary This PR builds on the changes in #16220 to pass a target Python version to the parser. It also adds the `Parser::unsupported_syntax_errors` field, which collects version-related syntax errors while parsing. These syntax errors are then turned into `Message`s in ruff (in preview mode). This PR only detects one syntax error (`match` statement before Python 3.10), but it has been pretty quick to extend to several other simple errors (see #16308 for example). ## Test Plan The current tests are CLI tests in the linter crate, but these could be supplemented with inline parser tests after #16357. I also tested the display of these syntax errors in VS Code: ![image](https://github.com/user-attachments/assets/062b4441-740e-46c3-887c-a954049ef26e) ![image](https://github.com/user-attachments/assets/101f55b8-146c-4d59-b6b0-922f19bcd0fa) --------- Co-authored-by: Alex Waygood <[email protected]>
astral-sh · Feb 26, 2025 · 7880636 · 7880636
1 parent b39a4ad
commit 7880636
Show file tree

Hide file tree

Showing 14 changed files with 356 additions and 37 deletions.
diff --git a/crates/ruff/src/cache.rs b/crates/ruff/src/cache.rs
@@ -586,14 +586,15 @@ mod tests {
     use anyhow::Result;
     use filetime::{set_file_mtime, FileTime};
     use itertools::Itertools;
+    use ruff_linter::settings::LinterSettings;
     use test_case::test_case;
 
     use ruff_cache::CACHE_DIR_NAME;
     use ruff_linter::message::Message;
     use ruff_linter::package::PackageRoot;
     use ruff_linter::settings::flags;
     use ruff_linter::settings::types::UnsafeFixes;
-    use ruff_python_ast::PySourceType;
+    use ruff_python_ast::{PySourceType, PythonVersion};
     use ruff_workspace::Settings;
 
     use crate::cache::{self, FileCache, FileCacheData, FileCacheKey};
@@ -611,6 +612,10 @@ mod tests {
 
         let settings = Settings {
             cache_dir,
+            linter: LinterSettings {
+                unresolved_target_version: PythonVersion::PY310,
+                ..Default::default()
+            },
             ..Settings::default()
         };
 

diff --git a/crates/ruff/tests/lint.rs b/crates/ruff/tests/lint.rs
@@ -2627,3 +2627,77 @@ class A(Generic[T]):
     "
     );
 }
+
+#[test]
+fn match_before_py310() {
+    // ok on 3.10
+    assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+        .args(STDIN_BASE_OPTIONS)
+        .args(["--stdin-filename", "test.py"])
+        .arg("--target-version=py310")
+        .arg("-")
+        .pass_stdin(
+            r#"
+match 2:
+    case 1:
+        print("it's one")
+"#
+        ),
+        @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    All checks passed!
+
+    ----- stderr -----
+    "
+    );
+
+    // ok on 3.9 without preview
+    assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+        .args(STDIN_BASE_OPTIONS)
+        .args(["--stdin-filename", "test.py"])
+        .arg("--target-version=py39")
+        .arg("-")
+        .pass_stdin(
+            r#"
+match 2:
+    case 1:
+        print("it's one")
+"#
+        ),
+        @r"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    All checks passed!
+
+    ----- stderr -----
+    "
+    );
+
+    // syntax error on 3.9 with preview
+    assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+        .args(STDIN_BASE_OPTIONS)
+        .args(["--stdin-filename", "test.py"])
+        .arg("--target-version=py39")
+        .arg("--preview")
+        .arg("-")
+        .pass_stdin(
+            r#"
+match 2:
+    case 1:
+        print("it's one")
+"#
+        ),
+        @r"
+    success: false
+    exit_code: 1
+    ----- stdout -----
+    test.py:2:1: SyntaxError: Cannot use `match` statement on Python 3.9 (syntax was added in Python 3.10)
+    Found 1 error.
+
+    ----- stderr -----
+    "
+    );
+}
diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs
@@ -10,10 +10,10 @@ use rustc_hash::FxHashMap;
 
 use ruff_diagnostics::Diagnostic;
 use ruff_notebook::Notebook;
-use ruff_python_ast::{ModModule, PySourceType};
+use ruff_python_ast::{ModModule, PySourceType, PythonVersion};
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::{ParseError, Parsed};
+use ruff_python_parser::{ParseError, ParseOptions, Parsed, UnsupportedSyntaxError};
 use ruff_source_file::SourceFileBuilder;
 use ruff_text_size::Ranged;
 
@@ -71,6 +71,7 @@ pub fn check_path(
     source_kind: &SourceKind,
     source_type: PySourceType,
     parsed: &Parsed<ModModule>,
+    target_version: PythonVersion,
 ) -> Vec<Diagnostic> {
     // Aggregate all diagnostics.
     let mut diagnostics = vec![];
@@ -104,8 +105,6 @@ pub fn check_path(
         ));
     }
 
-    let target_version = settings.resolve_target_version(path);
-
     // Run the filesystem-based rules.
     if settings
         .rules
@@ -335,7 +334,8 @@ pub fn add_noqa_to_path(
     settings: &LinterSettings,
 ) -> Result<usize> {
     // Parse once.
-    let parsed = ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type);
+    let target_version = settings.resolve_target_version(path);
+    let parsed = parse_unchecked_source(source_kind, source_type, target_version);
 
     // Map row and column locations to byte slices (lazily).
     let locator = Locator::new(source_kind.source_code());
@@ -367,6 +367,7 @@ pub fn add_noqa_to_path(
         source_kind,
         source_type,
         &parsed,
+        target_version,
     );
 
     // Add any missing `# noqa` pragmas.
@@ -393,7 +394,8 @@ pub fn lint_only(
     source_type: PySourceType,
     source: ParseSource,
 ) -> LinterResult {
-    let parsed = source.into_parsed(source_kind, source_type);
+    let target_version = settings.resolve_target_version(path);
+    let parsed = source.into_parsed(source_kind, source_type, target_version);
 
     // Map row and column locations to byte slices (lazily).
     let locator = Locator::new(source_kind.source_code());
@@ -425,12 +427,20 @@ pub fn lint_only(
         source_kind,
         source_type,
         &parsed,
+        target_version,
     );
 
+    let syntax_errors = if settings.preview.is_enabled() {
+        parsed.unsupported_syntax_errors()
+    } else {
+        &[]
+    };
+
     LinterResult {
         messages: diagnostics_to_messages(
             diagnostics,
             parsed.errors(),
+            syntax_errors,
             path,
             &locator,
             &directives,
@@ -443,6 +453,7 @@ pub fn lint_only(
 fn diagnostics_to_messages(
     diagnostics: Vec<Diagnostic>,
     parse_errors: &[ParseError],
+    unsupported_syntax_errors: &[UnsupportedSyntaxError],
     path: &Path,
     locator: &Locator,
     directives: &Directives,
@@ -461,6 +472,9 @@ fn diagnostics_to_messages(
     parse_errors
         .iter()
         .map(|parse_error| Message::from_parse_error(parse_error, locator, file.deref().clone()))
+        .chain(unsupported_syntax_errors.iter().map(|syntax_error| {
+            Message::from_unsupported_syntax_error(syntax_error, file.deref().clone())
+        }))
         .chain(diagnostics.into_iter().map(|diagnostic| {
             let noqa_offset = directives.noqa_line_for.resolve(diagnostic.start());
             Message::from_diagnostic(diagnostic, file.deref().clone(), noqa_offset)
@@ -491,11 +505,12 @@ pub fn lint_fix<'a>(
     // Track whether the _initial_ source code is valid syntax.
     let mut is_valid_syntax = false;
 
+    let target_version = settings.resolve_target_version(path);
+
     // Continuously fix until the source code stabilizes.
     loop {
         // Parse once.
-        let parsed =
-            ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type);
+        let parsed = parse_unchecked_source(&transformed, source_type, target_version);
 
         // Map row and column locations to byte slices (lazily).
         let locator = Locator::new(transformed.source_code());
@@ -527,6 +542,7 @@ pub fn lint_fix<'a>(
             &transformed,
             source_type,
             &parsed,
+            target_version,
         );
 
         if iterations == 0 {
@@ -573,11 +589,18 @@ pub fn lint_fix<'a>(
             report_failed_to_converge_error(path, transformed.source_code(), &diagnostics);
         }
 
+        let syntax_errors = if settings.preview.is_enabled() {
+            parsed.unsupported_syntax_errors()
+        } else {
+            &[]
+        };
+
         return Ok(FixerResult {
             result: LinterResult {
                 messages: diagnostics_to_messages(
                     diagnostics,
                     parsed.errors(),
+                    syntax_errors,
                     path,
                     &locator,
                     &directives,
@@ -680,16 +703,35 @@ pub enum ParseSource {
 impl ParseSource {
     /// Consumes the [`ParseSource`] and returns the parsed [`Parsed`], parsing the source code if
     /// necessary.
-    fn into_parsed(self, source_kind: &SourceKind, source_type: PySourceType) -> Parsed<ModModule> {
+    fn into_parsed(
+        self,
+        source_kind: &SourceKind,
+        source_type: PySourceType,
+        target_version: PythonVersion,
+    ) -> Parsed<ModModule> {
         match self {
-            ParseSource::None => {
-                ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type)
-            }
+            ParseSource::None => parse_unchecked_source(source_kind, source_type, target_version),
             ParseSource::Precomputed(parsed) => parsed,
         }
     }
 }
 
+/// Like [`ruff_python_parser::parse_unchecked_source`] but with an additional [`PythonVersion`]
+/// argument.
+fn parse_unchecked_source(
+    source_kind: &SourceKind,
+    source_type: PySourceType,
+    target_version: PythonVersion,
+) -> Parsed<ModModule> {
+    let options = ParseOptions::from(source_type).with_target_version(target_version);
+    // SAFETY: Safe because `PySourceType` always parses to a `ModModule`. See
+    // `ruff_python_parser::parse_unchecked_source`. We use `parse_unchecked` (and thus
+    // have to unwrap) in order to pass the `PythonVersion` via `ParseOptions`.
+    ruff_python_parser::parse_unchecked(source_kind.source_code(), options)
+        .try_into_module()
+        .expect("PySourceType always parses into a module")
+}
+
 #[cfg(test)]
 mod tests {
     use std::path::Path;

diff --git a/crates/ruff_linter/src/message/mod.rs b/crates/ruff_linter/src/message/mod.rs
@@ -16,7 +16,7 @@ pub use pylint::PylintEmitter;
 pub use rdjson::RdjsonEmitter;
 use ruff_diagnostics::{Diagnostic, DiagnosticKind, Fix};
 use ruff_notebook::NotebookIndex;
-use ruff_python_parser::ParseError;
+use ruff_python_parser::{ParseError, UnsupportedSyntaxError};
 use ruff_source_file::{SourceFile, SourceLocation};
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 pub use sarif::SarifEmitter;
@@ -121,6 +121,18 @@ impl Message {
         })
     }
 
+    /// Create a [`Message`] from the given [`UnsupportedSyntaxError`].
+    pub fn from_unsupported_syntax_error(
+        unsupported_syntax_error: &UnsupportedSyntaxError,
+        file: SourceFile,
+    ) -> Message {
+        Message::SyntaxError(SyntaxErrorMessage {
+            message: format!("SyntaxError: {unsupported_syntax_error}"),
+            range: unsupported_syntax_error.range,
+            file,
+        })
+    }
+
     pub const fn as_diagnostic_message(&self) -> Option<&DiagnosticMessage> {
         match self {
             Message::Diagnostic(m) => Some(m),

diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs
@@ -11,6 +11,7 @@ mod tests {
 
     use anyhow::Result;
     use regex::Regex;
+    use ruff_python_parser::ParseOptions;
     use rustc_hash::FxHashMap;
     use test_case::test_case;
 
@@ -744,8 +745,11 @@ mod tests {
         let source_type = PySourceType::default();
         let source_kind = SourceKind::Python(contents.to_string());
         let settings = LinterSettings::for_rules(Linter::Pyflakes.rules());
-        let parsed =
-            ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type);
+        let options =
+            ParseOptions::from(source_type).with_target_version(settings.unresolved_target_version);
+        let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options)
+            .try_into_module()
+            .expect("PySourceType always parses into a module");
         let locator = Locator::new(&contents);
         let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
         let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());
@@ -767,6 +771,7 @@ mod tests {
             &source_kind,
             source_type,
             &parsed,
+            settings.unresolved_target_version,
         );
         diagnostics.sort_by_key(Ranged::start);
         let actual = diagnostics

diff --git a/crates/ruff_linter/src/test.rs b/crates/ruff_linter/src/test.rs
@@ -16,7 +16,7 @@ use ruff_notebook::NotebookError;
 use ruff_python_ast::PySourceType;
 use ruff_python_codegen::Stylist;
 use ruff_python_index::Indexer;
-use ruff_python_parser::ParseError;
+use ruff_python_parser::{ParseError, ParseOptions};
 use ruff_python_trivia::textwrap::dedent;
 use ruff_source_file::SourceFileBuilder;
 use ruff_text_size::Ranged;
@@ -110,7 +110,11 @@ pub(crate) fn test_contents<'a>(
     settings: &LinterSettings,
 ) -> (Vec<Message>, Cow<'a, SourceKind>) {
     let source_type = PySourceType::from(path);
-    let parsed = ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type);
+    let target_version = settings.resolve_target_version(path);
+    let options = ParseOptions::from(source_type).with_target_version(target_version);
+    let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options.clone())
+        .try_into_module()
+        .expect("PySourceType always parses into a module");
     let locator = Locator::new(source_kind.source_code());
     let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
     let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());
@@ -134,6 +138,7 @@ pub(crate) fn test_contents<'a>(
         source_kind,
         source_type,
         &parsed,
+        target_version,
     );
 
     let source_has_errors = !parsed.is_valid();
@@ -174,7 +179,9 @@ pub(crate) fn test_contents<'a>(
             transformed = Cow::Owned(transformed.updated(fixed_contents, &source_map));
 
             let parsed =
-                ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type);
+                ruff_python_parser::parse_unchecked(transformed.source_code(), options.clone())
+                    .try_into_module()
+                    .expect("PySourceType always parses into a module");
             let locator = Locator::new(transformed.source_code());
             let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents());
             let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents());
@@ -197,6 +204,7 @@ pub(crate) fn test_contents<'a>(
                 &transformed,
                 source_type,
                 &parsed,
+                target_version,
             );
 
             if !parsed.is_valid() && !source_has_errors {