[WIP] Detect several simple syntax errors in the parser

crates/red_knot_project/src/lib.rs

@@ -6,8 +6,9 @@ use files::{Index, Indexed, IndexedFiles};
 use metadata::settings::Settings;
 pub use metadata::{ProjectDiscoveryError, ProjectMetadata};
 use red_knot_python_semantic::lint::{LintRegistry, LintRegistryBuilder, RuleSelection};
-use red_knot_python_semantic::register_lints;
+use red_knot_python_semantic::syntax::SyntaxDiagnostic;
 use red_knot_python_semantic::types::check_types;
+use red_knot_python_semantic::{register_lints, Program};
 use ruff_db::diagnostic::{Diagnostic, DiagnosticId, ParseDiagnostic, Severity, Span};
 use ruff_db::files::{system_path_to_file, File};
 use ruff_db::parsed::parsed_module;
@@ -333,12 +334,20 @@ fn check_file_impl(db: &dyn Db, file: File) -> Vec<Box<dyn Diagnostic>> {
         return diagnostics;
     }
 
-    let parsed = parsed_module(db.upcast(), file);
+    let parsed = parsed_module(db.upcast(), file, Program::get(db).python_version(db));
     diagnostics.extend(parsed.errors().iter().map(|error| {
         let diagnostic: Box<dyn Diagnostic> = Box::new(ParseDiagnostic::new(file, error.clone()));
         diagnostic
     }));
 
+    if parsed.is_valid() {
+        diagnostics.extend(parsed.syntax_errors().iter().map(|error| {
+            let diagnostic: Box<dyn Diagnostic> =
+                Box::new(SyntaxDiagnostic::from_syntax_error(error, file));
+            diagnostic
+        }));
+    }
+
     diagnostics.extend(check_types(db.upcast(), file).iter().map(|diagnostic| {
         let boxed: Box<dyn Diagnostic> = Box::new(diagnostic.clone());
         boxed
@@ -477,19 +486,31 @@ mod tests {
     use crate::db::tests::TestDb;
     use crate::{check_file_impl, ProjectMetadata};
     use red_knot_python_semantic::types::check_types;
+    use red_knot_python_semantic::{Program, ProgramSettings, PythonPlatform, SearchPathSettings};
     use ruff_db::diagnostic::Diagnostic;
     use ruff_db::files::system_path_to_file;
     use ruff_db::source::source_text;
     use ruff_db::system::{DbWithTestSystem, SystemPath, SystemPathBuf};
     use ruff_db::testing::assert_function_query_was_not_run;
     use ruff_python_ast::name::Name;
+    use ruff_python_ast::PythonVersion;
 
     #[test]
     fn check_file_skips_type_checking_when_file_cant_be_read() -> ruff_db::system::Result<()> {
         let project = ProjectMetadata::new(Name::new_static("test"), SystemPathBuf::from("/"));
         let mut db = TestDb::new(project);
         let path = SystemPath::new("test.py");
 
+        Program::from_settings(
+            &db,
+            ProgramSettings {
+                python_version: PythonVersion::default(),
+                python_platform: PythonPlatform::default(),
+                search_paths: SearchPathSettings::new(vec![SystemPathBuf::from(".")]),
+            },
+        )
+        .expect("Failed to configure program settings");
+
         db.write_file(path, "x = 10")?;
         let file = system_path_to_file(&db, path).unwrap();
 

crates/red_knot_project/tests/check.rs

@@ -1,6 +1,6 @@
 use anyhow::{anyhow, Context};
 use red_knot_project::{ProjectDatabase, ProjectMetadata};
-use red_knot_python_semantic::{HasType, SemanticModel};
+use red_knot_python_semantic::{HasType, Program, SemanticModel};
 use ruff_db::files::{system_path_to_file, File};
 use ruff_db::parsed::parsed_module;
 use ruff_db::system::{SystemPath, SystemPathBuf, TestSystem};
@@ -165,7 +165,7 @@ fn run_corpus_tests(pattern: &str) -> anyhow::Result<()> {
 fn pull_types(db: &ProjectDatabase, file: File) {
     let mut visitor = PullTypesVisitor::new(db, file);
 
-    let ast = parsed_module(db, file);
+    let ast = parsed_module(db, file, Program::get(db).python_version(db));
 
     visitor.visit_body(ast.suite());
 }

crates/red_knot_python_semantic/src/lib.rs

@@ -24,6 +24,7 @@ mod semantic_model;
 pub(crate) mod site_packages;
 mod suppression;
 pub(crate) mod symbol;
+pub mod syntax;
 pub mod types;
 mod unpack;
 mod util;

crates/red_knot_python_semantic/src/semantic_index.rs

@@ -20,7 +20,7 @@ use crate::semantic_index::symbol::{
     FileScopeId, NodeWithScopeKey, NodeWithScopeRef, Scope, ScopeId, ScopedSymbolId, SymbolTable,
 };
 use crate::semantic_index::use_def::{EagerBindingsKey, ScopedEagerBindingsId, UseDefMap};
-use crate::Db;
+use crate::{Db, Program};
 
 pub mod ast_ids;
 pub mod attribute_assignment;
@@ -45,7 +45,7 @@ type SymbolMap = hashbrown::HashMap<ScopedSymbolId, (), FxBuildHasher>;
 pub(crate) fn semantic_index(db: &dyn Db, file: File) -> SemanticIndex<'_> {
     let _span = tracing::trace_span!("semantic_index", file = %file.path(db)).entered();
 
-    let parsed = parsed_module(db.upcast(), file);
+    let parsed = parsed_module(db.upcast(), file, Program::get(db).python_version(db));
 
     SemanticIndexBuilder::new(db, file, parsed).build()
 }
@@ -407,11 +407,10 @@ impl FusedIterator for ChildrenIter<'_> {}
 mod tests {
     use ruff_db::files::{system_path_to_file, File};
     use ruff_db::parsed::parsed_module;
-    use ruff_db::system::DbWithTestSystem;
-    use ruff_python_ast as ast;
+    use ruff_python_ast::{self as ast, PythonVersion};
     use ruff_text_size::{Ranged, TextRange};
 
-    use crate::db::tests::TestDb;
+    use crate::db::tests::{TestDb, TestDbBuilder};
     use crate::semantic_index::ast_ids::{HasScopedUseId, ScopedUseId};
     use crate::semantic_index::definition::{Definition, DefinitionKind};
     use crate::semantic_index::symbol::{
@@ -438,11 +437,15 @@ mod tests {
         file: File,
     }
 
-    fn test_case(content: impl ToString) -> TestCase {
-        let mut db = TestDb::new();
-        db.write_file("test.py", content).unwrap();
+    fn test_case(content: &str) -> TestCase {
+        const FILENAME: &str = "test.py";
 
-        let file = system_path_to_file(&db, "test.py").unwrap();
+        let db = TestDbBuilder::new()
+            .with_file(FILENAME, content)
+            .build()
+            .unwrap();
+
+        let file = system_path_to_file(&db, FILENAME).unwrap();
 
         TestCase { db, file }
     }
@@ -829,7 +832,7 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs):
 
         let use_def = index.use_def_map(comprehension_scope_id);
 
-        let module = parsed_module(&db, file).syntax();
+        let module = parsed_module(&db, file, PythonVersion::default()).syntax();
         let element = module.body[0]
             .as_expr_stmt()
             .unwrap()
@@ -1078,7 +1081,7 @@ class C[T]:
     #[test]
     fn reachability_trivial() {
         let TestCase { db, file } = test_case("x = 1; x");
-        let parsed = parsed_module(&db, file);
+        let parsed = parsed_module(&db, file, PythonVersion::default());
         let scope = global_scope(&db, file);
         let ast = parsed.syntax();
         let ast::Stmt::Expr(ast::StmtExpr {
@@ -1111,7 +1114,7 @@ class C[T]:
         let TestCase { db, file } = test_case("x = 1;\ndef test():\n  y = 4");
 
         let index = semantic_index(&db, file);
-        let parsed = parsed_module(&db, file);
+        let parsed = parsed_module(&db, file, PythonVersion::default());
         let ast = parsed.syntax();
 
         let x_stmt = ast.body[0].as_assign_stmt().unwrap();

crates/red_knot_python_semantic/src/semantic_model.rs

@@ -166,6 +166,7 @@ impl_binding_has_ty!(ast::ParameterWithDefault);
 mod tests {
     use ruff_db::files::system_path_to_file;
     use ruff_db::parsed::parsed_module;
+    use ruff_python_ast::PythonVersion;
 
     use crate::db::tests::TestDbBuilder;
     use crate::{HasType, SemanticModel};
@@ -178,7 +179,7 @@ mod tests {
 
         let foo = system_path_to_file(&db, "/src/foo.py").unwrap();
 
-        let ast = parsed_module(&db, foo);
+        let ast = parsed_module(&db, foo, PythonVersion::default());
 
         let function = ast.suite()[0].as_function_def_stmt().unwrap();
         let model = SemanticModel::new(&db, foo);
@@ -197,7 +198,7 @@ mod tests {
 
         let foo = system_path_to_file(&db, "/src/foo.py").unwrap();
 
-        let ast = parsed_module(&db, foo);
+        let ast = parsed_module(&db, foo, PythonVersion::default());
 
         let class = ast.suite()[0].as_class_def_stmt().unwrap();
         let model = SemanticModel::new(&db, foo);
@@ -217,7 +218,7 @@ mod tests {
 
         let bar = system_path_to_file(&db, "/src/bar.py").unwrap();
 
-        let ast = parsed_module(&db, bar);
+        let ast = parsed_module(&db, bar, PythonVersion::default());
 
         let import = ast.suite()[0].as_import_from_stmt().unwrap();
         let alias = &import.names[0];

crates/red_knot_python_semantic/src/suppression.rs

@@ -1,5 +1,6 @@
 use crate::lint::{GetLintError, Level, LintMetadata, LintRegistry, LintStatus};
 use crate::types::{TypeCheckDiagnostic, TypeCheckDiagnostics};
+use crate::Program;
 use crate::{declare_lint, lint::LintId, Db};
 use ruff_db::diagnostic::DiagnosticId;
 use ruff_db::{files::File, parsed::parsed_module, source::source_text};
@@ -88,7 +89,7 @@ declare_lint! {
 
 #[salsa::tracked(return_ref)]
 pub(crate) fn suppressions(db: &dyn Db, file: File) -> Suppressions {
-    let parsed = parsed_module(db.upcast(), file);
+    let parsed = parsed_module(db.upcast(), file, Program::get(db).python_version(db));
     let source = source_text(db.upcast(), file);
 
     let mut builder = SuppressionsBuilder::new(&source, db.lint_registry());

crates/red_knot_python_semantic/src/syntax.rs

@@ -0,0 +1,43 @@
+use std::borrow::Cow;
+
+use ruff_db::diagnostic::{Diagnostic, DiagnosticId, Severity, Span};
+use ruff_db::files::File;
+use ruff_python_parser::SyntaxError;
+use ruff_text_size::TextRange;
+
+#[derive(Debug, Eq, PartialEq, Clone)]
+pub struct SyntaxDiagnostic {
+    id: DiagnosticId,
+    message: String,
+    file: File,
+    range: TextRange,
+}
+
+impl Diagnostic for SyntaxDiagnostic {
+    fn id(&self) -> ruff_db::diagnostic::DiagnosticId {
+        self.id
+    }
+
+    fn message(&self) -> Cow<str> {
+        Cow::from(&self.message)
+    }
+
+    fn severity(&self) -> ruff_db::diagnostic::Severity {
+        Severity::Error
+    }
+
+    fn span(&self) -> Option<ruff_db::diagnostic::Span> {
+        Some(Span::from(self.file).with_range(self.range))
+    }
+}
+
+impl SyntaxDiagnostic {
+    pub fn from_syntax_error(value: &SyntaxError, file: File) -> Self {
+        Self {
+            id: DiagnosticId::invalid_syntax(Some(value.kind.as_str())),
+            message: value.message(),
+            file,
+            range: value.range,
+        }
+    }
+}

crates/red_knot_python_semantic/src/types.rs

@@ -5445,7 +5445,9 @@ pub(crate) mod tests {
         );
         let events = db.take_salsa_events();
 
-        let call = &*parsed_module(&db, bar).syntax().body[1]
+        let call = &*parsed_module(&db, bar, Program::get(&db).python_version(&db))
+            .syntax()
+            .body[1]
             .as_assign_stmt()
             .unwrap()
             .value;

crates/red_knot_python_semantic/src/types/infer.rs

@@ -77,7 +77,7 @@ use crate::types::{
 };
 use crate::unpack::Unpack;
 use crate::util::subscript::{PyIndex, PySlice};
-use crate::Db;
+use crate::{Db, Program};
 
 use super::call::CallError;
 use super::context::{InNoTypeCheck, InferContext, WithDiagnostics};
@@ -538,10 +538,15 @@ impl<'db> TypeInferenceBuilder<'db> {
     }
 
     fn infer_region_scope(&mut self, scope: ScopeId<'db>) {
-        let node = scope.node(self.db());
+        let db = self.db();
+        let node = scope.node(db);
         match node {
             NodeWithScopeKind::Module => {
-                let parsed = parsed_module(self.db().upcast(), self.file());
+                let parsed = parsed_module(
+                    db.upcast(),
+                    self.file(),
+                    Program::get(db).python_version(db),
+                );
                 self.infer_module(parsed.syntax());
             }
             NodeWithScopeKind::Function(function) => self.infer_function_body(function.node()),
@@ -576,7 +581,7 @@ impl<'db> TypeInferenceBuilder<'db> {
         // Infer the deferred types for the definitions here to consider the end-of-scope
         // semantics.
         for definition in std::mem::take(&mut self.types.deferred) {
-            self.extend(infer_deferred_types(self.db(), definition));
+            self.extend(infer_deferred_types(db, definition));
         }
         assert!(
             self.types.deferred.is_empty(),
@@ -6394,6 +6399,7 @@ mod tests {
     use ruff_db::files::{system_path_to_file, File};
     use ruff_db::system::DbWithTestSystem;
     use ruff_db::testing::{assert_function_query_was_not_run, assert_function_query_was_run};
+    use ruff_python_ast::PythonVersion;
 
     use super::*;
 
@@ -6725,7 +6731,7 @@ mod tests {
     fn dependency_implicit_instance_attribute() -> anyhow::Result<()> {
         fn x_rhs_expression(db: &TestDb) -> Expression<'_> {
             let file_main = system_path_to_file(db, "/src/main.py").unwrap();
-            let ast = parsed_module(db, file_main);
+            let ast = parsed_module(db, file_main, PythonVersion::default());
             // Get the second statement in `main.py` (x = …) and extract the expression
             // node on the right-hand side:
             let x_rhs_node = &ast.syntax().body[1].as_assign_stmt().unwrap().value;
@@ -6808,7 +6814,7 @@ mod tests {
     fn dependency_own_instance_member() -> anyhow::Result<()> {
         fn x_rhs_expression(db: &TestDb) -> Expression<'_> {
             let file_main = system_path_to_file(db, "/src/main.py").unwrap();
-            let ast = parsed_module(db, file_main);
+            let ast = parsed_module(db, file_main, PythonVersion::default());
             // Get the second statement in `main.py` (x = …) and extract the expression
             // node on the right-hand side:
             let x_rhs_node = &ast.syntax().body[1].as_assign_stmt().unwrap().value;

crates/red_knot_test/src/assertion.rs

@@ -35,6 +35,7 @@
 //! ```
 
 use crate::db::Db;
+use red_knot_python_semantic::Program;
 use regex::Regex;
 use ruff_db::files::File;
 use ruff_db::parsed::parsed_module;
@@ -58,7 +59,7 @@ impl InlineFileAssertions {
     pub(crate) fn from_file(db: &Db, file: File) -> Self {
         let source = source_text(db, file);
         let lines = line_index(db, file);
-        let parsed = parsed_module(db, file);
+        let parsed = parsed_module(db, file, Program::get(db).python_version(db));
         let comment_ranges = CommentRanges::from(parsed.tokens());
         Self {
             comment_ranges,

crates/red_knot_test/src/lib.rs

@@ -193,7 +193,7 @@ fn run_test(
     let failures: Failures = test_files
         .into_iter()
         .filter_map(|test_file| {
-            let parsed = parsed_module(db, test_file.file);
+            let parsed = parsed_module(db, test_file.file, Program::get(db).python_version(db));
 
             let mut diagnostics: Vec<Box<_>> = parsed
                 .errors()

crates/red_knot_wasm/Cargo.toml

@@ -20,6 +20,7 @@ default = ["console_error_panic_hook"]
 
 [dependencies]
 red_knot_project = { workspace = true, default-features = false, features = ["deflate"] }
+red_knot_python_semantic = { workspace = true }
 
 ruff_db = { workspace = true, default-features = false, features = [] }
 ruff_python_ast = { workspace = true }