Skip to content

Commit

Permalink
baseline cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@KotlinIsland
KotlinIsland committed Jan 1, 2025
1 parent f24781b commit c1cbfa3
Show file tree
Hide file tree
Showing 7 changed files with 151 additions and 66 deletions.
54 changes: 39 additions & 15 deletions .mypy/baseline.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
"code": "no-any-explicit",
"column": 0,
"message": "Explicit \"Any\" is not allowed",
"offset": 14,
"offset": 15,
"src": "def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta:",
"target": "mypy.build.cache_meta_from_dict"
},
Expand Down Expand Up @@ -463,6 +463,22 @@
"src": "meta.get(\"plugin_data\", None),",
"target": "mypy.build.cache_meta_from_dict"
},
{
"code": "no-any-expr",
"column": 8,
"message": "Expression type contains \"Any\" (has type \"dict[str, Any]\")",
"offset": 1,
"src": "meta.get(\"baseline_hash\", None),",
"target": "mypy.build.cache_meta_from_dict"
},
{
"code": "no-any-expr",
"column": 8,
"message": "Expression has type \"Any\"",
"offset": 0,
"src": "meta.get(\"baseline_hash\", None),",
"target": "mypy.build.cache_meta_from_dict"
},
{
"code": "no-any-expr",
"column": 26,
Expand Down Expand Up @@ -1343,11 +1359,19 @@
"src": "if m.plugin_data != plugin_data:",
"target": "mypy.build.find_cache_meta"
},
{
"code": "no-any-expr",
"column": 24,
"message": "Expression type contains \"Any\" (has type \"CacheMeta\")",
"offset": 17,
"src": "if baseline_hash != m.baseline_hash:",
"target": "mypy.build.find_cache_meta"
},
{
"code": "no-any-expr",
"column": 11,
"message": "Expression type contains \"Any\" (has type \"CacheMeta\")",
"offset": 5,
"offset": 7,
"src": "return m",
"target": "mypy.build.find_cache_meta"
},
Expand Down Expand Up @@ -1595,15 +1619,15 @@
"code": "no-any-expr",
"column": 18,
"message": "Expression has type \"Any\"",
"offset": 55,
"offset": 56,
"src": "plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False))",
"target": "mypy.build.write_cache"
},
{
"code": "no-any-expr",
"column": 11,
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any]\")",
"offset": 51,
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any | None]\")",
"offset": 64,
"src": "meta = {",
"target": "mypy.build.write_cache"
},
Expand All @@ -1626,8 +1650,8 @@
{
"code": "no-any-expr",
"column": 26,
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any]\")",
"offset": 4,
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any | None]\")",
"offset": 5,
"src": "meta_str = json_dumps(meta, manager.options.debug_cache)",
"target": "mypy.build.write_cache"
},
Expand All @@ -1650,7 +1674,7 @@
{
"code": "no-any-expr",
"column": 48,
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any]\")",
"message": "Expression type contains \"Any\" (has type \"dict[str, str | int | list[str] | Mapping[str, object] | list[int] | Any | None]\")",
"offset": 0,
"src": "return interface_hash, cache_meta_from_dict(meta, data_json)",
"target": "mypy.build.write_cache"
Expand Down Expand Up @@ -1875,7 +1899,7 @@
"code": "no-any-expr",
"column": 15,
"message": "Expression type contains \"Any\" (has type \"CacheMeta | None\")",
"offset": 79,
"offset": 80,
"src": "if self.meta:",
"target": "mypy.build.State.generate_unused_ignore_notes"
},
Expand Down Expand Up @@ -2887,7 +2911,7 @@
"code": "explicit-override",
"column": 4,
"message": "Method \"visit_type_alias_expr\" is not using @override but is overriding a method in class \"mypy.visitor.ExpressionVisitor\"",
"offset": 46,
"offset": 47,
"src": "def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type:",
"target": "mypy.checkexpr.ExpressionChecker.visit_type_alias_expr"
},
Expand Down Expand Up @@ -6811,7 +6835,7 @@
"code": "truthy-bool",
"column": 11,
"message": "Member \"options\" has type \"Options\" which does not implement __bool__ or __len__ so it could always be true in boolean context",
"offset": 518,
"offset": 516,
"src": "if self.options:",
"target": "mypy.errors.Errors.is_error_code_enabled"
},
Expand All @@ -6835,7 +6859,7 @@
"code": "redundant-expr",
"column": 19,
"message": "Condition is always false",
"offset": 371,
"offset": 375,
"src": "if e.type is None:",
"target": "mypy.errors.Errors.render_messages"
},
Expand Down Expand Up @@ -10735,7 +10759,7 @@
"code": "redundant-expr",
"column": 18,
"message": "Condition is always true",
"offset": 105,
"offset": 104,
"src": "while True:",
"target": "mypy.ipc.IPCBase.read"
}
Expand Down Expand Up @@ -11341,7 +11365,7 @@
"code": "helpful-string",
"column": 12,
"message": "The string for \"None\" isn't helpful in a user-facing or semantic string",
"offset": 235,
"offset": 239,
"src": "\"Warning: unused section(s) in {}: {}\".format(",
"target": "mypy.main.run_build"
},
Expand Down Expand Up @@ -36453,7 +36477,7 @@
"code": "no-any-expr",
"column": 18,
"message": "Expression type contains \"Any\" (has type \"list[(str, Any)]\")",
"offset": 365,
"offset": 364,
"src": "members = inspect.getmembers(",
"target": "mypy.util.get_class_descriptors"
},
Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
## [Unreleased]
### Added
- `collections.User*` should have `__repr__`
### Fixed
- cache modules that only have baseline errors

## [2.8.1]
### Fixes
Expand Down
58 changes: 50 additions & 8 deletions mypy/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,7 @@ class CacheMeta(NamedTuple):
version_id: str # mypy version for cache invalidation
ignore_all: bool # if errors were ignored
plugin_data: Any # config data from plugins
baseline_hash: str | None # hash representing the baselined errors of this module


# NOTE: dependencies + suppressed == all reachable imports;
Expand Down Expand Up @@ -382,6 +383,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta:
meta.get("version_id", sentinel),
meta.get("ignore_all", True),
meta.get("plugin_data", None),
meta.get("baseline_hash", None),
)


Expand Down Expand Up @@ -1105,7 +1107,7 @@ def save_baseline(manager: BuildManager):
manager.stdout.write(f"Baseline successfully updated at {file}\n")


def load_baseline(options: Options, errors: Errors, stdout: TextIO) -> None:
def load_baseline(options: Options, errors: Errors, stdout: TextIO):
if not options.baseline_file:
return

Expand Down Expand Up @@ -1485,6 +1487,25 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No
manager.log(f"Metadata abandoned for {id}: plugin configuration differs")
return None

baseline_hash = None
baseline_errors = manager.errors.baseline.get(path)
if baseline_errors:
errors = [
{
"code": error["code"],
"column": error["column"],
"line": error["line"],
"message": error["message"],
}
for error in baseline_errors
]
baseline_hash = hash_digest(json.dumps(errors).encode())
if baseline_hash != m.baseline_hash:
manager.log(f"Metadata abandoned for {id}: baseline is different")
return None
if baseline_errors:
manager.errors.all_errors[path] = "fresh"

manager.add_stats(fresh_metas=1)
return m

Expand Down Expand Up @@ -1651,6 +1672,7 @@ def write_cache(
source_hash: str,
ignore_all: bool,
manager: BuildManager,
baseline_errors: list[ErrorInfo] | None,
) -> tuple[str, CacheMeta | None]:
"""Write cache files for a module.
Expand Down Expand Up @@ -1736,6 +1758,19 @@ def write_cache(
manager.log(f"Error in os.stat({data_json!r}), skipping cache write")
return interface_hash, None

baseline_hash = None
if baseline_errors:
errors = [
{
"code": error.code and error.code.code,
"column": error.column,
"line": error.line,
"message": error.message,
}
for error in baseline_errors
]
baseline_hash = hash_digest(json.dumps(errors).encode())

mtime = 0 if bazel else int(st.st_mtime)
size = st.st_size
# Note that the options we store in the cache are the options as
Expand All @@ -1761,6 +1796,7 @@ def write_cache(
"version_id": manager.version_id,
"ignore_all": ignore_all,
"plugin_data": plugin_data,
"baseline_hash": baseline_hash,
}

# Write meta cache file
Expand Down Expand Up @@ -2609,7 +2645,7 @@ def valid_references(self) -> set[str]:

return valid_refs

def write_cache(self) -> None:
def write_cache(self, *, baseline_errors: list[ErrorInfo] | None) -> None:
assert self.tree is not None, "Internal error: method must be called on parsed file only"
# We don't support writing cache files in fine-grained incremental mode.
if (
Expand Down Expand Up @@ -2648,6 +2684,7 @@ def write_cache(self) -> None:
self.source_hash,
self.ignore_all,
self.manager,
baseline_errors,
)
if new_interface_hash == self.interface_hash:
self.manager.log(f"Cached module {self.id} has same interface")
Expand Down Expand Up @@ -3599,12 +3636,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
for id in stale:
graph[id].transitive_error = True
for id in stale:
if graph[id].xpath not in manager.errors.ignored_files:
errors = manager.errors.file_messages(
graph[id].xpath, formatter=manager.error_formatter
)
manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False)
graph[id].write_cache()
file = graph[id].xpath
if file not in manager.errors.ignored_files:
errors = manager.errors.file_messages(file, formatter=manager.error_formatter)
manager.flush_errors(manager.errors.simplify_path(file), errors, False)
all_errors = manager.errors.all_errors.get(file)
if all_errors == "fresh":
raise ValueError("huh?")
if file in manager.errors.ignored_files:
all_errors = None
# here, the `all_errors` will only be cached if not `transitive_error`, so `all_errors` is baselined errors
graph[id].write_cache(baseline_errors=all_errors)
graph[id].mark_as_rechecked()


Expand Down
Loading

0 comments on commit c1cbfa3

Please sign in to comment.