Add _cached suffix to test cases in fine-grained tests with cache (#4558

)

This makes it obvious when a test case fails whether the test case
uses caching or not. Previously both kinds of failures looked similar,
which was confusing.

There is one non-obvious bit: the _cached suffix must be inserted
before other suffixes such as -skip since those suffixes are tested
using endswith(), and it's probably better to have such "effectful"
suffixes at the end so that they are prominent.

mypy/test/data.py

@@ -598,9 +598,23 @@ def collect(self) -> Iterator[pytest.Item]:  # type: ignore
                                          optional_out=suite.optional_out,
                                          native_sep=suite.native_sep):
                 if suite.filter(case):
+                    case.name = add_test_name_suffix(case.name, suite.test_name_suffix)
                     yield MypyDataCase(case.name, self, case)
 
 
+def add_test_name_suffix(name: str, suffix: str) -> str:
+    # Find magic suffix of form "-foobar" (used for things like "-skip").
+    m = re.search(r'-[-A-Za-z0-9]+$', name)
+    if m:
+        # Insert suite-specific test name suffix before the magic suffix
+        # which must be the last thing in the test case name since we
+        # are using endswith() checks.
+        magic_suffix = m.group(0)
+        return name[:-len(magic_suffix)] + suffix + magic_suffix
+    else:
+        return name + suffix
+
+
 def is_incremental(testcase: DataDrivenTestCase) -> bool:
     return 'incremental' in testcase.name.lower() or 'incremental' in testcase.file
 
@@ -661,6 +675,9 @@ class DataSuite:
     base_path = '.'
     optional_out = False
     native_sep = False
+    # Name suffix automatically added to each test case in the suite (can be
+    # used to distinguish test cases in suites that share data files)
+    test_name_suffix = ''
 
     # Assigned from MypyDataCase.runtest
     update_data = False

mypy/test/testfinegrainedcache.py

@@ -10,3 +10,4 @@
 
 class FineGrainedCacheSuite(mypy.test.testfinegrained.FineGrainedSuite):
     use_cache = True
+    test_name_suffix = '_cached'