From c3a4e832b4b85cab979e6966e65a11a1d6a4b120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 13 Nov 2023 09:45:05 +0100 Subject: [PATCH] XFAIL TestLocalPath.test_make_numbered_dir_multiprocess_safe The tested py.path.local.make_numbered_dir function is *not* multiprocess safe, because is uses os.listdir which itself is not. The os.listdir documentation explicitly states that: > If a file is removed from or added to the directory during the call > of this function, whether a name for that file be included is unspecified. This can lead to a race when: 1. process A attempts to create directory N 2. the creation fails, as another process already created it in the meantime 3. process A calls listdir to determine a more recent maxnum 4. processes B+ repeatedly create newer directories and they delete directory N 5. process A doesn't have directory N or any newer directory in listdir result 6. process A attempts to create directory N again and raises For details, see https://github.com/pytest-dev/pytest/issues/11603#issuecomment-1805708144 and bellow. Additionally, the test itself has a race in batch_make_numbered_dirs. When this functions attempts to write to repro-N/foo, repro-N may have already been removed by another process. For details, see https://github.com/pytest-dev/pytest/issues/11603#issuecomment-1804714313 and bellow. --- The tested py.path.local.make_numbered_dir function is not used in pytest. There is a different implementation in _pytest.pathlib. We plan to remove this function in pytest 8.x. Closes https://github.com/pytest-dev/pytest/issues/11603 --- changelog/11603.trivial.rst | 7 +++++++ testing/_py/test_local.py | 3 +++ 2 files changed, 10 insertions(+) create mode 100644 changelog/11603.trivial.rst diff --git a/changelog/11603.trivial.rst b/changelog/11603.trivial.rst new file mode 100644 index 00000000000..818f9fabb16 --- /dev/null +++ b/changelog/11603.trivial.rst @@ -0,0 +1,7 @@ +Marked the ``TestLocalPath.test_make_numbered_dir_multiprocess_safe`` test as xfail. + +There is a reproducible race condition in the test and a possible +race condition in the tested ``py.path.local.make_numbered_dir`` function itself. + +The tested function is not used within pytest and it is only kept +for the time being. We assume it will be removed in pytest 8.x. diff --git a/testing/_py/test_local.py b/testing/_py/test_local.py index aebee380cb9..0a56a113d6b 100644 --- a/testing/_py/test_local.py +++ b/testing/_py/test_local.py @@ -868,6 +868,9 @@ def test_fspath_protocol_other_class(self, fake_fspath_obj): py_path.strpath, str_path ) + @pytest.mark.xfail( + reason="#11603", strict=False, raises=(error.EEXIST, error.ENOENT) + ) def test_make_numbered_dir_multiprocess_safe(self, tmpdir): # https://github.com/pytest-dev/py/issues/30 with multiprocessing.Pool() as pool: