Update on "Adding lock mechanism to prevent on_disk_cache downloading…

… twice" Fixes #144 [ghstack-poisoned]
pytorch · May 18, 2022 · 748d4fc · 748d4fc
1 parent f4c18b6
commit 748d4fc
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/torchdata/datapipes/iter/util/cacheholder.py b/torchdata/datapipes/iter/util/cacheholder.py
@@ -114,7 +114,7 @@ def _hash_check(filepath, hash_dict, hash_type):
     else:
         hash_func = hashlib.md5()
 
-    with portalocker.Lock(filepath, "rb", flags=portalocker.LockFlags.EXCLUSIVE) as f:
+    with portalocker.Lock(filepath, "rb", flags=portalocker.LockFlags.SHARED) as f:
         chunk = f.read(1024 ** 2)
         while chunk:
             hash_func.update(chunk)

diff --git a/torchdata/datapipes/iter/util/saver.py b/torchdata/datapipes/iter/util/saver.py
@@ -59,6 +59,8 @@ def __iter__(self) -> Iterator[str]:
             if not os.path.exists(dirname):
                 os.makedirs(dirname)
             with portalocker.Lock(filepath, self.mode, flags=portalocker.LockFlags.EXCLUSIVE) as f:
+                # TODO(VitalyFedyunin): Enabling line above fails TorchText tests, need to investigate race condition
+                # with open(filepath, self.mode) as f:
                 f.write(data)
             yield filepath