Update on "Adding lock mechanism to prevent on_disk_cache downloading…

… twice"


Fixes #144

[ghstack-poisoned]

test/test_local_io.py

@@ -621,7 +621,7 @@ def test_disk_cache_locks(self):
             dp = dp.end_caching(mode="t", filepath_fn=_noop, timeout=120)
             dp = FileOpener(dp)
             dp = StreamReader(dp)
-            dl = DataLoader(dp, num_workers=10, multiprocessing_context="spawn", batch_size=1, collate_fn=_unbatch)
+            dl = DataLoader(dp, num_workers=5, multiprocessing_context="spawn", batch_size=1, collate_fn=_unbatch)
             result = list(dl)
             all_files = []
             for (_, _, filenames) in os.walk(tmpdirname):

torchdata/datapipes/iter/util/cacheholder.py

@@ -272,15 +272,16 @@ def _find_promise_file(filename):
 
 
 def _is_promise_pending(promise_filename):
-    try:
-        with portalocker.Lock(promise_filename, "r") as promise_fh:
-            data = promise_fh.read()
-            file_exists = len(data) > 0
-    except FileNotFoundError:
-        return False
-    except PermissionError:
-        return True
-    return file_exists
+    return os.path.exists(promise_filename)
+    # try:
+    #     with portalocker.Lock(promise_filename, "r") as promise_fh:
+    #         data = promise_fh.read()
+    #         file_exists = len(data) > 0
+    # except FileNotFoundError:
+    #     return False
+    # except PermissionError:
+    #     return True
+    # return file_exists
 
 
 def _wait_promise_fn(timeout, filename):