From 9002589a5996324fe3a355cd967c1a865918b8fc Mon Sep 17 00:00:00 2001 From: Vitaly Fedyunin Date: Tue, 17 May 2022 23:55:55 -0400 Subject: [PATCH] Update on "Adding lock mechanism to prevent on_disk_cache downloading twice" Fixes #144 [ghstack-poisoned] --- torchdata/datapipes/iter/util/cacheholder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/datapipes/iter/util/cacheholder.py b/torchdata/datapipes/iter/util/cacheholder.py index adc9b2360..51ea975b2 100644 --- a/torchdata/datapipes/iter/util/cacheholder.py +++ b/torchdata/datapipes/iter/util/cacheholder.py @@ -114,7 +114,7 @@ def _hash_check(filepath, hash_dict, hash_type): else: hash_func = hashlib.md5() - with portalocker.Lock(filepath, "rb") as f: + with portalocker.Lock(filepath, "rb", flags=portalocker.LockFlags.EXCLUSIVE) as f: chunk = f.read(1024 ** 2) while chunk: hash_func.update(chunk)