exclude temporary files fron DiskDict contents

requirements.txt

@@ -6,6 +6,6 @@ tqdm
 redis>=4.0.0
 pyyaml
 humanfriendly
-pydantic<2.0.0
+pydantic<3.0.0
 requests
 s3fs>=2023.1.0

tarn/__version__.py

@@ -1 +1 @@
-__version__ = '0.13.2'
+__version__ = '0.13.3'

tarn/compat.py

@@ -29,51 +29,42 @@
 try:
     from pydantic import BaseModel, field_validator as _field_validator, model_validator
 
-
     def field_validator(*args, always=None, **kwargs):
         # we just ignore `always`
         return _field_validator(*args, **kwargs)
 
-
     def model_validate(cls, data):
         return cls.model_validate(data)
 
-
     def model_dump(obj, **kwargs):
         return obj.model_dump(**kwargs)
 
-
     class NoExtra(BaseModel):
         model_config = {
-            'extra': 'forbid'
+            'extra': 'forbid',
         }
 
 
 except ImportError:
     from pydantic import BaseModel, root_validator, validator as _field_validator
 
-
     def model_validator(mode: str):
         assert mode == 'before'
         return root_validator(pre=True)
 
-
     def field_validator(*args, mode: str = 'after', **kwargs):
         # we just ignore `always`
         assert mode in ('before', 'after')
         if mode == 'before':
             kwargs['pre'] = True
         return _field_validator(*args, **kwargs)
 
-
     def model_validate(cls, data):
         return cls.parse_obj(data)
 
-
     def model_dump(obj, **kwargs):
         return obj.dict(**kwargs)
 
-
     class NoExtra(BaseModel):
         class Config:
             extra = 'forbid'
@@ -87,11 +78,9 @@ def remove_readonly(func, p, _):
 
         shutil.rmtree(path, ignore_errors=ignore_errors, onerror=remove_readonly)
 
-
     def get_path_group(path: PathOrStr) -> Union[int, None]:
         pass
 
-
     def remove_file(path: PathOrStr):
         os.chmod(path, stat.S_IWRITE)
         os.remove(path)
@@ -100,7 +89,6 @@ def remove_file(path: PathOrStr):
     rmtree = shutil.rmtree
     remove_file = os.remove
 
-
     def get_path_group(path: PathOrStr) -> Union[int, None]:
         return Path(path).stat().st_gid
 

tarn/location/disk_dict/config.py

@@ -52,13 +52,7 @@ def build(self):
 class ToolConfig(NoExtra):
     name: str
     args: Tuple = ()
-    kwargs: Optional[Dict[str, Any]] = None
-
-    @field_validator('kwargs', always=True)
-    def normalize_kwargs(cls, v):
-        if v is None:
-            return {}
-        return v
+    kwargs: Optional[Dict[str, Any]] = Field(default_factory=dict)
 
 
 class StorageConfig(NoExtra):

tarn/location/disk_dict/location.py

@@ -32,7 +32,8 @@
             config = load_config(root)
 
         if levels is not None and tuple(config.levels) != tuple(levels):
-            raise ValueError(f"The passed levels (levels) don't match with the ones from the config ({config.levels}).")
+            raise ValueError(
+                f"The passed levels ({levels}) don't match with the ones from the config ({config.levels}).")
 
         self.levels = config.levels
         # TODO: deprecate
@@ -60,9 +61,8 @@
         tools = self.root / 'tools'
         config = self.root / 'config.yml'
         for file in self.root.glob('/'.join('*' * len(self.levels))):
-            if file == config or file.is_relative_to(tools):
+            if file == config or file.is_relative_to(tools) or file.is_relative_to(self.tmp):
                 continue
-
             key = bytes.fromhex(''.join(file.relative_to(self.root).parts))
             with self.locker.read(key):
                 yield key, self, DiskDictMeta(key, self.usage_tracker, self.labels)
@@ -156,7 +156,6 @@
         with self.locker.write(key):
             if not file.exists():
                 return False
-
             # TODO: don't need this if no tracker is used
             if file.is_dir():
                 # TODO: legacy
@@ -213,6 +212,9 @@
     def labels(self) -> MaybeLabels:
         return self._labels.get(self._key)
 
+    def __str__(self):
+        return f"{self.last_used}, {self.labels}"
+
 
 def _is_pathlike(x):
     return isinstance(x, (os.PathLike, str))