fix KeyError 'dt' with s3 method

crypto-lake · Aug 21, 2024 · f09cfd6 · f09cfd6
1 parent 5030bfb
commit f09cfd6
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -2,7 +2,13 @@
 History
 =======
 
+0.18.0 (2024-08-21)
+-------------------
+
+* fix KeyError: 'dt' with s3 method
+
 0.17.0 (2024-08-09)
+-------------------
 
 * orderbook class performance improvements
 

diff --git a/lakeapi/main.py b/lakeapi/main.py
@@ -126,7 +126,8 @@ def load_data(
 
     def partition_filter(partition: Dict[str, str]) -> bool:
         return (
-            (
+            "dt" in partition
+            and (
                 start is None
                 or start.date() <= datetime.date.fromisoformat(partition["dt"])
             )
@@ -159,7 +160,6 @@ def partition_filter(partition: Dict[str, str]) -> bool:
             last_ex = None
             for _ in range(2):
                 try:
-                    # TODO: log & skip corrupted files
                     df = lakeapi._read_parquet.read_parquet(
                         path=f"s3://{bucket}/{table}/",
                         partition_filter=partition_filter,