diff --git a/HISTORY.rst b/HISTORY.rst index e0afb3f..993a1b7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,7 +2,13 @@ History ======= +0.18.0 (2024-08-21) +------------------- + +* fix KeyError: 'dt' with s3 method + 0.17.0 (2024-08-09) +------------------- * orderbook class performance improvements diff --git a/lakeapi/main.py b/lakeapi/main.py index c5b6310..2aacadf 100644 --- a/lakeapi/main.py +++ b/lakeapi/main.py @@ -126,7 +126,8 @@ def load_data( def partition_filter(partition: Dict[str, str]) -> bool: return ( - ( + "dt" in partition + and ( start is None or start.date() <= datetime.date.fromisoformat(partition["dt"]) ) @@ -159,7 +160,6 @@ def partition_filter(partition: Dict[str, str]) -> bool: last_ex = None for _ in range(2): try: - # TODO: log & skip corrupted files df = lakeapi._read_parquet.read_parquet( path=f"s3://{bucket}/{table}/", partition_filter=partition_filter,