From 9ac9d4c7ee0efb7cea03392a7bb3f5aa9b062e47 Mon Sep 17 00:00:00 2001 From: George Zubrienko Date: Mon, 8 Jul 2024 16:44:36 +0200 Subject: [PATCH] Fixes to S3 Storage blob methods --- adapta/storage/blob/s3_storage_client.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/adapta/storage/blob/s3_storage_client.py b/adapta/storage/blob/s3_storage_client.py index 0671bc9a..e84a8cfe 100644 --- a/adapta/storage/blob/s3_storage_client.py +++ b/adapta/storage/blob/s3_storage_client.py @@ -21,6 +21,7 @@ from typing import Optional, Callable, Type, Iterator, Dict, TypeVar, final from datetime import timedelta from boto3 import Session +from botocore.exceptions import ClientError from adapta.security.clients import AwsClient from adapta.storage.blob.base import StorageClient @@ -90,7 +91,7 @@ def blob_exists(self, blob_path: DataPath) -> bool: try: self._s3_resource.meta.client.head_object(Bucket=s3_path.bucket, Key=s3_path.path) return True - except StorageClientError: + except ClientError: return False def save_data_as_blob( @@ -146,10 +147,10 @@ def list_blobs( response = self._s3_resource.meta.client.list_objects(Bucket=s3_path.bucket, Prefix=s3_path.path) if "Contents" not in response: yield from iter([]) - - for blob in response["Contents"]: - if filter_predicate is None or filter_predicate(blob): - yield blob + else: + for blob in response["Contents"]: + if filter_predicate is None or filter_predicate(blob): + yield blob def read_blobs( self, @@ -195,8 +196,8 @@ def download_blobs( os.makedirs(os.path.dirname(local_file_path), exist_ok=True) try: self._s3_resource.meta.client.download_file(s3_path.bucket, blob.key, local_file_path) - except StorageClientError as error: - raise RuntimeError(f"Error downloading blob: {error}") from error + except ClientError as error: + raise StorageClientError(f"Error downloading blob: {error}") from error def copy_blob(self, blob_path: DataPath, target_blob_path: DataPath, doze_period_ms: int = 0) -> None: """