Add ability to fetch restored glacier object

embulk · Feb 27, 2021 · a20e9d3 · a20e9d3
1 parent 283f73c
commit a20e9d3
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 1 deletion.
diff --git a/...lk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java b/...lk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java
@@ -17,8 +17,10 @@
 package org.embulk.input.s3.explorer;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.embulk.input.s3.DefaultRetryable;
 import org.embulk.util.retryhelper.RetryExecutor;
@@ -53,6 +55,17 @@ public ObjectListing call()
         return ol.getObjectSummaries();
     }
 
+    @Override
+    protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) {
+        final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey());
+
+        return new DefaultRetryable<ObjectMetadata>("Get object metadata")
+        {
+            @Override
+            public ObjectMetadata call() { return s3Client.getObjectMetadata(req); }
+        }.executeWith(retryExecutor);
+    }
+
     @Override
     protected boolean hasNext()
     {

diff --git a/embulk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java b/embulk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java
@@ -17,6 +17,7 @@
 package org.embulk.input.s3.explorer;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
 import org.embulk.config.ConfigException;
@@ -54,7 +55,13 @@ public void addToBuilder(final FileList.Builder builder)
                         LOGGER.warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
                         continue;
                     }
-                    throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
+
+                    ObjectMetadata objectMetadata = fetchObjectMetadata(s);
+                    if (objectMetadata != null && objectMetadata.getRestoreExpirationTime() != null) {
+                        LOGGER.info("Restored Glacier object \"s3://{}/{}\" found", bucketName, s.getKey());
+                    } else {
+                        throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
+                    }
                 }
                 if (s.getSize() > 0) {
                     builder.add(s.getKey(), s.getSize());
@@ -69,5 +76,7 @@ public void addToBuilder(final FileList.Builder builder)
 
     protected abstract List<S3ObjectSummary> fetch();
 
+    protected abstract ObjectMetadata fetchObjectMetadata(final S3ObjectSummary obj);
+
     protected abstract boolean hasNext();
 }
diff --git a/...lk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java b/...lk-input-s3/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java
@@ -17,8 +17,10 @@
 package org.embulk.input.s3.explorer;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import org.embulk.input.s3.DefaultRetryable;
 import org.embulk.util.retryhelper.RetryExecutor;
@@ -72,6 +74,17 @@ public ObjectListing call()
                 .collect(Collectors.toList());
     }
 
+    @Override
+    protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) {
+        final GetObjectMetadataRequest req = new GetObjectMetadataRequest(obj.getBucketName(), obj.getKey());
+
+        return new DefaultRetryable<ObjectMetadata>("Get object metadata")
+        {
+            @Override
+            public ObjectMetadata call() { return s3Client.getObjectMetadata(req); }
+        }.executeWith(retryExecutor);
+    }
+
     @Override
     public boolean hasNext()
     {

diff --git a/embulk-input-s3/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java b/embulk-input-s3/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java
@@ -17,6 +17,7 @@
 package org.embulk.input.s3.explorer;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
 import org.embulk.EmbulkTestRuntime;
@@ -31,6 +32,7 @@
 import org.mockito.runners.MockitoJUnitRunner;
 
 import java.util.Collections;
+import java.util.Date;
 import java.util.List;
 
 import static org.mockito.Mockito.doReturn;
@@ -75,6 +77,20 @@ public void addToBuilder_should_throw_exception_if_notskipped_glacier_storage()
         s3PrefixFileExplorer.addToBuilder(builder);
     }
 
+    @Test
+    public void addToBuilder_should_not_throw_exception_if_restored_glacier_object()
+    {
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
+        when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
+        when(s3ObjectSummary.getSize()).thenReturn(1L);
+        ObjectMetadata objectMetadata = new ObjectMetadata();
+        objectMetadata.setRestoreExpirationTime(new Date());
+        doReturn(objectMetadata).when(s3PrefixFileExplorer).fetchObjectMetadata(s3ObjectSummary);
+        doReturn(true).when(s3PrefixFileExplorer).hasNext();
+        s3PrefixFileExplorer.addToBuilder(builder);
+        verify(builder).add(PATH_PREFIX + OBJECT_KEY, 1);
+    }
+
     @Test
     public void addToBuilder_should_skip_glacier_storage_if_allowed()
     {
@@ -132,6 +148,9 @@ protected List<S3ObjectSummary> fetch()
                 return null;
             }
 
+            @Override
+            protected ObjectMetadata fetchObjectMetadata(S3ObjectSummary obj) { return null; }
+
             @Override
             protected boolean hasNext()
             {