From a6646f01ade72065987a1bfb7588cbcf019ec8ce Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 17 Feb 2020 16:43:53 +1100 Subject: [PATCH 1/5] Naive headObject method wrapper for GLACIER_ARCHIVE restore. Observed header combinations while debugging do not seem to align docs, more investigation is needed. --- .../java/org/broad/igv/aws/S3LoadDialog.java | 46 +++++++++++++++++-- .../java/org/broad/igv/util/AmazonUtils.java | 13 ++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broad/igv/aws/S3LoadDialog.java b/src/main/java/org/broad/igv/aws/S3LoadDialog.java index 10631bfcdf..2ca737c8aa 100644 --- a/src/main/java/org/broad/igv/aws/S3LoadDialog.java +++ b/src/main/java/org/broad/igv/aws/S3LoadDialog.java @@ -48,7 +48,7 @@ import java.util.Collections; import java.util.List; -import software.amazon.awssdk.services.s3.model.S3Error; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; @@ -208,14 +208,50 @@ private boolean isObjectAccessible(Triple S3Obj) { String S3ObjectKey = S3Obj.getMiddle().toString(); String S3ObjectStorageClass = S3Obj.getRight().toString(); + HeadObjectResponse S3Meta; + boolean ObjAvailable; + String objCurrentState = ""; + if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || S3ObjectStorageClass.contains("GLACIER")) { - MessageUtils.showErrorMessage("Amazon S3 object is in " + S3ObjectStorageClass + " storage tier, not accessible at this moment. " + - "Please contact your local system administrator about object: s3://" + S3ObjectBucket + "/" + S3ObjectKey, null); - return false; + // Determine in which state this object really is: + // 1. Archived. + // 2. In the process of being restored. + // 3. Restored + // + // This is important because after restoration the object mantains the Tier (DEEP_ARCHIVE) instead of + // transitioning that attribute to STANDARD, we must look at head_object response for the "Restore" + // attribute. + + //https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/model/HeadObjectResponse.html#restore-- + S3Meta = AmazonUtils.getObjectMetadata(S3Obj); + objCurrentState = S3Meta.restore(); + + if(objCurrentState == null) { + ObjAvailable = false; + log.info("Object is archived and is not being restored"); + } else if(objCurrentState.contains("ongoing-request=\"true\"")) { + MessageUtils.showErrorMessage("Amazon S3 object is in " + S3ObjectStorageClass + " and being restored right now, please be patient, this can take up to 48h. " + + "For further enquiries about this dataset, please use the following path when communicating with your system administrator: s3://" + S3ObjectBucket + "/" + S3ObjectKey, null); + ObjAvailable = false; + log.info("isObjectAccessible(): The object "+ S3ObjectKey +" is being restored, bailing out."); + // "If an archive copy is already restored, the header value indicates when Amazon S3 is scheduled to delete the object copy" + } else if(objCurrentState.contains("ongoing-request=\"false\"") && objCurrentState.contains("expiry-date=")) { + ObjAvailable = true; + log.info("isObjectAccessible(): The object "+ S3ObjectKey +" has been restored, will be loaded next."); + } else { + MessageUtils.showErrorMessage("Amazon S3 object is in " + S3ObjectStorageClass + " storage tier, not accessible at this moment. " + + "Please contact your local system administrator about object: s3://" + S3ObjectBucket + "/" + S3ObjectKey, null); + ObjAvailable = false; + log.info("isObjectAccessible(): The object"+ S3ObjectKey +" is not available."); + } + } else { + // The object must be either in STANDARD, INFREQUENT_ACCESS, INTELLIGENT_TIERING or + // any other "immediately available" tier... + ObjAvailable = true; } - return true; + return ObjAvailable; } private void initComponents() { diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index b5e51c9565..75b09cade0 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -2,6 +2,7 @@ import com.google.gson.JsonObject; import htsjdk.samtools.util.Tuple; +import org.apache.commons.lang3.tuple.Triple; import org.apache.log4j.Logger; import org.broad.igv.Globals; import org.broad.igv.aws.IGVS3Object; @@ -203,6 +204,18 @@ public static List ListBucketsForUser() { return bucketsFinalList; } + public static HeadObjectResponse getObjectMetadata(Triple S3Obj) { + String S3ObjectBucket = S3Obj.getLeft().toString(); + String S3ObjectKey = S3Obj.getMiddle().toString(); + + HeadObjectRequest HeadObjReq = HeadObjectRequest.builder() + .bucket(S3ObjectBucket) + .key(S3ObjectKey).build(); + HeadObjectResponse HeadObjRes = s3Client.headObject(HeadObjReq); + log.debug("getObjectMetadata(): "+HeadObjRes.toString()); + return HeadObjRes; + } + private static List getReadableBuckets(List buckets) { List> futures = buckets.stream() From 55ee9c24995876f08fede5912a22c3e500e55add Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Mon, 17 Feb 2020 17:14:50 +1100 Subject: [PATCH 2/5] The AWS SDK Java v2 HeadObject.restore() is not reliable as of version 2.8.5, see https://github.com/aws/aws-sdk-java-v2/issues/670#issuecomment-415835721 --- src/main/java/org/broad/igv/aws/S3LoadDialog.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broad/igv/aws/S3LoadDialog.java b/src/main/java/org/broad/igv/aws/S3LoadDialog.java index 2ca737c8aa..f1cc63892c 100644 --- a/src/main/java/org/broad/igv/aws/S3LoadDialog.java +++ b/src/main/java/org/broad/igv/aws/S3LoadDialog.java @@ -210,7 +210,7 @@ private boolean isObjectAccessible(Triple S3Obj) { HeadObjectResponse S3Meta; boolean ObjAvailable; - String objCurrentState = ""; + String objCurrentState; if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || S3ObjectStorageClass.contains("GLACIER")) { @@ -225,7 +225,8 @@ private boolean isObjectAccessible(Triple S3Obj) { //https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/model/HeadObjectResponse.html#restore-- S3Meta = AmazonUtils.getObjectMetadata(S3Obj); - objCurrentState = S3Meta.restore(); + //objCurrentState = S3Meta.restore(); + objCurrentState = S3Meta.sdkHttpResponse().headers().get("x-amz-restore").toString(); if(objCurrentState == null) { ObjAvailable = false; From 47080075d564535e292133192ea5708b8c2aae24 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 18 Feb 2020 15:49:11 +1100 Subject: [PATCH 3/5] Other than fixing the Glacier/Archive tier error handling, also implemented support for File->Load From URL if the user knows the S3 URL, they can paste it there directly. Also aws-java-sdk-v2 version bump from Maven --- build.gradle | 8 +- .../java/org/broad/igv/aws/S3LoadDialog.java | 62 ++----------- .../igv/ui/action/LoadFromURLMenuAction.java | 23 +++++ .../java/org/broad/igv/util/AmazonUtils.java | 89 +++++++++++++++++++ 4 files changed, 122 insertions(+), 60 deletions(-) diff --git a/build.gradle b/build.gradle index 5409ed1e37..83d35c51a3 100644 --- a/build.gradle +++ b/build.gradle @@ -131,10 +131,10 @@ dependencies { [group: 'org.netbeans.external', name: 'AbsoluteLayout', version: 'RELEASE110'], // Amazon deps - [group: 'software.amazon.awssdk', name: 'http-client-spi', version: '2.8.5'], - [group: 'software.amazon.awssdk', name: 'cognitoidentity', version: '2.8.5'], - [group: 'software.amazon.awssdk', name: 'sts', version: '2.8.5'], - [group: 'software.amazon.awssdk', name: 's3', version: '2.8.5'], + [group: 'software.amazon.awssdk', name: 'http-client-spi', version: '2.10.65'], + [group: 'software.amazon.awssdk', name: 'cognitoidentity', version: '2.10.65'], + [group: 'software.amazon.awssdk', name: 'sts', version: '2.10.65'], + [group: 'software.amazon.awssdk', name: 's3', version: '2.10.65'], ) testImplementation( diff --git a/src/main/java/org/broad/igv/aws/S3LoadDialog.java b/src/main/java/org/broad/igv/aws/S3LoadDialog.java index f1cc63892c..9a9a28ca5b 100644 --- a/src/main/java/org/broad/igv/aws/S3LoadDialog.java +++ b/src/main/java/org/broad/igv/aws/S3LoadDialog.java @@ -51,6 +51,8 @@ import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import static org.broad.igv.util.AmazonUtils.isObjectAccessible; + public class S3LoadDialog extends JDialog { @@ -102,7 +104,8 @@ private void loadButtonActionPerformed(ActionEvent e) { if (isFilePath(path)) { Triple bucketKeyTier = getBucketKeyTierFromTreePath(path); - if(!isObjectAccessible(bucketKeyTier)) return; + AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); + if(!res.getObjAvailable()) { MessageUtils.showErrorMessage(res.getErrorReason(), null); return; } preLocatorPaths.add(bucketKeyTier); } @@ -200,60 +203,6 @@ private void updateModel(DefaultMutableTreeNode parent) { } } - // Determines whether the object is immediately available. - // On AWS this means present in STANDARD, STANDARD_IA, INTELLIGENT_TIERING object access tiers. - // Tiers GLACIER and DEEP_ARCHIVE are not immediately retrievable without action. - private boolean isObjectAccessible(Triple S3Obj) { - String S3ObjectBucket = S3Obj.getLeft().toString(); - String S3ObjectKey = S3Obj.getMiddle().toString(); - String S3ObjectStorageClass = S3Obj.getRight().toString(); - - HeadObjectResponse S3Meta; - boolean ObjAvailable; - String objCurrentState; - - if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || - S3ObjectStorageClass.contains("GLACIER")) { - // Determine in which state this object really is: - // 1. Archived. - // 2. In the process of being restored. - // 3. Restored - // - // This is important because after restoration the object mantains the Tier (DEEP_ARCHIVE) instead of - // transitioning that attribute to STANDARD, we must look at head_object response for the "Restore" - // attribute. - - //https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/model/HeadObjectResponse.html#restore-- - S3Meta = AmazonUtils.getObjectMetadata(S3Obj); - //objCurrentState = S3Meta.restore(); - objCurrentState = S3Meta.sdkHttpResponse().headers().get("x-amz-restore").toString(); - - if(objCurrentState == null) { - ObjAvailable = false; - log.info("Object is archived and is not being restored"); - } else if(objCurrentState.contains("ongoing-request=\"true\"")) { - MessageUtils.showErrorMessage("Amazon S3 object is in " + S3ObjectStorageClass + " and being restored right now, please be patient, this can take up to 48h. " + - "For further enquiries about this dataset, please use the following path when communicating with your system administrator: s3://" + S3ObjectBucket + "/" + S3ObjectKey, null); - ObjAvailable = false; - log.info("isObjectAccessible(): The object "+ S3ObjectKey +" is being restored, bailing out."); - // "If an archive copy is already restored, the header value indicates when Amazon S3 is scheduled to delete the object copy" - } else if(objCurrentState.contains("ongoing-request=\"false\"") && objCurrentState.contains("expiry-date=")) { - ObjAvailable = true; - log.info("isObjectAccessible(): The object "+ S3ObjectKey +" has been restored, will be loaded next."); - } else { - MessageUtils.showErrorMessage("Amazon S3 object is in " + S3ObjectStorageClass + " storage tier, not accessible at this moment. " + - "Please contact your local system administrator about object: s3://" + S3ObjectBucket + "/" + S3ObjectKey, null); - ObjAvailable = false; - log.info("isObjectAccessible(): The object"+ S3ObjectKey +" is not available."); - } - } else { - // The object must be either in STANDARD, INFREQUENT_ACCESS, INTELLIGENT_TIERING or - // any other "immediately available" tier... - ObjAvailable = true; - } - - return ObjAvailable; - } private void initComponents() { // JFormDesigner - Component initialization - DO NOT MODIFY //GEN-BEGIN:initComponents @@ -293,7 +242,8 @@ public void mousePressed(MouseEvent e) { if (isFilePath(selPath)) { Triple bucketKeyTier = getBucketKeyTierFromTreePath(selPath); - if(!isObjectAccessible(bucketKeyTier)) return; + AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); + if(!res.getObjAvailable()) { MessageUtils.showErrorMessage(res.getErrorReason(), null); return;} ResourceLocator loc = getResourceLocatorFromBucketKey(bucketKeyTier); IGV.getInstance().loadTracks(Collections.singletonList(loc)); diff --git a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java index 5907d7b39c..cd7368c3d7 100644 --- a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java +++ b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java @@ -29,7 +29,10 @@ */ package org.broad.igv.ui.action; +import htsjdk.samtools.util.Tuple; +import org.apache.commons.lang3.tuple.Triple; import org.apache.log4j.Logger; +import org.broad.igv.aws.S3Object; import org.broad.igv.exceptions.HttpResponseException; import org.broad.igv.feature.genome.GenomeManager; import org.broad.igv.google.GoogleUtils; @@ -40,6 +43,7 @@ import org.broad.igv.ui.IGVMenuBar; import org.broad.igv.ui.util.LoadFromURLDialog; import org.broad.igv.ui.util.MessageUtils; +import org.broad.igv.util.AmazonUtils; import org.broad.igv.util.HttpUtils; import org.broad.igv.util.ResourceLocator; @@ -52,6 +56,8 @@ import java.util.HashMap; import java.util.Map; +import static org.broad.igv.util.AmazonUtils.isObjectAccessible; + /** * @author jrobinso */ @@ -96,6 +102,23 @@ public void actionPerformed(ActionEvent e) { MessageUtils.showMessage("Error loading url: " + url + " (" + ex.toString() + ")"); } } else { + try { + // If AWS support is active, check if objects are in accessible tiers via Load URL menu... + if (AmazonUtils.isAwsS3Path(url)) { + Tuple S3ObjTuple = AmazonUtils.bucketAndKey(url); + Triple bucketKeyTier = Triple.of(S3ObjTuple.a, S3ObjTuple.b, ""); + + AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); + if (!res.getObjAvailable()) { + MessageUtils.showMessage(res.getErrorReason()); + return; + } + } + } catch (NullPointerException npe) { + // User has not yet done Amazon->Login sequence + AmazonUtils.checkLogin(); + } + ResourceLocator rl = new ResourceLocator(url.trim()); if (dlg.getIndexURL() != null) { diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index 75b09cade0..2ccc06089d 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -216,6 +216,95 @@ public static HeadObjectResponse getObjectMetadata(Triple S3Obj) { return HeadObjRes; } + + // Holds whether a S3 object is accessible or not and reason/error msg in case it's not. + public static class S3ObjectAccessResult { + private boolean objAvailable; + private String errorReason; + + public boolean getObjAvailable() { + return objAvailable; + } + + public void setObjAvailable(boolean objAvailable) { + this.objAvailable = objAvailable; + } + + public String getErrorReason() { + return errorReason; + } + + public void setErrorReason(String errorReason) { + this.errorReason = errorReason; + } + } + + // Determines whether the object is immediately available. + // On AWS this means present in STANDARD, STANDARD_IA, INTELLIGENT_TIERING object access tiers. + // Tiers GLACIER and DEEP_ARCHIVE are not immediately retrievable without action. + public static S3ObjectAccessResult isObjectAccessible(Triple S3Obj) { + String S3ObjectBucket = S3Obj.getLeft().toString(); + String S3ObjectKey = S3Obj.getMiddle().toString(); + S3ObjectAccessResult res = new S3ObjectAccessResult(); + + HeadObjectResponse S3Meta; + + String S3ObjectStorageStatus = null; + String S3ObjectStorageClass; + + S3Meta = AmazonUtils.getObjectMetadata(S3Obj); + S3ObjectStorageClass = S3Meta.storageClass().toString(); + + // Determine in which state this object really is: + // 1. Archived. + // 2. In the process of being restored. + // 3. Restored + // + // This is important because after restoration the object mantains the Tier (DEEP_ARCHIVE) instead of + // transitioning that attribute to STANDARD, we must look at head_object response for the "Restore" + // attribute. + // + // Possible error reason messages for the users are: + + String archived = "Amazon S3 object is in " + S3ObjectStorageClass + " storage tier, not accessible at this moment. " + + "Please contact your local system administrator about object: s3://" + S3ObjectBucket + "/" + S3ObjectKey; + String restoreInProgress = "Amazon S3 object is in " + S3ObjectStorageClass + " and being restored right now, please be patient, this can take up to 48h. " + + "For further enquiries about this dataset, please use the following path when communicating with your system administrator: s3://" + S3ObjectBucket + "/" + S3ObjectKey; + + if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || + S3ObjectStorageClass.contains("GLACIER")) { + try { + // XXX: .restore() incorrectly returning null, using sdkHttpResponse directly as a workaround for aws-java-sdk-v2 bug + //objCurrentState = S3Meta.restore(); + S3ObjectStorageStatus = S3Meta.sdkHttpResponse().headers().get("x-amz-restore").toString(); + } catch(NullPointerException npe) { + res.setObjAvailable(false); + res.setErrorReason(archived); + return res; + } + + if(S3ObjectStorageStatus.contains("ongoing-request=\"true\"")) { + res.setObjAvailable(false); + res.setErrorReason(restoreInProgress); + + // "If an archive copy is already restored, the header value indicates when Amazon S3 is scheduled to delete the object copy" + } else if(S3ObjectStorageStatus.contains("ongoing-request=\"false\"") && S3ObjectStorageStatus.contains("expiry-date=")) { + res.setObjAvailable(true); + } else { + // The object has never been restored? + res.setObjAvailable(false); + res.setErrorReason(archived); + } + } else { + // The object must be either in STANDARD, INFREQUENT_ACCESS, INTELLIGENT_TIERING or + // any other "immediately available" tier... + res.setErrorReason("Object is in an accessible tier, no errors are expected"); + res.setObjAvailable(true); + } + + return res; + } + private static List getReadableBuckets(List buckets) { List> futures = buckets.stream() From 639f827754d50749d01ae0ff760d263701ead8f0 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Tue, 18 Feb 2020 16:16:02 +1100 Subject: [PATCH 4/5] This field gets populated on newer versions of the aws-java-sdk-v2, fortunately --- src/main/java/org/broad/igv/util/AmazonUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index 2ccc06089d..cf7bbb2881 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -274,9 +274,7 @@ public static S3ObjectAccessResult isObjectAccessible(Triple S3Obj) { if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || S3ObjectStorageClass.contains("GLACIER")) { try { - // XXX: .restore() incorrectly returning null, using sdkHttpResponse directly as a workaround for aws-java-sdk-v2 bug - //objCurrentState = S3Meta.restore(); - S3ObjectStorageStatus = S3Meta.sdkHttpResponse().headers().get("x-amz-restore").toString(); + S3ObjectStorageStatus = S3Meta.restore(); } catch(NullPointerException npe) { res.setObjAvailable(false); res.setErrorReason(archived); From 1abce3c830f35bdedb29d40a29586c4d1f0be996 Mon Sep 17 00:00:00 2001 From: Roman Valls Guimera Date: Wed, 19 Feb 2020 12:51:14 +1100 Subject: [PATCH 5/5] Apply suggestions/code review from @reisingerf. Rollback aws-sdk version bump since it affects error messaging and "germane-ity" of this PR. Killing Triple/Tuple in next commit. --- build.gradle | 8 +-- .../java/org/broad/igv/aws/S3LoadDialog.java | 5 +- .../igv/ui/action/LoadFromURLMenuAction.java | 17 ++--- .../java/org/broad/igv/util/AmazonUtils.java | 69 +++++++++---------- 4 files changed, 43 insertions(+), 56 deletions(-) diff --git a/build.gradle b/build.gradle index 83d35c51a3..5409ed1e37 100644 --- a/build.gradle +++ b/build.gradle @@ -131,10 +131,10 @@ dependencies { [group: 'org.netbeans.external', name: 'AbsoluteLayout', version: 'RELEASE110'], // Amazon deps - [group: 'software.amazon.awssdk', name: 'http-client-spi', version: '2.10.65'], - [group: 'software.amazon.awssdk', name: 'cognitoidentity', version: '2.10.65'], - [group: 'software.amazon.awssdk', name: 'sts', version: '2.10.65'], - [group: 'software.amazon.awssdk', name: 's3', version: '2.10.65'], + [group: 'software.amazon.awssdk', name: 'http-client-spi', version: '2.8.5'], + [group: 'software.amazon.awssdk', name: 'cognitoidentity', version: '2.8.5'], + [group: 'software.amazon.awssdk', name: 'sts', version: '2.8.5'], + [group: 'software.amazon.awssdk', name: 's3', version: '2.8.5'], ) testImplementation( diff --git a/src/main/java/org/broad/igv/aws/S3LoadDialog.java b/src/main/java/org/broad/igv/aws/S3LoadDialog.java index 9a9a28ca5b..ab6c447501 100644 --- a/src/main/java/org/broad/igv/aws/S3LoadDialog.java +++ b/src/main/java/org/broad/igv/aws/S3LoadDialog.java @@ -48,7 +48,6 @@ import java.util.Collections; import java.util.List; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; import static org.broad.igv.util.AmazonUtils.isObjectAccessible; @@ -104,7 +103,7 @@ private void loadButtonActionPerformed(ActionEvent e) { if (isFilePath(path)) { Triple bucketKeyTier = getBucketKeyTierFromTreePath(path); - AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); + AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucketKeyTier.getLeft(), bucketKeyTier.getMiddle()); if(!res.getObjAvailable()) { MessageUtils.showErrorMessage(res.getErrorReason(), null); return; } preLocatorPaths.add(bucketKeyTier); @@ -242,7 +241,7 @@ public void mousePressed(MouseEvent e) { if (isFilePath(selPath)) { Triple bucketKeyTier = getBucketKeyTierFromTreePath(selPath); - AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); + AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucketKeyTier.getLeft(), bucketKeyTier.getMiddle()); if(!res.getObjAvailable()) { MessageUtils.showErrorMessage(res.getErrorReason(), null); return;} ResourceLocator loc = getResourceLocatorFromBucketKey(bucketKeyTier); diff --git a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java index cd7368c3d7..c0939cf318 100644 --- a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java +++ b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java @@ -29,14 +29,10 @@ */ package org.broad.igv.ui.action; -import htsjdk.samtools.util.Tuple; -import org.apache.commons.lang3.tuple.Triple; import org.apache.log4j.Logger; -import org.broad.igv.aws.S3Object; import org.broad.igv.exceptions.HttpResponseException; import org.broad.igv.feature.genome.GenomeManager; import org.broad.igv.google.GoogleUtils; -import org.broad.igv.google.OAuthUtils; import org.broad.igv.prefs.Constants; import org.broad.igv.prefs.PreferencesManager; import org.broad.igv.ui.IGV; @@ -105,14 +101,11 @@ public void actionPerformed(ActionEvent e) { try { // If AWS support is active, check if objects are in accessible tiers via Load URL menu... if (AmazonUtils.isAwsS3Path(url)) { - Tuple S3ObjTuple = AmazonUtils.bucketAndKey(url); - Triple bucketKeyTier = Triple.of(S3ObjTuple.a, S3ObjTuple.b, ""); - - AmazonUtils.S3ObjectAccessResult res = isObjectAccessible(bucketKeyTier); - if (!res.getObjAvailable()) { - MessageUtils.showMessage(res.getErrorReason()); - return; - } + String bucket = AmazonUtils.getBucketFromS3URL(url); + String key = AmazonUtils.getKeyFromS3URL(url); + + AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucket, key); + if (!res.getObjAvailable()) { MessageUtils.showErrorMessage(res.getErrorReason(), null); return; } } } catch (NullPointerException npe) { // User has not yet done Amazon->Login sequence diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index cf7bbb2881..0ad0ebdc17 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -204,13 +204,10 @@ public static List ListBucketsForUser() { return bucketsFinalList; } - public static HeadObjectResponse getObjectMetadata(Triple S3Obj) { - String S3ObjectBucket = S3Obj.getLeft().toString(); - String S3ObjectKey = S3Obj.getMiddle().toString(); - + public static HeadObjectResponse getObjectMetadata(String bucket, String key) { HeadObjectRequest HeadObjReq = HeadObjectRequest.builder() - .bucket(S3ObjectBucket) - .key(S3ObjectKey).build(); + .bucket(bucket) + .key(key).build(); HeadObjectResponse HeadObjRes = s3Client.headObject(HeadObjReq); log.debug("getObjectMetadata(): "+HeadObjRes.toString()); return HeadObjRes; @@ -218,7 +215,7 @@ public static HeadObjectResponse getObjectMetadata(Triple S3Obj) { // Holds whether a S3 object is accessible or not and reason/error msg in case it's not. - public static class S3ObjectAccessResult { + public static class s3ObjectAccessResult { private boolean objAvailable; private String errorReason; @@ -242,18 +239,15 @@ public void setErrorReason(String errorReason) { // Determines whether the object is immediately available. // On AWS this means present in STANDARD, STANDARD_IA, INTELLIGENT_TIERING object access tiers. // Tiers GLACIER and DEEP_ARCHIVE are not immediately retrievable without action. - public static S3ObjectAccessResult isObjectAccessible(Triple S3Obj) { - String S3ObjectBucket = S3Obj.getLeft().toString(); - String S3ObjectKey = S3Obj.getMiddle().toString(); - S3ObjectAccessResult res = new S3ObjectAccessResult(); - - HeadObjectResponse S3Meta; + public static s3ObjectAccessResult isObjectAccessible(String bucket, String key) { + s3ObjectAccessResult res = new s3ObjectAccessResult(); + HeadObjectResponse s3Meta; - String S3ObjectStorageStatus = null; - String S3ObjectStorageClass; + String s3ObjectStorageStatus = null; + String s3ObjectStorageClass; - S3Meta = AmazonUtils.getObjectMetadata(S3Obj); - S3ObjectStorageClass = S3Meta.storageClass().toString(); + s3Meta = AmazonUtils.getObjectMetadata(bucket, key); + s3ObjectStorageClass = s3Meta.storageClass().toString(); // Determine in which state this object really is: // 1. Archived. @@ -266,27 +260,28 @@ public static S3ObjectAccessResult isObjectAccessible(Triple S3Obj) { // // Possible error reason messages for the users are: - String archived = "Amazon S3 object is in " + S3ObjectStorageClass + " storage tier, not accessible at this moment. " + - "Please contact your local system administrator about object: s3://" + S3ObjectBucket + "/" + S3ObjectKey; - String restoreInProgress = "Amazon S3 object is in " + S3ObjectStorageClass + " and being restored right now, please be patient, this can take up to 48h. " + - "For further enquiries about this dataset, please use the following path when communicating with your system administrator: s3://" + S3ObjectBucket + "/" + S3ObjectKey; + String archived = "Amazon S3 object is in " + s3ObjectStorageClass + " storage tier, not accessible at this moment. " + + "Please contact your local system administrator about object: s3://" + bucket + "/" + key; + String restoreInProgress = "Amazon S3 object is in " + s3ObjectStorageClass + " and being restored right now, please be patient, this can take up to 48h. " + + "For further enquiries about this dataset, please use the following path when communicating with your system administrator: s3://" + bucket + "/" + key; - if (S3ObjectStorageClass.contains("DEEP_ARCHIVE") || - S3ObjectStorageClass.contains("GLACIER")) { + if (s3ObjectStorageClass.contains("DEEP_ARCHIVE") || + s3ObjectStorageClass.contains("GLACIER")) { try { - S3ObjectStorageStatus = S3Meta.restore(); + s3ObjectStorageStatus = s3Meta.sdkHttpResponse().headers().get("x-amz-restore").toString(); + //S3ObjectStorageStatus = S3Meta.restore(); } catch(NullPointerException npe) { res.setObjAvailable(false); res.setErrorReason(archived); return res; } - if(S3ObjectStorageStatus.contains("ongoing-request=\"true\"")) { + if(s3ObjectStorageStatus.contains("ongoing-request=\"true\"")) { res.setObjAvailable(false); res.setErrorReason(restoreInProgress); // "If an archive copy is already restored, the header value indicates when Amazon S3 is scheduled to delete the object copy" - } else if(S3ObjectStorageStatus.contains("ongoing-request=\"false\"") && S3ObjectStorageStatus.contains("expiry-date=")) { + } else if(s3ObjectStorageStatus.contains("ongoing-request=\"false\"") && s3ObjectStorageStatus.contains("expiry-date=")) { res.setObjAvailable(true); } else { // The object has never been restored? @@ -389,18 +384,19 @@ public static ArrayList ListBucketObjects(String bucketName, String return objects; } - public static Tuple bucketAndKey(String S3urlString) { - AmazonS3URI s3URI = new AmazonS3URI(S3urlString); - String bucket = s3URI.getBucket(); - String key = s3URI.getKey(); + public static String getBucketFromS3URL(String s3URL) { + AmazonS3URI s3URI = new AmazonS3URI(s3URL); + return s3URI.getBucket(); - log.debug("bucketAndKey(): " + bucket + " , " + key); - return new Tuple(bucket, key); + } + + public static String getKeyFromS3URL(String s3URL) { + AmazonS3URI s3URI = new AmazonS3URI(s3URL); + return s3URI.getKey(); } // Amazon S3 Presign URLs // Also keeps an internal mapping between ResourceLocator and active/valid signed URLs. - private static String createPresignedURL(String s3Path) throws IOException { // Make sure access token are valid (refreshes token internally) OAuthProvider provider = OAuthUtils.getInstance().getProvider("Amazon"); @@ -418,11 +414,10 @@ private static String createPresignedURL(String s3Path) throws IOException { .region(getAWSREGION()) .build(); - Tuple bandk = bucketAndKey(s3Path); - String bucket = bandk.a; - String filename = bandk.b; + String bucket = getBucketFromS3URL(s3Path); + String key = getKeyFromS3URL(s3Path); - URI presigned = s3Presigner.presignS3DownloadLink(bucket, filename); + URI presigned = s3Presigner.presignS3DownloadLink(bucket, key); log.debug("AWS presigned URL from translateAmazonCloudURL is: " + presigned); return presigned.toString(); }