From 65bb8c07f52f98dbd702f5b4c23686708aeffa9a Mon Sep 17 00:00:00 2001 From: subodh Date: Thu, 7 Apr 2022 15:37:57 +0530 Subject: [PATCH] check should upload data into the bucket path provided --- .../destination/gcs/GcsDestination.java | 4 ++-- .../destination/s3/S3Destination.java | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java index 5c3685ac2243..ef007eb0a142 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java @@ -46,10 +46,10 @@ public AirbyteConnectionStatus check(final JsonNode config) { final AmazonS3 s3Client = destinationConfig.getS3Client(); // Test single upload (for small files) permissions - S3Destination.testSingleUpload(s3Client, destinationConfig.getBucketName()); + S3Destination.testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); // Test multipart upload with stream transfer manager - S3Destination.testMultipartUpload(s3Client, destinationConfig.getBucketName()); + S3Destination.testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (final Exception e) { diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java index 2953da5f9c5d..f37d17f7fae5 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java @@ -62,10 +62,10 @@ public AirbyteConnectionStatus check(final JsonNode config) { S3Destination.attemptS3WriteAndDelete(storageOperations, destinationConfig, destinationConfig.getBucketName()); // Test single upload (for small files) permissions - testSingleUpload(s3Client, destinationConfig.getBucketName()); + testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); // Test multipart upload with stream transfer manager - testMultipartUpload(s3Client, destinationConfig.getBucketName()); + testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath()); return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); } catch (final Exception e) { @@ -77,9 +77,12 @@ public AirbyteConnectionStatus check(final JsonNode config) { } } - public static void testSingleUpload(final AmazonS3 s3Client, final String bucketName) { + public static void testSingleUpload(final AmazonS3 s3Client, final String bucketName, String bucketPath) { LOGGER.info("Started testing if all required credentials assigned to user for single file uploading"); - final String testFile = "test_" + System.currentTimeMillis(); + if (bucketPath.endsWith("/")) { + throw new RuntimeException("Bucket Path should not end with /"); + } + final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis(); try { s3Client.putObject(bucketName, testFile, "this is a test file"); } finally { @@ -88,10 +91,12 @@ public static void testSingleUpload(final AmazonS3 s3Client, final String bucket LOGGER.info("Finished checking for normal upload mode"); } - public static void testMultipartUpload(final AmazonS3 s3Client, final String bucketName) throws IOException { + public static void testMultipartUpload(final AmazonS3 s3Client, final String bucketName, String bucketPath) throws IOException { LOGGER.info("Started testing if all required credentials assigned to user for multipart upload"); - - final String testFile = "test_" + System.currentTimeMillis(); + if (bucketPath.endsWith("/")) { + throw new RuntimeException("Bucket Path should not end with /"); + } + final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis(); final StreamTransferManager manager = StreamTransferManagerHelper.getDefault( bucketName, testFile,