Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

check should upload data into the bucket path provided #11795

Merged
merged 1 commit into from
Apr 9, 2022
Merged

check should upload data into the bucket path provided #11795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,10 @@ public AirbyteConnectionStatus check(final JsonNode config) {
final AmazonS3 s3Client = destinationConfig.getS3Client();

// Test single upload (for small files) permissions
S3Destination.testSingleUpload(s3Client, destinationConfig.getBucketName());
S3Destination.testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath());

// Test multipart upload with stream transfer manager
S3Destination.testMultipartUpload(s3Client, destinationConfig.getBucketName());
S3Destination.testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath());

return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED);
} catch (final Exception e) {
Expand Down
19 changes: 12 additions & 7 deletions ...rs/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ public AirbyteConnectionStatus check(final JsonNode config) {
S3Destination.attemptS3WriteAndDelete(storageOperations, destinationConfig, destinationConfig.getBucketName());

// Test single upload (for small files) permissions
testSingleUpload(s3Client, destinationConfig.getBucketName());
testSingleUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath());

// Test multipart upload with stream transfer manager
testMultipartUpload(s3Client, destinationConfig.getBucketName());
testMultipartUpload(s3Client, destinationConfig.getBucketName(), destinationConfig.getBucketPath());

return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED);
} catch (final Exception e) {
Expand All @@ -77,9 +77,12 @@ public AirbyteConnectionStatus check(final JsonNode config) {
}
}

public static void testSingleUpload(final AmazonS3 s3Client, final String bucketName) {
public static void testSingleUpload(final AmazonS3 s3Client, final String bucketName, String bucketPath) {
LOGGER.info("Started testing if all required credentials assigned to user for single file uploading");
final String testFile = "test_" + System.currentTimeMillis();
if (bucketPath.endsWith("/")) {
throw new RuntimeException("Bucket Path should not end with /");
}
final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis();
try {
s3Client.putObject(bucketName, testFile, "this is a test file");
} finally {
Expand All @@ -88,10 +91,12 @@ public static void testSingleUpload(final AmazonS3 s3Client, final String bucket
LOGGER.info("Finished checking for normal upload mode");
}

public static void testMultipartUpload(final AmazonS3 s3Client, final String bucketName) throws IOException {
public static void testMultipartUpload(final AmazonS3 s3Client, final String bucketName, String bucketPath) throws IOException {
LOGGER.info("Started testing if all required credentials assigned to user for multipart upload");

final String testFile = "test_" + System.currentTimeMillis();
if (bucketPath.endsWith("/")) {
throw new RuntimeException("Bucket Path should not end with /");
}
final String testFile = bucketPath + "/" + "test_" + System.currentTimeMillis();
final StreamTransferManager manager = StreamTransferManagerHelper.getDefault(
bucketName,
testFile,
Expand Down