Added support for privatelink.

.changes/next-release/feature-AmazonS3-1112ce8.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "Amazon S3",
+    "contributor": "",
+    "description": "Amazon S3 now supports AWS PrivateLink, providing direct access to S3 via a private endpoint within your virtual private network."
+}

core/aws-core/src/main/java/software/amazon/awssdk/awscore/client/handler/AwsClientHandlerUtils.java

@@ -63,6 +63,8 @@ static <InputT extends SdkRequest, OutputT extends SdkResponse> ExecutionContext
         ClientExecutionParams<InputT, OutputT> executionParams,
         SdkClientConfiguration clientConfig,
         ExecutionAttributes executionAttributes) {
+        // Note: This is currently copied to DefaultS3Presigner and other presigners. Don't edit this without considering those
+        // as well. TODO: Probably don't copy all of this manually.
 
         SdkRequest originalRequest = executionParams.getInput();
         AwsCredentialsProvider clientCredentials = clientConfig.option(AwsClientOption.CREDENTIALS_PROVIDER);
@@ -92,8 +94,8 @@ static <InputT extends SdkRequest, OutputT extends SdkResponse> ExecutionContext
             .putAttribute(SdkExecutionAttribute.CLIENT_TYPE, clientConfig.option(SdkClientOption.CLIENT_TYPE))
             .putAttribute(SdkExecutionAttribute.SERVICE_NAME, clientConfig.option(SdkClientOption.SERVICE_NAME))
             .putAttribute(SdkExecutionAttribute.OPERATION_NAME, executionParams.getOperationName())
-            .putAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN,
-                          clientConfig.option(SdkClientOption.ENDPOINT_OVERRIDDEN));
+            .putAttribute(SdkExecutionAttribute.CLIENT_ENDPOINT, clientConfig.option(SdkClientOption.ENDPOINT))
+            .putAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN, clientConfig.option(SdkClientOption.ENDPOINT_OVERRIDDEN));
 
         ExecutionInterceptorChain executionInterceptorChain =
                 new ExecutionInterceptorChain(clientConfig.option(SdkClientOption.EXECUTION_INTERCEPTORS));

core/sdk-core/src/main/java/software/amazon/awssdk/core/interceptor/SdkExecutionAttribute.java

@@ -15,6 +15,7 @@
 
 package software.amazon.awssdk.core.interceptor;
 
+import java.net.URI;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 import software.amazon.awssdk.core.ClientType;
 import software.amazon.awssdk.core.ServiceConfiguration;
@@ -58,7 +59,13 @@ public class SdkExecutionAttribute {
      * If true indicates that the configured endpoint of the client is a value that was supplied as an override and not
      * generated from regional metadata.
      */
-    public static final ExecutionAttribute<Boolean> ENDPOINT_OVERRIDDEN = new ExecutionAttribute<>("EndpointOverride");
+    public static final ExecutionAttribute<Boolean> ENDPOINT_OVERRIDDEN = new ExecutionAttribute<>("EndpointOverridden");
+
+    /**
+     * The endpoint resolved at client creation time. This is either the endpointOverride (if {@link #ENDPOINT_OVERRIDDEN} is
+     * true) or the endpoint derived from the region metadata (if {@link #ENDPOINT_OVERRIDDEN} is false).
+     */
+    public static final ExecutionAttribute<URI> CLIENT_ENDPOINT = new ExecutionAttribute<>("EndpointOverride");
 
     protected SdkExecutionAttribute() {
     }

services/s3/src/main/java/software/amazon/awssdk/services/s3/S3Utilities.java

@@ -148,7 +148,6 @@ public URL getUrl(Consumer<GetUrlRequest.Builder> getUrlRequest) {
     public URL getUrl(GetUrlRequest getUrlRequest) {
         Region resolvedRegion = resolveRegionForGetUrl(getUrlRequest);
         URI resolvedEndpoint = resolveEndpoint(getUrlRequest.endpoint(), resolvedRegion);
-        boolean endpointOverridden = getUrlRequest.endpoint() != null;
 
         SdkHttpFullRequest marshalledRequest = createMarshalledRequest(getUrlRequest, resolvedEndpoint);
 
@@ -162,7 +161,7 @@ public URL getUrl(GetUrlRequest getUrlRequest) {
                                                                              .request(marshalledRequest)
                                                                              .originalRequest(getObjectRequest)
                                                                              .region(resolvedRegion)
-                                                                             .endpointOverridden(endpointOverridden)
+                                                                             .endpointOverride(getUrlRequest.endpoint())
                                                                              .serviceConfiguration(s3Configuration)
                                                                              .build();
 

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/endpoints/S3AccessPointEndpointResolver.java

@@ -74,12 +74,13 @@ public ConfiguredS3SdkHttpRequest applyEndpointConfiguration(S3EndpointResolverC
                                   + "appear to be a valid S3 access point ARN.");
 
         URI accessPointUri = getUriForAccessPointResource(context, arnRegion, clientPartitionMetadata, s3EndpointResource);
-        String key = context.originalRequest().getValueForField("Key", String.class).orElse(null);
+        String path = buildPath(accessPointUri, context);
+
         SdkHttpRequest httpRequest = context.request().toBuilder()
                                             .protocol(accessPointUri.getScheme())
                                             .host(accessPointUri.getHost())
                                             .port(accessPointUri.getPort())
-                                            .encodedPath(key)
+                                            .encodedPath(path)
                                             .build();
 
         String signingServiceModification = s3EndpointResource.parentS3Resource()
@@ -94,6 +95,23 @@ public ConfiguredS3SdkHttpRequest applyEndpointConfiguration(S3EndpointResolverC
                                          .build();
     }
 
+    private String buildPath(URI accessPointUri, S3EndpointResolverContext context) {
+        String key = context.originalRequest().getValueForField("Key", String.class).orElse(null);
+
+        StringBuilder pathBuilder = new StringBuilder();
+        if (accessPointUri.getPath() != null) {
+            pathBuilder.append(accessPointUri.getPath());
+        }
+
+        if (key != null) {
+            if (pathBuilder.length() > 0) {
+                pathBuilder.append('/');
+            }
+            pathBuilder.append(key);
+        }
+        return pathBuilder.length() > 0 ? pathBuilder.toString() : null;
+    }
+
     private String validateConfiguration(S3EndpointResolverContext context, S3Resource s3Resource) {
         Region region = context.region();
         String arnRegion = s3Resource.region().orElseThrow(() -> new IllegalArgumentException(
@@ -113,12 +131,6 @@ private String validateConfiguration(S3EndpointResolverContext context, S3Resour
                                                + "addressing enabled.");
         }
 
-        if (context.endpointOverridden()) {
-            throw new IllegalArgumentException("An access point ARN cannot be passed as a bucket parameter to an S3"
-                                               + " operation if the S3 client has been configured with an endpoint "
-                                               + "override.");
-        }
-
         if (!isArnRegionEnabled(serviceConfiguration) && clientRegionDiffersFromArnRegion(region, arnRegion)) {
             throw new IllegalArgumentException(
                 String.format("The region field of the ARN being passed as a bucket parameter to an S3 operation "
@@ -157,11 +169,6 @@ private String getBucketName(S3EndpointResolverContext context) {
     private URI getUriForAccessPointResource(S3EndpointResolverContext context, String arnRegion,
                                                     PartitionMetadata clientPartitionMetadata,
                                                     S3AccessPointResource s3EndpointResource) {
-
-        boolean dualstackEnabled = isDualstackEnabled(context.serviceConfiguration());
-        boolean fipsRegionProvided = isFipsRegionProvided(context.region().toString(), arnRegion,
-                                                          isArnRegionEnabled(context.serviceConfiguration()));
-
         String accountId = s3EndpointResource.accountId().orElseThrow(() -> new IllegalArgumentException(
             "An S3 access point ARN must have an account ID"));
         String accessPointName = s3EndpointResource.accessPointName();
@@ -170,7 +177,11 @@ private URI getUriForAccessPointResource(S3EndpointResolverContext context, Stri
             return getOutpostAccessPointUri(context, arnRegion, clientPartitionMetadata, s3EndpointResource);
         }
 
+        boolean dualstackEnabled = isDualstackEnabled(context.serviceConfiguration());
+        boolean fipsRegionProvided = isFipsRegionProvided(context.region().toString(), arnRegion,
+                                                          isArnRegionEnabled(context.serviceConfiguration()));
         return S3AccessPointBuilder.create()
+                                   .endpointOverride(context.endpointOverride())
                                    .accessPointName(accessPointName)
                                    .accountId(accountId)
                                    .fipsEnabled(fipsRegionProvided)
@@ -200,6 +211,7 @@ private URI getOutpostAccessPointUri(S3EndpointResolverContext context, String a
 
         S3OutpostResource parentResource = (S3OutpostResource) s3EndpointResource.parentS3Resource().get();
         return S3OutpostAccessPointBuilder.create()
+                                          .endpointOverride(context.endpointOverride())
                                           .accountId(s3EndpointResource.accountId().get())
                                           .outpostId(parentResource.outpostId())
                                           .region(arnRegion)

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/endpoints/S3EndpointResolverContext.java

@@ -15,6 +15,7 @@
 
 package software.amazon.awssdk.services.s3.internal.endpoints;
 
+import java.net.URI;
 import java.util.Objects;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.core.SdkRequest;
@@ -31,14 +32,14 @@ public final class S3EndpointResolverContext {
     private final SdkRequest originalRequest;
     private final Region region;
     private final S3Configuration serviceConfiguration;
-    private final boolean endpointOverridden;
+    private final URI endpointOverride;
 
     private S3EndpointResolverContext(Builder builder) {
         this.request = builder.request;
         this.originalRequest = builder.originalRequest;
         this.region = builder.region;
         this.serviceConfiguration = builder.serviceConfiguration;
-        this.endpointOverridden = builder.endpointOverridden;
+        this.endpointOverride = builder.endpointOverride;
     }
 
     public static Builder builder() {
@@ -61,8 +62,8 @@ public S3Configuration serviceConfiguration() {
         return serviceConfiguration;
     }
 
-    public boolean endpointOverridden() {
-        return endpointOverridden;
+    public URI endpointOverride() {
+        return endpointOverride;
     }
 
 
@@ -75,7 +76,7 @@ public boolean equals(Object o) {
             return false;
         }
         S3EndpointResolverContext that = (S3EndpointResolverContext) o;
-        return endpointOverridden == that.endpointOverridden &&
+        return Objects.equals(endpointOverride, that.endpointOverride) &&
                Objects.equals(request, that.request) &&
                Objects.equals(originalRequest, that.originalRequest) &&
                Objects.equals(region, that.region) &&
@@ -89,12 +90,12 @@ public int hashCode() {
         hashCode = 31 * hashCode + Objects.hashCode(originalRequest());
         hashCode = 31 * hashCode + Objects.hashCode(region());
         hashCode = 31 * hashCode + Objects.hashCode(serviceConfiguration());
-        hashCode = 31 * hashCode + Objects.hashCode(endpointOverridden());
+        hashCode = 31 * hashCode + Objects.hashCode(endpointOverride());
         return hashCode;
     }
 
     public Builder toBuilder() {
-        return builder().endpointOverridden(endpointOverridden)
+        return builder().endpointOverride(endpointOverride)
                         .request(request)
                         .originalRequest(originalRequest)
                         .region(region)
@@ -106,7 +107,7 @@ public static final class Builder {
         private SdkRequest originalRequest;
         private Region region;
         private S3Configuration serviceConfiguration;
-        private boolean endpointOverridden;
+        private URI endpointOverride;
 
         private Builder() {
         }
@@ -131,8 +132,8 @@ public Builder serviceConfiguration(S3Configuration serviceConfiguration) {
             return this;
         }
 
-        public Builder endpointOverridden(boolean endpointOverridden) {
-            this.endpointOverridden = endpointOverridden;
+        public Builder endpointOverride(URI endpointOverride) {
+            this.endpointOverride = endpointOverride;
             return this;
         }
 

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/handlers/EndpointAddressInterceptor.java

@@ -15,6 +15,7 @@
 
 package software.amazon.awssdk.services.s3.internal.handlers;
 
+import java.net.URI;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute;
 import software.amazon.awssdk.awscore.AwsExecutionAttribute;
@@ -35,16 +36,19 @@ public final class EndpointAddressInterceptor implements ExecutionInterceptor {
     public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
                                             ExecutionAttributes executionAttributes) {
 
-        boolean endpointOverride =
+        boolean endpointOverridden =
             Boolean.TRUE.equals(executionAttributes.getAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN));
+        URI endpointOverride = endpointOverridden ? executionAttributes.getAttribute(SdkExecutionAttribute.CLIENT_ENDPOINT)
+                                                  : null;
+
         S3Configuration serviceConfiguration =
             (S3Configuration) executionAttributes.getAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG);
         S3EndpointResolverContext resolverContext =
             S3EndpointResolverContext.builder()
                                      .request(context.httpRequest())
                                      .originalRequest(context.request())
                                      .region(executionAttributes.getAttribute(AwsExecutionAttribute.AWS_REGION))
-                                     .endpointOverridden(endpointOverride)
+                                     .endpointOverride(endpointOverride)
                                      .serviceConfiguration(serviceConfiguration)
                                      .build();
 

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/presigner/DefaultS3Presigner.java

@@ -115,6 +115,7 @@ public final class DefaultS3Presigner extends DefaultSdkPresigner implements S3P
     private final UploadPartRequestMarshaller uploadPartRequestMarshaller;
     private final CompleteMultipartUploadRequestMarshaller completeMultipartUploadRequestMarshaller;
     private final AbortMultipartUploadRequestMarshaller abortMultipartUploadRequestMarshaller;
+    private final SdkClientConfiguration clientConfiguration;
 
     private DefaultS3Presigner(Builder b) {
         super(b);
@@ -128,9 +129,11 @@ private DefaultS3Presigner(Builder b) {
 
         this.clientInterceptors = initializeInterceptors();
 
+        this.clientConfiguration = createClientConfiguration();
+
         // Copied from DefaultS3Client#init
         AwsS3ProtocolFactory protocolFactory = AwsS3ProtocolFactory.builder()
-                                                                   .clientConfiguration(createClientConfiguration())
+                                                                   .clientConfiguration(clientConfiguration)
                                                                    .build();
 
         // Copied from DefaultS3Client#getObject
@@ -308,6 +311,9 @@ private ExecutionContext createExecutionContext(PresignRequest presignRequest, S
             .putAttribute(SdkExecutionAttribute.SERVICE_NAME, SERVICE_NAME)
             .putAttribute(SdkExecutionAttribute.OPERATION_NAME, operationName)
             .putAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG, serviceConfiguration())
+            .putAttribute(SdkExecutionAttribute.CLIENT_ENDPOINT, clientConfiguration.option(SdkClientOption.ENDPOINT))
+            .putAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN,
+                          clientConfiguration.option(SdkClientOption.ENDPOINT_OVERRIDDEN))
             .putAttribute(PRESIGNER_EXPIRATION, signatureExpiration);
 
         ExecutionInterceptorChain executionInterceptorChain = new ExecutionInterceptorChain(clientInterceptors);

services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/resource/S3AccessPointBuilder.java

@@ -22,6 +22,7 @@
 import java.util.regex.Pattern;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.utils.Validate;
 
 /**
  * This class is used to construct an endpoint host for an S3 access point.
@@ -31,6 +32,7 @@ public class S3AccessPointBuilder {
     private static final Pattern HOSTNAME_COMPLIANT_PATTERN = Pattern.compile("[A-Za-z0-9\\-]+");
     private static final int HOSTNAME_MAX_LENGTH = 63;
 
+    private URI endpointOverride;
     private Boolean dualstackEnabled;
     private String accessPointName;
     private String region;
@@ -46,6 +48,14 @@ public static S3AccessPointBuilder create() {
         return new S3AccessPointBuilder();
     }
 
+    /**
+     * The endpoint override configured on the client (null if no endpoint override was set).
+     */
+    public S3AccessPointBuilder endpointOverride(URI endpointOverride) {
+        this.endpointOverride = endpointOverride;
+        return this;
+    }
+
     /**
      * Enable DualStack endpoint.
      */
@@ -109,16 +119,36 @@ public URI toUri() {
         validateHostnameCompliant(accountId, "accountId");
         validateHostnameCompliant(accessPointName, "accessPointName");
 
-        String fipsSegment = Boolean.TRUE.equals(fipsEnabled) ? "fips-" : "";
+        String uri;
+        if (endpointOverride == null) {
+            String fipsSegment = Boolean.TRUE.equals(fipsEnabled) ? "fips-" : "";
+            String dualStackSegment = Boolean.TRUE.equals(dualstackEnabled) ? ".dualstack" : "";
+
+            uri = String.format("%s://%s-%s.s3-accesspoint%s.%s%s.%s", protocol, urlEncode(accessPointName),
+                                accountId, dualStackSegment, fipsSegment, region, domain);
+        } else {
+            Validate.isTrue(!Boolean.TRUE.equals(fipsEnabled),
+                            "FIPS regions are not supported with an endpoint override specified");
+            Validate.isTrue(!Boolean.TRUE.equals(dualstackEnabled),
+                            "Dual stack is not supported with an endpoint override specified");
+
+            StringBuilder uriSuffix = new StringBuilder(endpointOverride.getHost());
+            if (endpointOverride.getPort() > 0) {
+                uriSuffix.append(":").append(endpointOverride.getPort());
+            }
+            if (endpointOverride.getPath() != null) {
+                uriSuffix.append(endpointOverride.getPath());
+            }
+
+            uri = String.format("%s://%s-%s.%s", protocol, urlEncode(accessPointName), accountId, uriSuffix);
+        }
 
-        String dualStackSegment = Boolean.TRUE.equals(dualstackEnabled) ? ".dualstack" : "";
-        String uriString = String.format("%s://%s-%s.s3-accesspoint%s.%s%s.%s", protocol, urlEncode(accessPointName), accountId,
-                                         dualStackSegment, fipsSegment, region, domain);
-        URI uri = URI.create(uriString);
-        if (uri.getHost() == null) {
-            throw SdkClientException.create("ARN region (" + region + ") resulted in an invalid URI:" + uri);
+        URI result = URI.create(uri);
+        if (result.getHost() == null) {
+            throw SdkClientException.create("Request resulted in an invalid URI: " + result);
         }
-        return uri;
+
+        return result;
     }
 
     private static void validateHostnameCompliant(String hostnameComponent, String paramName) {