feat: flexible checksum updates

.changelog/9ebe24c4791541e0840da49eab6f9d97.json

@@ -0,0 +1,11 @@
+{
+    "id": "9ebe24c4-7915-41e0-840d-a49eab6f9d97",
+    "type": "feature",
+    "description": "S3 client behavior is updated to always calculate a checksum by default for operations that support it (such as PutObject or UploadPart), or require it (such as DeleteObjects). The checksum algorithm used by default now becomes CRC32. Checksum behavior can be configured using `when_supported` and `when_required` options - in code using RequestChecksumCalculation, in shared config using request_checksum_calculation, or as env variable using AWS_REQUEST_CHECKSUM_CALCULATION. The S3 client attempts to validate response checksums for all S3 API operations that support checksums. However, if the SDK has not implemented the specified checksum algorithm then this validation is skipped. Checksum validation behavior can be configured using `when_supported` and `when_required` options - in code using ResponseChecksumValidation, in shared config using response_checksum_validation, or as env variable using AWS_RESPONSE_CHECKSUM_VALIDATION.",
+    "modules": [
+        ".",
+        "config",
+        "service/internal/checksum",
+        "service/s3"
+    ]
+}

aws/checksum.go

@@ -0,0 +1,33 @@
+package aws
+
+// RequestChecksumCalculation controls request checksum calculation workflow
+type RequestChecksumCalculation int
+
+const (
+	// RequestChecksumCalculationUnset is the unset value for RequestChecksumCalculation
+	RequestChecksumCalculationUnset RequestChecksumCalculation = iota
+
+	// RequestChecksumCalculationWhenSupported indicates request checksum will be calculated
+	// if the operation supports input checksums
+	RequestChecksumCalculationWhenSupported
+
+	// RequestChecksumCalculationWhenRequired indicates request checksum will be calculated
+	// if required by the operation or if user elects to set a checksum algorithm in request
+	RequestChecksumCalculationWhenRequired
+)
+
+// ResponseChecksumValidation controls response checksum validation workflow
+type ResponseChecksumValidation int
+
+const (
+	// ResponseChecksumValidationUnset is the unset value for ResponseChecksumValidation
+	ResponseChecksumValidationUnset ResponseChecksumValidation = iota
+
+	// ResponseChecksumValidationWhenSupported indicates response checksum will be validated
+	// if the operation supports output checksums
+	ResponseChecksumValidationWhenSupported
+
+	// ResponseChecksumValidationWhenRequired indicates response checksum will only
+	// be validated if the operation requires output checksum validation
+	ResponseChecksumValidationWhenRequired
+)

aws/config.go

@@ -165,6 +165,33 @@ type Config struct {
 
 	// Controls how a resolved AWS account ID is handled for endpoint routing.
 	AccountIDEndpointMode AccountIDEndpointMode
+
+	// RequestChecksumCalculation determines when request checksum calculation is performed.
+	//
+	// There are two possible values for this setting:
+	//
+	// 1. RequestChecksumCalculationWhenSupported (default): The checksum is always calculated
+	//    if the operation supports it, regardless of whether the user sets an algorithm in the request.
+	//
+	// 2. RequestChecksumCalculationWhenRequired: The checksum is only calculated if the user
+	//    explicitly sets a checksum algorithm in the request.
+	//
+	// This setting is sourced from the environment variable AWS_REQUEST_CHECKSUM_CALCULATION
+	// or the shared config profile attribute "request_checksum_calculation".
+	RequestChecksumCalculation RequestChecksumCalculation
+
+	// ResponseChecksumValidation determines when response checksum validation is performed
+	//
+	// There are two possible values for this setting:
+	//
+	// 1. ResponseChecksumValidationWhenSupported (default): The checksum is always validated
+	//    if the operation supports it, regardless of whether the user sets the validation mode to ENABLED in request.
+	//
+	// 2. ResponseChecksumValidationWhenRequired: The checksum is only validated if the user
+	//    explicitly sets the validation mode to ENABLED in the request
+	// This variable is sourced from environment variable AWS_RESPONSE_CHECKSUM_VALIDATION or
+	// the shared config profile attribute "response_checksum_validation".
+	ResponseChecksumValidation ResponseChecksumValidation
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder

aws/middleware/user_agent.go

@@ -76,19 +76,28 @@ type UserAgentFeature string
 
 // Enumerates UserAgentFeature.
 const (
-	UserAgentFeatureResourceModel          UserAgentFeature = "A" // n/a (we don't generate separate resource types)
-	UserAgentFeatureWaiter                                  = "B"
-	UserAgentFeaturePaginator                               = "C"
-	UserAgentFeatureRetryModeLegacy                         = "D" // n/a (equivalent to standard)
-	UserAgentFeatureRetryModeStandard                       = "E"
-	UserAgentFeatureRetryModeAdaptive                       = "F"
-	UserAgentFeatureS3Transfer                              = "G"
-	UserAgentFeatureS3CryptoV1N                             = "H" // n/a (crypto client is external)
-	UserAgentFeatureS3CryptoV2                              = "I" // n/a
-	UserAgentFeatureS3ExpressBucket                         = "J"
-	UserAgentFeatureS3AccessGrants                          = "K" // not yet implemented
-	UserAgentFeatureGZIPRequestCompression                  = "L"
-	UserAgentFeatureProtocolRPCV2CBOR                       = "M"
+	UserAgentFeatureResourceModel                 UserAgentFeature = "A" // n/a (we don't generate separate resource types)
+	UserAgentFeatureWaiter                                         = "B"
+	UserAgentFeaturePaginator                                      = "C"
+	UserAgentFeatureRetryModeLegacy                                = "D" // n/a (equivalent to standard)
+	UserAgentFeatureRetryModeStandard                              = "E"
+	UserAgentFeatureRetryModeAdaptive                              = "F"
+	UserAgentFeatureS3Transfer                                     = "G"
+	UserAgentFeatureS3CryptoV1N                                    = "H" // n/a (crypto client is external)
+	UserAgentFeatureS3CryptoV2                                     = "I" // n/a
+	UserAgentFeatureS3ExpressBucket                                = "J"
+	UserAgentFeatureS3AccessGrants                                 = "K" // not yet implemented
+	UserAgentFeatureGZIPRequestCompression                         = "L"
+	UserAgentFeatureProtocolRPCV2CBOR                              = "M"
+	UserAgentFeatureRequestChecksumCRC32                           = "U"
+	UserAgentFeatureRequestChecksumCRC32C                          = "V"
+	UserAgentFeatureRequestChecksumCRC64                           = "W"
+	UserAgentFeatureRequestChecksumSHA1                            = "X"
+	UserAgentFeatureRequestChecksumSHA256                          = "Y"
+	UserAgentFeatureRequestChecksumWhenSupported                   = "Z"
+	UserAgentFeatureRequestChecksumWhenRequired                    = "a"
+	UserAgentFeatureResponseChecksumWhenSupported                  = "b"
+	UserAgentFeatureResponseChecksumWhenRequired                   = "c"
 )
 
 // RequestUserAgent is a build middleware that set the User-Agent for the request.

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java

@@ -84,6 +84,10 @@ public class AddAwsConfigFields implements GoIntegration {
 
     private static final String SDK_ACCOUNTID_ENDPOINT_MODE = "AccountIDEndpointMode";
 
+    private static final String REQUEST_CHECKSUM_CALCULATION = "RequestChecksumCalculation";
+
+    private static final String RESPONSE_CHECKSUM_VALIDATION = "ResponseChecksumValidation";
+
     private static final List<AwsConfigField> AWS_CONFIG_FIELDS = ListUtils.of(
             AwsConfigField.builder()
                     .name(REGION_CONFIG_NAME)
@@ -244,6 +248,18 @@ public class AddAwsConfigFields implements GoIntegration {
                     .type(SdkGoTypes.Aws.AccountIDEndpointMode)
                     .documentation("Indicates how aws account ID is applied in endpoint2.0 routing")
                     .servicePredicate(AccountIDEndpointRouting::hasAccountIdEndpoints)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(REQUEST_CHECKSUM_CALCULATION)
+                    .type(SdkGoTypes.Aws.RequestChecksumCalculation)
+                    .documentation("Indicates how user opt-in/out request checksum calculation")
+                    .servicePredicate(AwsHttpChecksumGenerator::hasInputChecksumTrait)
+                    .build(),
+            AwsConfigField.builder()
+                    .name(RESPONSE_CHECKSUM_VALIDATION)
+                    .type(SdkGoTypes.Aws.ResponseChecksumValidation)
+                    .documentation("Indicates how user opt-in/out response checksum validation")
+                    .servicePredicate(AwsHttpChecksumGenerator::hasOutputChecksumTrait)
                     .build()
     );
 

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpChecksumGenerator.java

@@ -22,6 +22,7 @@
 import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar;
 import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
 import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.knowledge.TopDownIndex;
 import software.amazon.smithy.model.shapes.MemberShape;
 import software.amazon.smithy.model.shapes.OperationShape;
 import software.amazon.smithy.model.shapes.ServiceShape;
@@ -73,9 +74,7 @@ public byte getOrder() {
     @Override
     public void processFinalizedModel(GoSettings settings, Model model) {
         ServiceShape service = settings.getService(model);
-        for (ShapeId operationId : service.getAllOperations()) {
-            final OperationShape operation = model.expectShape(operationId, OperationShape.class);
-
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
             // Create a symbol provider because one is not available in this call.
             SymbolProvider symbolProvider = GoCodegenPlugin.createSymbolProvider(model, settings);
 
@@ -128,8 +127,7 @@ public void writeAdditionalFiles(
         boolean supportsComputeInputChecksumsWorkflow = false;
         boolean supportsChecksumValidationWorkflow = false;
 
-        for (ShapeId operationID : service.getAllOperations()) {
-            OperationShape operation = model.expectShape(operationID, OperationShape.class);
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
             if (!hasChecksumTrait(model, service, operation)) {
                 continue;
             }
@@ -178,26 +176,44 @@ public List<RuntimeClientPlugin> getClientPlugins() {
     }
 
     // return true if operation shape is decorated with `httpChecksum` trait.
-    private boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    private static boolean hasChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         return operation.hasTrait(HttpChecksumTrait.class);
     }
 
-    private boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    private static boolean hasInputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         if (!hasChecksumTrait(model, service, operation)) {
             return false;
         }
         HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
         return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
     }
 
-    private boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
+    public static boolean hasInputChecksumTrait(Model model, ServiceShape service) {
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
+                if (hasInputChecksumTrait(model, service, operation)) {
+                    return true;
+                }
+        }
+        return false;
+    }
+
+        private static boolean hasOutputChecksumTrait(Model model, ServiceShape service, OperationShape operation) {
         if (!hasChecksumTrait(model, service, operation)) {
             return false;
         }
         HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
         return trait.getRequestValidationModeMember().isPresent() && !trait.getResponseAlgorithms().isEmpty();
     }
 
+    public static boolean hasOutputChecksumTrait(Model model, ServiceShape service) {
+        for (OperationShape operation : TopDownIndex.of(model).getContainedOperations(service)) {
+            if (hasOutputChecksumTrait(model, service, operation)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private boolean isS3ServiceShape(Model model, ServiceShape service) {
         String serviceId = service.expectTrait(ServiceTrait.class).getSdkId();
         return serviceId.equalsIgnoreCase("S3");
@@ -244,6 +260,7 @@ private void writeInputMiddlewareHelper(
                                     return $T(stack, $T{
                                         GetAlgorithm: $L,
                                         RequireChecksum: $L,
+                                        RequestChecksumCalculation: options.RequestChecksumCalculation,
                                         EnableTrailingChecksum: $L,
                                         EnableComputeSHA256PayloadHash: true,
                                         EnableDecodedContentLengthHeader: $L,
@@ -284,6 +301,7 @@ private void writeOutputMiddlewareHelper(
                     writer.write("""
                                     return $T(stack, $T{
                                         GetValidationMode: $L,
+                                        ResponseChecksumValidation: options.ResponseChecksumValidation,
                                         ValidationAlgorithms: $L,
                                         IgnoreMultipartValidation: $L,
                                         LogValidationSkipped: true,
@@ -293,7 +311,6 @@ private void writeOutputMiddlewareHelper(
                                     AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
                             SymbolUtils.createValueSymbolBuilder("OutputMiddlewareOptions",
                                     AwsGoDependency.SERVICE_INTERNAL_CHECKSUM).build(),
-
                             getRequestValidationModeAccessorFuncName(operationName),
                             convertToGoStringList(responseAlgorithms),
                             ignoreMultipartChecksumValidationMap.getOrDefault(

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsHttpPresignURLClientGenerator.java

@@ -23,6 +23,7 @@
 import java.util.TreeSet;
 import software.amazon.smithy.aws.go.codegen.customization.AwsCustomGoDependency;
 import software.amazon.smithy.aws.go.codegen.customization.PresignURLAutoFill;
+import software.amazon.smithy.aws.traits.HttpChecksumTrait;
 import software.amazon.smithy.aws.traits.ServiceTrait;
 import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait;
 import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait;
@@ -67,7 +68,7 @@ public class AwsHttpPresignURLClientGenerator implements GoIntegration {
     private static final String CONVERT_TO_PRESIGN_MIDDLEWARE_NAME = "convertToPresignMiddleware";
     private static final String CONVERT_TO_PRESIGN_TYPE_NAME = "presignConverter";
     private static final String NOP_HTTP_CLIENT_OPTION_FUNC_NAME = "withNopHTTPClientAPIOption";
-
+    private static final String NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME = "withNoDefaultChecksumAPIOption";
     private static final String PRESIGN_CLIENT = "PresignClient";
     private static final Symbol presignClientSymbol = buildSymbol(PRESIGN_CLIENT, true);
 
@@ -218,7 +219,11 @@ public void writeAdditionalFiles(
             writeConvertToPresignMiddleware(writer, model, symbolProvider, serviceShape);
         });
 
+        boolean supportsComputeInputChecksumsWorkflow = false;
         for (OperationShape operationShape : TopDownIndex.of(model).getContainedOperations(serviceShape)) {
+            if (hasInputChecksumTrait(operationShape)) {
+                supportsComputeInputChecksumsWorkflow = true;
+            }
             if (!validOperations.contains(operationShape.getId())) {
                 continue;
             }
@@ -231,6 +236,10 @@ public void writeAdditionalFiles(
                 writeS3AddAsUnsignedPayloadHelper(writer, model, symbolProvider, serviceShape, operationShape);
             });
         }
+
+        if (supportsComputeInputChecksumsWorkflow) {
+            writePresignRequestChecksumConfigHelpers(settings, goDelegator);
+        }
     }
 
     private void writePresignOperationFunction(
@@ -263,6 +272,10 @@ private void writePresignOperationFunction(
 
                     writer.write("clientOptFns := append(options.ClientOptions, $L)", NOP_HTTP_CLIENT_OPTION_FUNC_NAME);
                     writer.write("");
+                    if (hasInputChecksumTrait(operationShape)) {
+                        writer.write("clientOptFns = append(options.ClientOptions, $L)", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME);
+                        writer.write("");
+                    }
 
                     writer.openBlock("result, _, err := c.client.invokeOperation(ctx, $S, params, clientOptFns,", ")",
                             operationSymbol.getName(), () -> {
@@ -572,6 +585,29 @@ private void writePresignClientHelpers(
         writer.write("");
     }
 
+    private void writePresignRequestChecksumConfigHelpers(
+            GoSettings settings,
+            GoDelegator goDelegator
+    ) {
+        goDelegator.useFileWriter("api_client.go", settings.getModuleName(), goTemplate("""
+            func $fn:L(options *Options) {
+                options.RequestChecksumCalculation = $requestChecksumCalculationWhenRequired:T
+            }""",
+                Map.of(
+                        "fn", NO_DEFAULT_CHECKSUM_OPTION_FUNC_NAME,
+                        "requestChecksumCalculationWhenRequired",
+                        AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculationWhenRequired")
+                    )));
+    }
+
+    private static boolean hasInputChecksumTrait(OperationShape operation) {
+        if (!operation.hasTrait(HttpChecksumTrait.class)) {
+            return false;
+        }
+        HttpChecksumTrait trait = operation.expectTrait(HttpChecksumTrait.class);
+        return trait.isRequestChecksumRequired() || trait.getRequestAlgorithmMember().isPresent();
+    }
+
     /**
      * Writes the presigner interface used by the presign url client
      */

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/SdkGoTypes.java

@@ -39,6 +39,8 @@ public static final class Aws {
         public static final Symbol AccountIDEndpointModeRequired = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeRequired");
         public static final Symbol AccountIDEndpointModeDisabled = AwsGoDependency.AWS_CORE.valueSymbol("AccountIDEndpointModeDisabled");
 
+        public static final Symbol RequestChecksumCalculation = AwsGoDependency.AWS_CORE.valueSymbol("RequestChecksumCalculation");
+        public static final Symbol ResponseChecksumValidation = AwsGoDependency.AWS_CORE.valueSymbol("ResponseChecksumValidation");
 
         public static final class Middleware {
             public static final Symbol GetRequiresLegacyEndpoints = AwsGoDependency.AWS_MIDDLEWARE.valueSymbol("GetRequiresLegacyEndpoints");