Support trailing checksum with no signing

include/aws/s3/private/s3_client_impl.h

@@ -169,6 +169,10 @@ struct aws_s3_client_vtable {
 
     struct aws_parallel_input_stream *(
         *parallel_input_stream_new_from_file)(struct aws_allocator *allocator, struct aws_byte_cursor file_name);
+
+    struct aws_http_stream *(*http_connection_make_request)(
+        struct aws_http_connection *client_connection,
+        const struct aws_http_make_request_options *options);
 };
 
 struct aws_s3_upload_part_timeout_stats {
@@ -524,6 +528,9 @@ void aws_s3_client_update_upload_part_timeout(
     struct aws_s3_request *finished_upload_part_request,
     int finished_error_code);
 
+AWS_S3_API
+extern struct aws_s3_client_vtable g_s3_client_default_vtable;
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_CLIENT_IMPL_H */

include/aws/s3/private/s3_request.h

@@ -212,6 +212,9 @@ struct aws_s3_request {
 
         /* The metrics for the request telemetry */
         struct aws_s3_request_metrics *metrics;
+
+        /* The request is required to have the unsigned payload */
+        uint32_t require_streaming_unsigned_payload_header : 1;
     } send_data;
 
     /* When true, response headers from the request will be stored in the request's response_headers variable. */

source/s3_client.c

@@ -132,7 +132,7 @@ static struct aws_s3_meta_request *s_s3_client_meta_request_factory_default(
     struct aws_s3_client *client,
     const struct aws_s3_meta_request_options *options);
 
-static struct aws_s3_client_vtable s_s3_client_default_vtable = {
+struct aws_s3_client_vtable g_s3_client_default_vtable = {
     .meta_request_factory = s_s3_client_meta_request_factory_default,
     .acquire_http_connection = aws_http_connection_manager_acquire_connection,
     .get_host_address_count = aws_host_resolver_get_host_address_count,
@@ -141,6 +141,7 @@ static struct aws_s3_client_vtable s_s3_client_default_vtable = {
     .endpoint_shutdown_callback = s_s3_client_endpoint_shutdown_callback,
     .finish_destroy = s_s3_client_finish_destroy_default,
     .parallel_input_stream_new_from_file = aws_parallel_input_stream_new_from_file,
+    .http_connection_make_request = aws_http_connection_make_request,
 };
 
 void aws_s3_set_dns_ttl(size_t ttl) {
@@ -360,7 +361,7 @@ struct aws_s3_client *aws_s3_client_new(
         goto on_error;
     }
 
-    client->vtable = &s_s3_client_default_vtable;
+    client->vtable = &g_s3_client_default_vtable;
 
     aws_ref_count_init(&client->ref_count, client, (aws_simple_completion_callback *)s_s3_client_start_destroy);
 

source/s3_meta_request.c

@@ -864,7 +864,7 @@ static void s_get_original_credentials_callback(struct aws_credentials *credenti
     }
 }
 
-static int s_meta_request_resolve_signing_config(
+static void s_meta_request_resolve_signing_config(
     struct aws_signing_config_aws *out_signing_config,
     struct aws_s3_request *request,
     struct aws_s3_meta_request *meta_request) {
@@ -897,7 +897,12 @@ static int s_meta_request_resolve_signing_config(
             &out_signing_config->signed_body_value, &g_aws_signed_body_value_streaming_unsigned_payload_trailer)) {
         out_signing_config->signed_body_value = g_aws_signed_body_value_unsigned_payload;
     }
-    return AWS_OP_SUCCESS;
+
+    /**
+     * In case of the signing was skipped for anonymous credentials, or presigned URL.
+     */
+    request->send_data.require_streaming_unsigned_payload_header = aws_byte_cursor_eq(
+        &out_signing_config->signed_body_value, &g_aws_signed_body_value_streaming_unsigned_payload_trailer);
 }
 
 void aws_s3_meta_request_sign_request_default_impl(
@@ -915,19 +920,19 @@ void aws_s3_meta_request_sign_request_default_impl(
 
     struct aws_signing_config_aws signing_config;
 
-    if (s_meta_request_resolve_signing_config(&signing_config, request, meta_request)) {
-        AWS_LOGF_DEBUG(
+    s_meta_request_resolve_signing_config(&signing_config, request, meta_request);
+
+    request->send_data.signable = aws_signable_new_http_request(meta_request->allocator, request->send_data.message);
+    if (request->send_data.signable == NULL) {
+        AWS_LOGF_ERROR(
             AWS_LS_S3_META_REQUEST,
-            "id=%p: No signing config present. Not signing request %p.",
+            "id=%p: Could not allocate signable for request %p",
             (void *)meta_request,
             (void *)request);
 
-        on_signing_complete(NULL, AWS_ERROR_SUCCESS, user_data);
+        on_signing_complete(NULL, aws_last_error_or_unknown(), user_data);
         return;
     }
-
-    request->send_data.signable = aws_signable_new_http_request(meta_request->allocator, request->send_data.message);
-
     AWS_LOGF_TRACE(
         AWS_LS_S3_META_REQUEST,
         "id=%p Created signable %p for request %p with message %p",
@@ -936,17 +941,6 @@ void aws_s3_meta_request_sign_request_default_impl(
         (void *)request,
         (void *)request->send_data.message);
 
-    if (request->send_data.signable == NULL) {
-        AWS_LOGF_ERROR(
-            AWS_LS_S3_META_REQUEST,
-            "id=%p: Could not allocate signable for request %p",
-            (void *)meta_request,
-            (void *)request);
-
-        on_signing_complete(NULL, aws_last_error_or_unknown(), user_data);
-        return;
-    }
-
     if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS && !disable_s3_express_signing) {
         /* Fetch credentials from S3 Express provider. */
         struct aws_get_s3express_credentials_user_data *context =
@@ -1064,6 +1058,17 @@ static void s_s3_meta_request_request_on_signed(
 
 finish:
 
+    /**
+     * Add the x-amz-content-sha256 header to support trailing checksum.
+     */
+    if (error_code == AWS_ERROR_SUCCESS && request->send_data.require_streaming_unsigned_payload_header) {
+        struct aws_http_headers *headers = aws_http_message_get_headers(request->send_data.message);
+        error_code = aws_http_headers_set(
+            headers,
+            aws_byte_cursor_from_c_str("x-amz-content-sha256"),
+            g_aws_signed_body_value_streaming_unsigned_payload_trailer);
+    }
+
     if (error_code != AWS_ERROR_SUCCESS) {
 
         AWS_LOGF_ERROR(
@@ -1082,6 +1087,8 @@ void aws_s3_meta_request_send_request(struct aws_s3_meta_request *meta_request,
     AWS_PRECONDITION(connection);
     AWS_PRECONDITION(connection->http_connection);
 
+    struct aws_s3_client *client = meta_request->client;
+    AWS_PRECONDITION(client);
     struct aws_s3_request *request = connection->request;
     AWS_PRECONDITION(request);
 
@@ -1104,7 +1111,8 @@ void aws_s3_meta_request_send_request(struct aws_s3_meta_request *meta_request,
         request->upload_timeout_ms = (size_t)options.response_first_byte_timeout_ms;
     }
 
-    struct aws_http_stream *stream = aws_http_connection_make_request(connection->http_connection, &options);
+    struct aws_http_stream *stream =
+        client->vtable->http_connection_make_request(connection->http_connection, &options);
 
     if (stream == NULL) {
         AWS_LOGF_ERROR(

tests/CMakeLists.txt

@@ -312,6 +312,8 @@ add_net_test_case(test_s3_list_bucket_valid)
 # Tests against local mock server
 if(ENABLE_MOCK_SERVER_TESTS)
     add_net_test_case(multipart_upload_mock_server)
+    add_net_test_case(multipart_upload_unsigned_with_trailer_checksum_mock_server)
+    add_net_test_case(single_upload_unsigned_with_trailer_checksum_mock_server)
     add_net_test_case(multipart_upload_with_network_interface_names_mock_server)
     add_net_test_case(multipart_upload_checksum_with_retry_mock_server)
     add_net_test_case(multipart_download_checksum_with_retry_mock_server)

tests/s3_mock_server_tests.c

@@ -167,6 +167,206 @@ TEST_CASE(multipart_upload_mock_server) {
     return AWS_OP_SUCCESS;
 }
 
+/* Singleton used by tests in this file */
+static struct get_requests_header_tester {
+    struct aws_allocator *alloc;
+
+    /* Store the requests headers in the array. Array of struct aws_http_headers * */
+    struct aws_array_list headers_array;
+    struct aws_mutex lock;
+} s_get_requests_header_tester;
+
+static int s_get_requests_header_tester_init(struct aws_allocator *alloc) {
+    ASSERT_SUCCESS(aws_array_list_init_dynamic(
+        &s_get_requests_header_tester.headers_array, alloc, 1, sizeof(struct aws_http_headers *)));
+    ASSERT_SUCCESS(aws_mutex_init(&s_get_requests_header_tester.lock));
+    return AWS_OP_SUCCESS;
+}
+
+static void s_get_requests_header_tester_clean_up(void) {
+    /* iterate thought the headers array to clean up the headers */
+    for (size_t i = 0; i < aws_array_list_length(&s_get_requests_header_tester.headers_array); ++i) {
+        struct aws_http_headers *headers = NULL;
+        aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, i);
+        aws_http_headers_release(headers);
+    }
+    aws_mutex_clean_up(&s_get_requests_header_tester.lock);
+    aws_array_list_clean_up(&s_get_requests_header_tester.headers_array);
+}
+
+struct aws_http_stream *s_get_requests_header_make_request(
+    struct aws_http_connection *client_connection,
+    const struct aws_http_make_request_options *options) {
+    /**
+     * Record the headers in the array.
+     */
+    aws_mutex_lock(&s_get_requests_header_tester.lock);
+    struct aws_http_headers *headers = aws_http_message_get_headers(options->request);
+    /* Keep the headers alive until we clean up the tester. */
+    aws_http_headers_acquire(headers);
+    aws_array_list_push_back(&s_get_requests_header_tester.headers_array, &headers);
+    aws_mutex_unlock(&s_get_requests_header_tester.lock);
+
+    struct aws_http_stream *stream = aws_http_connection_make_request(client_connection, options);
+    return stream;
+}
+
+TEST_CASE(multipart_upload_unsigned_with_trailer_checksum_mock_server) {
+    (void)ctx;
+
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+    ASSERT_SUCCESS(s_get_requests_header_tester_init(allocator));
+
+    struct aws_s3_client_config client_config = {
+        .tls_mode = AWS_MR_TLS_DISABLED,
+    };
+
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION));
+
+    struct aws_s3_client_vtable s3_client_get_requests_header_vtable = g_s3_client_default_vtable;
+    s3_client_get_requests_header_vtable.http_connection_make_request = s_get_requests_header_make_request;
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_NOT_NULL(client);
+    client->vtable = &s3_client_get_requests_header_vtable;
+
+    struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/default");
+
+    struct aws_s3_tester_meta_request_options put_options = {
+        .allocator = allocator,
+        .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
+        .client = client,
+        .checksum_algorithm = AWS_SCA_CRC32,
+        .validate_get_response_checksum = false,
+        .put_options =
+            {
+                .object_size_mb = 10,
+                .object_path_override = object_path,
+            },
+        .mock_server = true,
+    };
+    struct aws_s3_meta_request_test_results out_results;
+    aws_s3_meta_request_test_results_init(&out_results, allocator);
+    ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results));
+    ASSERT_SUCCESS(s_validate_mpu_mock_server_metrics(&out_results.synced_data.metrics));
+
+    /**
+     * Check the recorded headers.
+     * 4 requests should be made:
+     * - Create MPU
+     * - 2 Upload Part
+     * - Complete MPU
+     */
+    ASSERT_UINT_EQUALS(4, aws_array_list_length(&s_get_requests_header_tester.headers_array));
+    struct aws_byte_cursor content_sha256_header = aws_byte_cursor_from_c_str("x-amz-content-sha256");
+    struct aws_byte_cursor content_sha256_header_val;
+    AWS_ZERO_STRUCT(content_sha256_header_val);
+    struct aws_byte_cursor authorization_header = aws_byte_cursor_from_c_str("Authorization");
+    struct aws_byte_cursor authorization_header_val;
+    AWS_ZERO_STRUCT(authorization_header_val);
+    /* The first request should be Create MPU, and it should not have x-amz-content-sha256 header. */
+    struct aws_http_headers *headers = NULL;
+    ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 0));
+    /* No x-amz-content-sha256 header should be found. */
+    ASSERT_FAILS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
+    /* The second and third requests should be Upload Part, and it should have x-amz-content-sha256 header with
+     * STREAMING-UNSIGNED-PAYLOAD-TRAILER. */
+    ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 1));
+    /* x-amz-content-sha256 header should be found. */
+    ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
+    ASSERT_TRUE(
+        aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
+    /* But the Authorization header should not be found, since we are not signing the request. */
+    ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));
+    ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 2));
+    /* x-amz-content-sha256 header should be found. */
+    ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
+    ASSERT_TRUE(
+        aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
+    /* But the Authorization header should not be found, since we are not signing the request. */
+    ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));
+    /* The last request should be Complete MPU, and it should not have x-amz-content-sha256 header. */
+    ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 3));
+    /* No x-amz-content-sha256 header should be found. */
+    ASSERT_FAILS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
+
+    aws_s3_meta_request_test_results_clean_up(&out_results);
+    aws_s3_client_release(client);
+    aws_s3_tester_clean_up(&tester);
+    s_get_requests_header_tester_clean_up();
+
+    return AWS_OP_SUCCESS;
+}
+
+TEST_CASE(single_upload_unsigned_with_trailer_checksum_mock_server) {
+    (void)ctx;
+
+    struct aws_s3_tester tester;
+    ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
+    ASSERT_SUCCESS(s_get_requests_header_tester_init(allocator));
+
+    struct aws_s3_client_config client_config = {
+        .tls_mode = AWS_MR_TLS_DISABLED,
+        .part_size = 20 * 1024 * 1024,
+    };
+
+    ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION));
+
+    struct aws_s3_client_vtable s3_client_get_requests_header_vtable = g_s3_client_default_vtable;
+    s3_client_get_requests_header_vtable.http_connection_make_request = s_get_requests_header_make_request;
+    struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
+    ASSERT_NOT_NULL(client);
+    client->vtable = &s3_client_get_requests_header_vtable;
+
+    struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/default");
+
+    struct aws_s3_tester_meta_request_options put_options = {
+        .allocator = allocator,
+        .meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
+        .client = client,
+        .checksum_algorithm = AWS_SCA_CRC32,
+        .validate_get_response_checksum = false,
+        .put_options =
+            {
+                .object_size_mb = 10,
+                .object_path_override = object_path,
+            },
+        .mock_server = true,
+    };
+    struct aws_s3_meta_request_test_results out_results;
+    aws_s3_meta_request_test_results_init(&out_results, allocator);
+    ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results));
+
+    /**
+     * Check the recorded headers.
+     * 1 request should be made:
+     * - Put Object
+     */
+    ASSERT_UINT_EQUALS(1, aws_array_list_length(&s_get_requests_header_tester.headers_array));
+    struct aws_byte_cursor content_sha256_header = aws_byte_cursor_from_c_str("x-amz-content-sha256");
+    struct aws_byte_cursor content_sha256_header_val;
+    AWS_ZERO_STRUCT(content_sha256_header_val);
+    struct aws_byte_cursor authorization_header = aws_byte_cursor_from_c_str("Authorization");
+    struct aws_byte_cursor authorization_header_val;
+    AWS_ZERO_STRUCT(authorization_header_val);
+    /* The request should be Put Object, and it should have x-amz-content-sha256 header and not Authorization header. */
+    struct aws_http_headers *headers = NULL;
+    ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 0));
+    /* x-amz-content-sha256 header should be found. */
+    ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
+    ASSERT_TRUE(
+        aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
+    /* But the Authorization header should not be found, since we are not signing the request. */
+    ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));
+
+    aws_s3_meta_request_test_results_clean_up(&out_results);
+    aws_s3_client_release(client);
+    aws_s3_tester_clean_up(&tester);
+    s_get_requests_header_tester_clean_up();
+
+    return AWS_OP_SUCCESS;
+}
+
 TEST_CASE(multipart_upload_with_network_interface_names_mock_server) {
     (void)ctx;
 #if defined(AWS_OS_WINDOWS)