Skip to content

Commit

Permalink
PUT Blob Structured Message (Azure#43130)
Browse files Browse the repository at this point in the history 
* putblob structured message

* testproxy

* fixes
  • Loading branch information
@jaschrep-msft
jaschrep-msft committed Aug 12, 2024
1 parent 73d5345 commit 003b439
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 15 deletions.
44 changes: 35 additions & 9 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -875,14 +875,36 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
scope.Start();
Errors.VerifyStreamPosition(content, nameof(content));

// compute hash BEFORE attaching progress handler
ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);

content = content?.WithNoDispose().WithProgress(progressHandler);
ContentHasher.GetHashResult hashResult = null;
long contentLength = (content?.Length - content?.Position) ?? 0;
long? structuredContentLength = default;
string structuredBodyType = null;
if (content != null &&
validationOptions != null &&
validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
validationOptions.PrecalculatedChecksum.IsEmpty &&
ClientSideEncryption == null) // don't allow feature combination
{
// report progress in terms of caller bytes, not encoded bytes
structuredContentLength = contentLength;
structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
content = content.WithNoDispose().WithProgress(progressHandler);
content = new StructuredMessageEncodingStream(
content,
Constants.StructuredMessage.DefaultSegmentContentLength,
StructuredMessage.Flags.StorageCrc64);
contentLength = content.Length - content.Position;
}
else
{
// compute hash BEFORE attaching progress handler
hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);
content = content.WithNoDispose().WithProgress(progressHandler);
}

ResponseWithHeaders<BlockBlobUploadHeaders> response;

Expand Down Expand Up @@ -921,6 +943,8 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
legalHold: legalHold,
transactionalContentMD5: hashResult?.MD5AsArray,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
Expand Down Expand Up @@ -953,6 +977,8 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
legalHold: legalHold,
transactionalContentMD5: hashResult?.MD5AsArray,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken);
}

Expand Down Expand Up @@ -2817,7 +2843,7 @@ internal async Task<Stream> OpenWriteInternal(
immutabilityPolicy: default,
legalHold: default,
progressHandler: default,
transferValidationOverride: default,
transferValidationOverride: new() { ChecksumAlgorithm = StorageChecksumAlgorithm.None },
operationName: default,
async: async,
cancellationToken: cancellationToken)
Expand Down
31 changes: 25 additions & 6 deletions sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Azure.Core;
using Azure.Core.Pipeline;
Expand Down Expand Up @@ -1020,8 +1021,14 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(
checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down Expand Up @@ -1108,8 +1115,14 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions(
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(
clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down Expand Up @@ -1160,8 +1173,14 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(
overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down

0 comments on commit 003b439

Please sign in to comment.