Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PUT Blob Structured Message #43130

Merged
merged 3 commits into from
Apr 4, 2024
Merged

PUT Blob Structured Message #43130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
13cc4d9
putblob structured message
jaschrep-msft Apr 1, 2024
f22f837
testproxy
jaschrep-msft Apr 1, 2024
d7ab2dc
fixes
jaschrep-msft Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.Blobs/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.Blobs",
"Tag": "net/storage/Azure.Storage.Blobs_4d941ee320"
"Tag": "net/storage/Azure.Storage.Blobs_61b439d720"
}
44 changes: 35 additions & 9 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -875,14 +875,36 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
scope.Start();
Errors.VerifyStreamPosition(content, nameof(content));

// compute hash BEFORE attaching progress handler
ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);

content = content?.WithNoDispose().WithProgress(progressHandler);
ContentHasher.GetHashResult hashResult = null;
long contentLength = (content?.Length - content?.Position) ?? 0;
long? structuredContentLength = default;
string structuredBodyType = null;
if (content != null &&
validationOptions != null &&
validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
validationOptions.PrecalculatedChecksum.IsEmpty &&
ClientSideEncryption == null) // don't allow feature combination
{
jaschrep-msft marked this conversation as resolved.
Show resolved Hide resolved
// report progress in terms of caller bytes, not encoded bytes
structuredContentLength = contentLength;
structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
content = content.WithNoDispose().WithProgress(progressHandler);
content = new StructuredMessageEncodingStream(
content,
Constants.StructuredMessage.DefaultSegmentContentLength,
StructuredMessage.Flags.StorageCrc64);
contentLength = content.Length - content.Position;
}
else
{
// compute hash BEFORE attaching progress handler
hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);
content = content.WithNoDispose().WithProgress(progressHandler);
}

ResponseWithHeaders<BlockBlobUploadHeaders> response;

Expand Down Expand Up @@ -921,6 +943,8 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
legalHold: legalHold,
transactionalContentMD5: hashResult?.MD5AsArray,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
Expand Down Expand Up @@ -953,6 +977,8 @@ internal virtual async Task<Response<BlobContentInfo>> UploadInternal(
legalHold: legalHold,
transactionalContentMD5: hashResult?.MD5AsArray,
transactionalContentCrc64: hashResult?.StorageCrc64AsArray,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
cancellationToken: cancellationToken);
}

Expand Down Expand Up @@ -2817,7 +2843,7 @@ internal async Task<Stream> OpenWriteInternal(
immutabilityPolicy: default,
legalHold: default,
progressHandler: default,
transferValidationOverride: default,
transferValidationOverride: new() { ChecksumAlgorithm = StorageChecksumAlgorithm.None },
operationName: default,
async: async,
cancellationToken: cancellationToken)
Expand Down
31 changes: 25 additions & 6 deletions sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using Azure.Core;
using Azure.Core.Pipeline;
Expand Down Expand Up @@ -1020,8 +1021,14 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(
checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down Expand Up @@ -1108,8 +1115,14 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions(
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(
clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down Expand Up @@ -1160,8 +1173,14 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions
};

// make pipeline assertion for checking checksum was present on upload
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(
overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected));
#if BlobSDK
var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split
? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength)
: GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#else
var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected);
#endif
var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion);
var clientOptions = ClientBuilder.GetOptions();
clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall);

Expand Down
Loading