Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/storage/put blob from url #15978

Merged
merged 10 commits into from
Nov 6, 2020
Merged

Feature/storage/put blob from url #15978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
182f50c
Inital commit
seanmcc-msft Oct 14, 2020
4fd35fc
fixed build
seanmcc-msft Oct 14, 2020
7c13cbe
Added unit tests
seanmcc-msft Oct 14, 2020
22ed3a1
Added some unit tests
seanmcc-msft Oct 14, 2020
da45e35
finished recording tests
seanmcc-msft Oct 14, 2020
5832dc5
PR comments
seanmcc-msft Nov 5, 2020
ed63884
Merge branch 'feature/storage/stg75base' into feature/storage/putBlob…
seanmcc-msft Nov 5, 2020
6dc5cbd
PR comments
seanmcc-msft Nov 5, 2020
0ca0d91
PR comments
seanmcc-msft Nov 6, 2020
e00c037
PR comments
seanmcc-msft Nov 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,11 @@ internal BlobCopyInfo() { }
public System.DateTimeOffset LastModified { get { throw null; } }
public string VersionId { get { throw null; } }
}
public enum BlobCopySourceBlobPropertiesOption
{
Copy = 0,
Overwrite = 1,
}
public partial class BlobCorsRule
{
public BlobCorsRule() { }
Expand Down Expand Up @@ -938,6 +943,16 @@ public enum BlobType
Page = 1,
Append = 2,
}
public partial class BlobUploadFromUriOptions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should take metadata as well.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately there is a service bug, setting metadata on doesn't currently work.

#15969

{
public BlobUploadFromUriOptions() { }
public Azure.Storage.Blobs.Models.AccessTier? AccessTier { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobCopySourceBlobPropertiesOption CopySourceBlobPropertiesOption { get { throw null; } set { } }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is boolean on the server side. enum might be overkill.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue here is that the server side bool is true by default. This violates One SDK's guidelines.

Put Blob from URL spec

x-ms-copy-source-blob-properties: true/false. Optional. Specifies whether the properties of the source blob should be copied to the destination. This is true by default. If false, no properties from the source blob are copied, and only those properties provided in the request are applied to the destination blob.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tg-msft should we invert boolean or keep enum ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we use bool? CopyBlobProperties { get; set; } where the default null means do whatever the service wants? That might also help make it clear customers only need to think about setting it when they're doing a copy vs. uploading a random URI?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good to me. That's how Java equivalent is implemented now.

public Azure.Storage.Blobs.Models.BlobRequestConditions DestinationConditions { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobRequestConditions SourceConditions { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Tags { get { throw null; } set { } }
}
public partial class BlobUploadOptions
{
public BlobUploadOptions() { }
Expand Down Expand Up @@ -1367,6 +1382,8 @@ public BlockBlobClient(System.Uri blobUri, Azure.Storage.StorageSharedKeyCredent
[System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)]
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo>> UploadAsync(System.IO.Stream content, Azure.Storage.Blobs.Models.BlobHttpHeaders httpHeaders = null, System.Collections.Generic.IDictionary<string, string> metadata = null, Azure.Storage.Blobs.Models.BlobRequestConditions conditions = null, Azure.Storage.Blobs.Models.AccessTier? accessTier = default(Azure.Storage.Blobs.Models.AccessTier?), System.IProgress<long> progressHandler = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo>> UploadAsync(System.IO.Stream content, Azure.Storage.Blobs.Models.BlobUploadOptions options, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public virtual Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo> UploadFromUri(System.Uri copySource, Azure.Storage.Blobs.Models.BlobUploadFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Storage.Blobs.Models.BlobContentInfo>> UploadFromUriAsync(System.Uri copySource, Azure.Storage.Blobs.Models.BlobUploadFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
public new Azure.Storage.Blobs.Specialized.BlockBlobClient WithSnapshot(string snapshot) { throw null; }
protected sealed override Azure.Storage.Blobs.Specialized.BlobBaseClient WithSnapshotCore(string snapshot) { throw null; }
public new Azure.Storage.Blobs.Specialized.BlockBlobClient WithVersion(string versionId) { throw null; }
Expand Down
189 changes: 189 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2160,6 +2160,195 @@ private async Task<Stream> OpenWriteInternal(
}
#endregion OpenWrite

#region UploadFromUri
/// <summary>
/// The Upload from Uri operation creates a new Block Blob where the contents of the
/// blob are read from a given URL. This API is supported beginning with the 2020-04-08 version.
///
/// Partial updates are not supported with Put Blob from URL; the content of an existing blob is
/// overwritten with the content of the new blob. To perform partial updates to a block blob’s
/// contents using a source URL, use the Put Block from URL API in conjunction with Put Block List.
/// </summary>
/// <param name="copySource">
/// Required. Specifies the URL of the source blob. The source blob may be of any type,
/// including a block blob, append blob, or page blob. The value may be a URL of up to 2
/// KiB in length that specifies a blob. The value should be URL-encoded as it would appear
/// in a request URI. The source blob must either be public or must be authorized via a
/// shared access signature. If the source blob is public, no authorization is required
/// to perform the operation.
/// </param>
/// <param name="options">
/// Optional parameters.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public virtual Response<BlobContentInfo> UploadFromUri(
Uri copySource,
BlobUploadFromUriOptions options = default,
CancellationToken cancellationToken = default)
=> UploadFromUriInternal(
copySource,
options,
async: false,
cancellationToken)
.EnsureCompleted();

/// <summary>
/// The Upload from Uri operation creates a new Block Blob where the contents of the
/// blob are read from a given URL. This API is supported beginning with the 2020-04-08 version.
///
/// Partial updates are not supported with Put Blob from URL; the content of an existing blob is
/// overwritten with the content of the new blob. To perform partial updates to a block blob’s
/// contents using a source URL, use the Put Block from URL API in conjunction with Put Block List.
/// </summary>
/// <param name="copySource">
/// Required. Specifies the URL of the source blob. The source blob may be of any type,
/// including a block blob, append blob, or page blob. The value may be a URL of up to 2
/// KiB in length that specifies a blob. The value should be URL-encoded as it would appear
/// in a request URI. The source blob must either be public or must be authorized via a
/// shared access signature. If the source blob is public, no authorization is required
/// to perform the operation.
/// </param>
/// <param name="options">
/// Optional parameters.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
public virtual async Task<Response<BlobContentInfo>> UploadFromUriAsync(
Uri copySource,
BlobUploadFromUriOptions options = default,
CancellationToken cancellationToken = default)
=> await UploadFromUriInternal(
copySource,
options,
async: true,
cancellationToken)
.ConfigureAwait(false);

/// <summary>
/// The Upload from Uri operation creates a new Block Blob where the contents of the
/// blob are read from a given URL. This API is supported beginning with the 2020-04-08 version.
///
/// Partial updates are not supported with Put Blob from URL; the content of an existing blob is
/// overwritten with the content of the new blob. To perform partial updates to a block blob’s
/// contents using a source URL, use the Put Block from URL API in conjunction with Put Block List.
/// </summary>
/// <param name="copySource">
/// Required. Specifies the URL of the source blob. The source blob may be of any type,
/// including a block blob, append blob, or page blob. The value may be a URL of up to 2
/// KiB in length that specifies a blob. The value should be URL-encoded as it would appear
/// in a request URI. The source blob must either be public or must be authorized via a
/// shared access signature. If the source blob is public, no authorization is required
/// to perform the operation.
/// </param>
/// <param name="options">
/// Optional parameters.
/// </param>
/// <param name="async">
/// Whether to invoke the operation asynchronously.
/// </param>
/// <param name="cancellationToken">
/// Optional <see cref="CancellationToken"/> to propagate
/// notifications that the operation should be cancelled.
/// </param>
/// <returns>
/// A <see cref="Response{BlobContentInfo}"/> describing the
/// state of the updated block blob.
/// </returns>
/// <remarks>
/// A <see cref="RequestFailedException"/> will be thrown if
/// a failure occurs.
/// </remarks>
internal virtual async Task<Response<BlobContentInfo>> UploadFromUriInternal(
Uri copySource,
BlobUploadFromUriOptions options,
bool async,
CancellationToken cancellationToken)
{
using (Pipeline.BeginLoggingScope(nameof(BlockBlobClient)))
{
Pipeline.LogMethodEnter(
nameof(BlockBlobClient),
message:
$"{nameof(Uri)}: {Uri}\n" +
$"{nameof(options.HttpHeaders)}: {options?.HttpHeaders}\n" +
$"{nameof(options.DestinationConditions)}: {options?.DestinationConditions}");
try
{
return await BlobRestClient.BlockBlob.PutBlobFromUrlAsync(
clientDiagnostics: ClientDiagnostics,
pipeline: Pipeline,
resourceUri: Uri,
contentLength: 0,
version: Version.ToVersionString(),
copySource: copySource,
timeout: default,
transactionalContentHash: default,
blobContentType: options?.HttpHeaders?.ContentType,
blobContentEncoding: options?.HttpHeaders?.ContentEncoding,
blobContentLanguage: options?.HttpHeaders?.ContentLanguage,
blobContentHash: options?.HttpHeaders?.ContentHash,
blobCacheControl: options?.HttpHeaders?.CacheControl,
// TODO service bug. https://github.com/Azure/azure-sdk-for-net/issues/15969
// metadata: options?.Metadata,
leaseId: options?.DestinationConditions?.LeaseId,
blobContentDisposition: options?.HttpHeaders?.ContentDisposition,
encryptionKey: CustomerProvidedKey?.EncryptionKey,
encryptionKeySha256: CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: CustomerProvidedKey?.EncryptionAlgorithm,
encryptionScope: EncryptionScope,
tier: options?.AccessTier,
ifModifiedSince: options?.DestinationConditions?.IfModifiedSince,
ifUnmodifiedSince: options?.DestinationConditions?.IfUnmodifiedSince,
ifMatch: options?.DestinationConditions?.IfMatch,
ifNoneMatch: options?.DestinationConditions?.IfNoneMatch,
ifTags: options?.DestinationConditions?.TagConditions,
sourceIfModifiedSince: options?.SourceConditions?.IfModifiedSince,
sourceIfUnmodifiedSince: options?.SourceConditions?.IfUnmodifiedSince,
sourceIfMatch: options?.SourceConditions?.IfMatch,
sourceIfNoneMatch: options?.SourceConditions?.IfNoneMatch,
sourceIfTags: options?.SourceConditions?.TagConditions,
requestId: default,
blobTagsString: options?.Tags?.ToTagsString(),
copySourceBlobProperties: options?.CopySourceBlobPropertiesOption == BlobCopySourceBlobPropertiesOption.Overwrite ? false : true,
async: async,
operationName: $"{nameof(BlockBlobClient)}.{nameof(UploadFromUri)}",
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
Pipeline.LogException(ex);
throw;
}
finally
{
Pipeline.LogMethodExit(nameof(BlockBlobClient));
}
}
}
#endregion UploadFromUri

#region PartitionedUploader
internal PartitionedUploader<BlobUploadOptions, BlobContentInfo> GetPartitionedUploader(
StorageTransferOptions transferOptions,
Expand Down
Loading