From df4aaf98e4845a83387adc000f9b08169683c3e8 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Tue, 13 Aug 2024 14:30:41 -0400 Subject: [PATCH 01/18] Initial commit --- .../src/DataLakeExtensions.cs | 23 ++ .../src/DataLakeFileClient.cs | 228 +++++++++++++++++- .../src/Models/DataLakeFileReadResult.cs | 25 ++ .../Models/DataLakeFileReadStreamingResult.cs | 36 +++ 4 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadResult.cs create mode 100644 sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadStreamingResult.cs diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs index 9d50525a1120c..77f5257c56923 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeExtensions.cs @@ -99,6 +99,29 @@ internal static FileDownloadInfo ToFileDownloadInfo(this Response blobDownloadStreamingResultResponse) + { + blobDownloadStreamingResultResponse.GetRawResponse().Headers.TryGetValue(Constants.DataLake.EncryptionContextHeaderName, out string encryptionContext); + blobDownloadStreamingResultResponse.GetRawResponse().Headers.TryGetValue(Constants.DataLake.AclHeaderName, out string accessControlList); + DataLakeFileReadStreamingResult dataLakeFileReadStreamingResult = new DataLakeFileReadStreamingResult() + { + Content = blobDownloadStreamingResultResponse.Value.Content, + Details = blobDownloadStreamingResultResponse.Value.Details.ToFileDownloadDetails(encryptionContext, accessControlList) + }; + return dataLakeFileReadStreamingResult; + } + + internal static DataLakeFileReadResult ToDataLakeFileReadResult(this Response blobDownloadResult) + { + blobDownloadResult.GetRawResponse().Headers.TryGetValue(Constants.DataLake.EncryptionContextHeaderName, out string encryptionContext); + blobDownloadResult.GetRawResponse().Headers.TryGetValue(Constants.DataLake.AclHeaderName, out string accessControlList); + DataLakeFileReadResult dataLakeFileReadResult = new DataLakeFileReadResult() + { + Content = blobDownloadResult.Value.Content, + Details = blobDownloadResult.Value.Details.ToFileDownloadDetails(encryptionContext, accessControlList) + }; + return dataLakeFileReadResult; + } internal static PathProperties ToPathProperties(this Response blobPropertiesResponse) { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 3d2bd710e25aa..3da9eca92b479 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -2827,6 +2827,114 @@ internal virtual async Task> FlushInternal( #endregion #region Read Data + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadStreaming( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadStreaming( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadContent( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadContent( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + /// /// The operation downloads a file from /// the service, including its metadata and properties. @@ -2844,6 +2952,7 @@ internal virtual async Task> FlushInternal( /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual Response Read() { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -2869,6 +2978,116 @@ public virtual Response Read() } } + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadStreamingAsync( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadStreamingAsync( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional parameters. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadContentAsync( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadContentAsync( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + /// /// The operation downloads a file from /// the service, including its metadata and properties. @@ -2886,6 +3105,7 @@ public virtual Response Read() /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual async Task> ReadAsync() { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -2933,6 +3153,7 @@ public virtual async Task> ReadAsync() /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual Response Read( CancellationToken cancellationToken = default) { @@ -2980,6 +3201,7 @@ public virtual Response Read( /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual async Task> ReadAsync( CancellationToken cancellationToken = default) { @@ -3157,7 +3379,7 @@ public virtual async Task> ReadAsync( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3181,6 +3403,7 @@ public virtual async Task> ReadAsync( /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual Response Read( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) @@ -3211,7 +3434,7 @@ public virtual Response Read( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3235,6 +3458,7 @@ public virtual Response Read( /// A will be thrown if /// a failure occurs. /// + [EditorBrowsable(EditorBrowsableState.Never)] public virtual async Task> ReadAsync( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadResult.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadResult.cs new file mode 100644 index 0000000000000..6059019eb6cdd --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadResult.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Storage.Files.DataLake.Models +{ + /// + /// The details and content returned from reading a DataLake File. + /// + public class DataLakeFileReadResult + { + internal DataLakeFileReadResult() { } + + /// + /// Details returned when reading a DataLake file + /// + public FileDownloadDetails Details { get; internal set; } + + /// + /// Content. + /// + public BinaryData Content { get; internal set; } + } +} diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadStreamingResult.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadStreamingResult.cs new file mode 100644 index 0000000000000..985b1e9ab8374 --- /dev/null +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeFileReadStreamingResult.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using Azure.Storage.Shared; + +namespace Azure.Storage.Files.DataLake.Models +{ + /// + /// The details and content returned from reading a datalake file. + /// + public class DataLakeFileReadStreamingResult : IDisposable + { + internal DataLakeFileReadStreamingResult() { } + + /// + /// Details returned when reading a datalake file. + /// + public FileDownloadDetails Details { get; internal set; } + + /// + /// Content. + /// + public Stream Content { get; internal set; } + + /// + /// Disposes the by calling Dispose on the underlying stream. + /// + public void Dispose() + { + Content?.Dispose(); + GC.SuppressFinalize(this); + } + } +} From a1cb572cb2ff932123c49934b10d346f34378228 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Tue, 13 Aug 2024 19:55:02 -0400 Subject: [PATCH 02/18] Exported Api --- .../Azure.Storage.Files.DataLake.net6.0.cs | 23 +++++++++++++++++++ ...e.Storage.Files.DataLake.netstandard2.0.cs | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index efd9e87cdaeff..89592db960d80 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -170,16 +170,26 @@ public DataLakeFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCred public virtual System.Threading.Tasks.Task OpenWriteAsync(bool overwrite, Azure.Storage.Files.DataLake.Models.DataLakeFileOpenWriteOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response Query(string querySqlExpression, Azure.Storage.Files.DataLake.Models.DataLakeQueryOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual System.Threading.Tasks.Task> QueryAsync(string querySqlExpression, Azure.Storage.Files.DataLake.Models.DataLakeQueryOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read() { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(Azure.HttpRange range, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, bool rangeGetContentHash, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync() { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(Azure.HttpRange range, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, bool rangeGetContentHash, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeFileReadToOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, Azure.Storage.StorageTransferOptions transferOptions, System.Threading.CancellationToken cancellationToken) { throw null; } @@ -644,6 +654,19 @@ public DataLakeFileReadOptions() { } public Azure.HttpRange Range { get { throw null; } set { } } public Azure.Storage.DownloadTransferValidationOptions TransferValidation { get { throw null; } set { } } } + public partial class DataLakeFileReadResult + { + internal DataLakeFileReadResult() { } + public System.BinaryData Content { get { throw null; } } + public Azure.Storage.Files.DataLake.Models.FileDownloadDetails Details { get { throw null; } } + } + public partial class DataLakeFileReadStreamingResult : System.IDisposable + { + internal DataLakeFileReadStreamingResult() { } + public System.IO.Stream Content { get { throw null; } } + public Azure.Storage.Files.DataLake.Models.FileDownloadDetails Details { get { throw null; } } + public void Dispose() { } + } public partial class DataLakeFileReadToOptions { public DataLakeFileReadToOptions() { } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index efd9e87cdaeff..89592db960d80 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -170,16 +170,26 @@ public DataLakeFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCred public virtual System.Threading.Tasks.Task OpenWriteAsync(bool overwrite, Azure.Storage.Files.DataLake.Models.DataLakeFileOpenWriteOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response Query(string querySqlExpression, Azure.Storage.Files.DataLake.Models.DataLakeQueryOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual System.Threading.Tasks.Task> QueryAsync(string querySqlExpression, Azure.Storage.Files.DataLake.Models.DataLakeQueryOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read() { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(Azure.HttpRange range, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, bool rangeGetContentHash, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response Read(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync() { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(Azure.HttpRange range, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, bool rangeGetContentHash, System.Threading.CancellationToken cancellationToken) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeFileReadToOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, Azure.Storage.StorageTransferOptions transferOptions, System.Threading.CancellationToken cancellationToken) { throw null; } @@ -644,6 +654,19 @@ public DataLakeFileReadOptions() { } public Azure.HttpRange Range { get { throw null; } set { } } public Azure.Storage.DownloadTransferValidationOptions TransferValidation { get { throw null; } set { } } } + public partial class DataLakeFileReadResult + { + internal DataLakeFileReadResult() { } + public System.BinaryData Content { get { throw null; } } + public Azure.Storage.Files.DataLake.Models.FileDownloadDetails Details { get { throw null; } } + } + public partial class DataLakeFileReadStreamingResult : System.IDisposable + { + internal DataLakeFileReadStreamingResult() { } + public System.IO.Stream Content { get { throw null; } } + public Azure.Storage.Files.DataLake.Models.FileDownloadDetails Details { get { throw null; } } + public void Dispose() { } + } public partial class DataLakeFileReadToOptions { public DataLakeFileReadToOptions() { } From 010cf0018411d834b6cfde236b00fab36110e267 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Tue, 13 Aug 2024 20:11:51 -0400 Subject: [PATCH 03/18] Added CHANGELOG --- sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md index 3fe9b88537cb6..2fdd1536a5f61 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md @@ -3,6 +3,7 @@ ## 12.20.0-beta.2 (Unreleased) ### Features Added +- Added feature parity for DataLake download APIs with Blobs #45418 ### Breaking Changes From 61a5314cbeafcb4a0806e62679e0a4256b931da7 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 15:11:47 -0400 Subject: [PATCH 04/18] Added Tests --- .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../tests/FileClientTests.cs | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 442889d04be63..025bb75c145d8 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_186c14971d" + "Tag": "net/storage/Azure.Storage.Files.DataLake_6d98e1e469" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index 6d9f0d5750800..6a73b9b22656d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Mime; using System.Reflection; using System.Security.Cryptography; using System.Text; @@ -3455,6 +3456,71 @@ public async Task ReadAsync() TestHelper.AssertSequenceEqual(data, actual.ToArray()); } + [RecordedTest] + public async Task ReadStreamingAsync() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + byte[] data = GetRandomBuffer(Constants.KB); + DataLakeFileClient fileClient = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (MemoryStream stream = new MemoryStream(data)) + { + await fileClient.AppendAsync(stream, 0); + } + + await fileClient.FlushAsync(Constants.KB); + + // Act + Response response = await fileClient.ReadStreamingAsync(); + + // Assert + Assert.IsNotNull(response.Value.Details.LastModified); + Assert.IsNotNull(response.Value.Details.AcceptRanges); + Assert.IsNotNull(response.Value.Details.ETag); + Assert.IsNotNull(response.Value.Details.LeaseStatus); + Assert.IsNotNull(response.Value.Details.LeaseState); + Assert.IsNotNull(response.Value.Details.IsServerEncrypted); + Assert.IsNotNull(response.Value.Details.CreatedOn); + Assert.IsNotNull(response.Value.Details.Metadata); + + MemoryStream actual = new MemoryStream(); + await response.Value.Content.CopyToAsync(actual); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + + [RecordedTest] + public async Task ReadContentAsync() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + byte[] data = GetRandomBuffer(Constants.KB); + DataLakeFileClient fileClient = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (MemoryStream stream = new MemoryStream(data)) + { + await fileClient.AppendAsync(stream, 0); + } + + await fileClient.FlushAsync(Constants.KB); + + // Act + Response response = await fileClient.ReadContentAsync(); + + // Assert + Assert.IsNotNull(response.Value.Details.LastModified); + Assert.IsNotNull(response.Value.Details.AcceptRanges); + Assert.IsNotNull(response.Value.Details.ETag); + Assert.IsNotNull(response.Value.Details.LeaseStatus); + Assert.IsNotNull(response.Value.Details.LeaseState); + Assert.IsNotNull(response.Value.Details.IsServerEncrypted); + Assert.IsNotNull(response.Value.Details.CreatedOn); + Assert.IsNotNull(response.Value.Details.Metadata); + + BinaryData actual = new BinaryData(response.Value.Content); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2024_05_04)] public async Task ReadAsyncACL() From 447c7a6ff9721512f28cc8544dffcd4aabf77f49 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 16:43:02 -0400 Subject: [PATCH 05/18] Added factory method + more tests + sample --- .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../samples/Sample01a_HelloWorld.cs | 108 ++++++++ .../src/Models/DataLakeModelFactory.cs | 28 ++ .../tests/FileClientTests.cs | 255 ++++++++++++++++++ 4 files changed, 392 insertions(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 025bb75c145d8..d3270d0cbc634 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_6d98e1e469" + "Tag": "net/storage/Azure.Storage.Files.DataLake_43c4036919" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs index 3f8cfdd32c7c5..ed8ab29134cb0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Pipes; using System.Linq; using System.Reflection.Metadata.Ecma335; using Azure.Storage.Files.DataLake; @@ -325,6 +326,113 @@ public void Read() } } + /// + /// Download a DataLake File stream to a file. + /// + [Test] + public void ReadStreaming() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string originalPath = CreateTempFile(SampleFileContent); + + // Get a temporary path on disk where we can download the file + string downloadPath = CreateTempPath(); + + // Make StorageSharedKeyCredential to pass to the serviceClient + string storageAccountName = StorageAccountName; + string storageAccountKey = StorageAccountKey; + Uri serviceUri = StorageAccountBlobUri; + StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + + // Create DataLakeServiceClient using StorageSharedKeyCredentials + DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); + + // Get a reference to a filesystem named "sample-filesystem-read" and then create it + DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient("sample-filesystem-read"); + filesystem.Create(); + try + { + // Get a reference to a file named "sample-file" in a filesystem + DataLakeFileClient file = filesystem.GetFileClient("sample-file"); + + // First upload something the DataLake file so we have something to download + file.Upload(File.OpenRead(originalPath)); + + // Download the DataLake file's contents and save it to a file + // The ReadStreamingAsync() API downloads a file in a single requests. + // For large files, it may be faster to call ReadTo() + #region Snippet:SampleSnippetDataLakeFileClient_ReadStreaming + Response fileContents = file.ReadStreaming(); + #endregion Snippet:SampleSnippetDataLakeFileClient_ReadStreaming + using (FileStream stream = File.OpenWrite(downloadPath)) + { + fileContents.Value.Content.CopyTo(stream); + } + + // Verify the contents + Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); + } + finally + { + // Clean up after the test when we're finished + filesystem.Delete(); + } + } + + /// + /// Download a DataLake File content to a file. + /// + [Test] + public void ReadContent() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string originalPath = CreateTempFile(SampleFileContent); + + // Get a temporary path on disk where we can download the file + string downloadPath = CreateTempPath(); + + // Make StorageSharedKeyCredential to pass to the serviceClient + string storageAccountName = StorageAccountName; + string storageAccountKey = StorageAccountKey; + Uri serviceUri = StorageAccountBlobUri; + StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + + // Create DataLakeServiceClient using StorageSharedKeyCredentials + DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); + + // Get a reference to a filesystem named "sample-filesystem-read" and then create it + DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient("sample-filesystem-read"); + filesystem.Create(); + try + { + // Get a reference to a file named "sample-file" in a filesystem + DataLakeFileClient file = filesystem.GetFileClient("sample-file"); + + // First upload something the DataLake file so we have something to download + file.Upload(File.OpenRead(originalPath)); + + // Download the DataLake file's contents and save it to a file + // The ReadContentAsync() API downloads a file in a single requests. + // For large files, it may be faster to call ReadTo() + #region Snippet:SampleSnippetDataLakeFileClient_ReadContent + Response fileContents = file.ReadContent(); + #endregion Snippet:SampleSnippetDataLakeFileClient_ReadContent + byte[] data = fileContents.Value.Content.ToArray(); + using (FileStream stream = File.OpenWrite(downloadPath)) + { + stream.Write(data, 0, data.Length); + } + + // Verify the contents + Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); + } + finally + { + // Clean up after the test when we're finished + filesystem.Delete(); + } + } + /// /// Download a DataLake File to a file. /// diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs index f4b7370f69f4e..55b7c47d1f8d6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs @@ -13,6 +13,34 @@ namespace Azure.Storage.Files.DataLake.Models /// public static partial class DataLakeModelFactory { + #region DataLakeFileReadResult + /// + /// Creates a new instance for mocking. + /// + public static DataLakeFileReadResult DataLakeFileReadResult( + BinaryData content, + FileDownloadDetails details) + => new DataLakeFileReadResult() + { + Content = content, + Details = details + }; + #endregion DataLakeFileReadResult + + #region DataLakeFileReadStreamingResult + /// + /// Creates a new instance for mocking. + /// + public static DataLakeFileReadStreamingResult DataLakeFileReadStreamingResult( + Stream content, + FileDownloadDetails details) + => new DataLakeFileReadStreamingResult() + { + Content = content, + Details = details + }; + #endregion DataLakeFileReadStreamingResult + #region FileDownloadDetails /// /// Creates a new FileDownloadDetails instance for mocking. diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index 6a73b9b22656d..70e78dd1c2d93 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -3568,6 +3568,97 @@ public async Task ReadAsyncACL() TestHelper.AssertSequenceEqual(data, actual.ToArray()); } + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2024_05_04)] + public async Task ReadStreamingAsyncACL() + { + await using DisposingFileSystem test = await GetNewFileSystem(publicAccessType: PublicAccessType.None); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeFileClient fileClient = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + AccessControlList = AccessControlList + } + }; + + await fileClient.CreateAsync(options: options); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + using (var stream = new MemoryStream(data)) + { + await fileClient.AppendAsync(stream, 0); + } + + await fileClient.FlushAsync(Constants.KB); + + // Act + Response response = await fileClient.ReadStreamingAsync(); + + // Assert + Assert.IsNotNull(response.Value.Details.LastModified); + Assert.IsNotNull(response.Value.Details.AcceptRanges); + Assert.IsNotNull(response.Value.Details.ETag); + Assert.IsNotNull(response.Value.Details.LeaseStatus); + Assert.IsNotNull(response.Value.Details.LeaseState); + Assert.IsNotNull(response.Value.Details.IsServerEncrypted); + Assert.IsNotNull(response.Value.Details.CreatedOn); + AssertAccessControlListEquality(AccessControlList, response.Value.Details.AccessControlList.ToList()); + + var actual = new MemoryStream(); + await response.Value.Content.CopyToAsync(actual); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + + [RecordedTest] + [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2024_05_04)] + public async Task ReadContentAsyncACL() + { + await using DisposingFileSystem test = await GetNewFileSystem(publicAccessType: PublicAccessType.None); + DataLakeDirectoryClient directory = await test.FileSystem.CreateDirectoryAsync(GetNewDirectoryName()); + + DataLakeFileClient fileClient = InstrumentClient(directory.GetFileClient(GetNewFileName())); + + DataLakePathCreateOptions options = new DataLakePathCreateOptions + { + AccessOptions = new DataLakeAccessOptions + { + AccessControlList = AccessControlList + } + }; + + await fileClient.CreateAsync(options: options); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + using (var stream = new MemoryStream(data)) + { + await fileClient.AppendAsync(stream, 0); + } + + await fileClient.FlushAsync(Constants.KB); + + // Act + Response response = await fileClient.ReadContentAsync(); + + // Assert + Assert.IsNotNull(response.Value.Details.LastModified); + Assert.IsNotNull(response.Value.Details.AcceptRanges); + Assert.IsNotNull(response.Value.Details.ETag); + Assert.IsNotNull(response.Value.Details.LeaseStatus); + Assert.IsNotNull(response.Value.Details.LeaseState); + Assert.IsNotNull(response.Value.Details.IsServerEncrypted); + Assert.IsNotNull(response.Value.Details.CreatedOn); + AssertAccessControlListEquality(AccessControlList, response.Value.Details.AccessControlList.ToList()); + + BinaryData actual = new BinaryData(response.Value.Content); + TestHelper.AssertSequenceEqual(data, actual.ToArray()); + } + [RecordedTest] [ServiceVersion(Min = DataLakeClientOptions.ServiceVersion.V2024_05_04)] public async Task GetPropertiesAsyncACL() @@ -3696,6 +3787,76 @@ public async Task ReadAsync_Conditions() } } + [RecordedTest] + public async Task ReadStreamingAsync_Conditions() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in Conditions_Data) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + DataLakeFileClient file = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (var stream = new MemoryStream(data)) + { + await file.AppendAsync(stream, 0); + } + + await file.FlushAsync(Constants.KB); + + parameters.Match = await SetupPathMatchCondition(file, parameters.Match); + parameters.LeaseId = await SetupPathLeaseCondition(file, parameters.LeaseId, garbageLeaseId); + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions( + parameters: parameters, + lease: true); + + // Act + Response response = await file.ReadStreamingAsync(new DataLakeFileReadOptions + { + Conditions = conditions + }); + + // Assert + Assert.IsNotNull(response.GetRawResponse().Headers.RequestId); + } + } + + [RecordedTest] + public async Task ReadContentAsync_Conditions() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in Conditions_Data) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + DataLakeFileClient file = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (var stream = new MemoryStream(data)) + { + await file.AppendAsync(stream, 0); + } + + await file.FlushAsync(Constants.KB); + + parameters.Match = await SetupPathMatchCondition(file, parameters.Match); + parameters.LeaseId = await SetupPathLeaseCondition(file, parameters.LeaseId, garbageLeaseId); + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions( + parameters: parameters, + lease: true); + + // Act + Response response = await file.ReadContentAsync(new DataLakeFileReadOptions + { + Conditions = conditions + }); + + // Assert + Assert.IsNotNull(response.GetRawResponse().Headers.RequestId); + } + } + [RecordedTest] public async Task ReadAsync_ConditionsFail() { @@ -3729,6 +3890,72 @@ await TestHelper.CatchAsync( } } + [RecordedTest] + public async Task ReadStreamingAsync_ConditionsFail() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in GetConditionsFail_Data(garbageLeaseId)) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + DataLakeFileClient file = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (var stream = new MemoryStream(data)) + { + await file.AppendAsync(stream, 0); + } + + await file.FlushAsync(Constants.KB); + + parameters.NoneMatch = await SetupPathMatchCondition(file, parameters.NoneMatch); + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions(parameters); + + // Act + await TestHelper.CatchAsync( + async () => + { + var _ = (await file.ReadStreamingAsync(new DataLakeFileReadOptions + { + Conditions = conditions + })).Value; + }); + } + } + + [RecordedTest] + public async Task ReadContentAsync_ConditionsFail() + { + var garbageLeaseId = GetGarbageLeaseId(); + foreach (AccessConditionParameters parameters in GetConditionsFail_Data(garbageLeaseId)) + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + var data = GetRandomBuffer(Constants.KB); + DataLakeFileClient file = await test.FileSystem.CreateFileAsync(GetNewFileName()); + using (var stream = new MemoryStream(data)) + { + await file.AppendAsync(stream, 0); + } + + await file.FlushAsync(Constants.KB); + + parameters.NoneMatch = await SetupPathMatchCondition(file, parameters.NoneMatch); + DataLakeRequestConditions conditions = BuildDataLakeRequestConditions(parameters); + + // Act + await TestHelper.CatchAsync( + async () => + { + var _ = (await file.ReadContentAsync(new DataLakeFileReadOptions + { + Conditions = conditions + })).Value; + }); + } + } + [RecordedTest] public async Task ReadAsync_Error() { @@ -3743,6 +3970,34 @@ await TestHelper.AssertExpectedExceptionAsync( e => Assert.AreEqual("BlobNotFound", e.ErrorCode)); } + [RecordedTest] + public async Task ReadStreamingAsync_Error() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeFileClient file = InstrumentClient(test.FileSystem.GetFileClient(GetNewFileName())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + file.ReadStreamingAsync(), + e => Assert.AreEqual("BlobNotFound", e.ErrorCode)); + } + + [RecordedTest] + public async Task ReadContentAsync_Error() + { + await using DisposingFileSystem test = await GetNewFileSystem(); + + // Arrange + DataLakeFileClient file = InstrumentClient(test.FileSystem.GetFileClient(GetNewFileName())); + + // Act + await TestHelper.AssertExpectedExceptionAsync( + file.ReadContentAsync(), + e => Assert.AreEqual("BlobNotFound", e.ErrorCode)); + } + [RecordedTest] public async Task AcquireLeaseAsync() { From 9fc1ec1c901ab7156121a0d393083beeb83ca684 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 16:46:10 -0400 Subject: [PATCH 06/18] Hiding FileDownloadInfo factory method --- .../src/Models/DataLakeModelFactory.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs index 55b7c47d1f8d6..34e280c2b2f63 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Models/DataLakeModelFactory.cs @@ -263,6 +263,7 @@ public static FileDownloadDetails FileDownloadDetails( /// /// Creates a new instance for mocking. /// + [EditorBrowsable(EditorBrowsableState.Never)] public static FileDownloadInfo FileDownloadInfo( long contentLength, Stream content, From 33265caa518883f95d07c4de1b2aa3ea9bb7e8b2 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 17:04:21 -0400 Subject: [PATCH 07/18] Small test refactor --- .../Azure.Storage.Files.DataLake/tests/FileClientTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs index 70e78dd1c2d93..b0be1c794ff28 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/FileClientTests.cs @@ -3517,8 +3517,8 @@ public async Task ReadContentAsync() Assert.IsNotNull(response.Value.Details.CreatedOn); Assert.IsNotNull(response.Value.Details.Metadata); - BinaryData actual = new BinaryData(response.Value.Content); - TestHelper.AssertSequenceEqual(data, actual.ToArray()); + byte[] actual = response.Value.Content.ToArray(); + TestHelper.AssertSequenceEqual(data, actual); } [RecordedTest] @@ -3655,8 +3655,8 @@ public async Task ReadContentAsyncACL() Assert.IsNotNull(response.Value.Details.CreatedOn); AssertAccessControlListEquality(AccessControlList, response.Value.Details.AccessControlList.ToList()); - BinaryData actual = new BinaryData(response.Value.Content); - TestHelper.AssertSequenceEqual(data, actual.ToArray()); + byte[] actual = response.Value.Content.ToArray(); + TestHelper.AssertSequenceEqual(data, actual); } [RecordedTest] From 82e3c162a7613dc61a2813f03edc335c165852a6 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 17:19:24 -0400 Subject: [PATCH 08/18] Added Readme changes --- sdk/storage/Azure.Storage.Files.DataLake/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/README.md b/sdk/storage/Azure.Storage.Files.DataLake/README.md index 249a5c0dced06..c10c561dc0b1d 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/README.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/README.md @@ -156,6 +156,16 @@ file.Flush(SampleFileContent.Length); Response fileContents = file.Read(); ``` +### Reading Streaming Data from a DataLake File +```C# Snippet:SampleSnippetDataLakeFileClient_ReadStreaming +Response fileContents = file.ReadStreaming(); +``` + +### Reading Content Data from a DataLake File +```C# Snippet:SampleSnippetDataLakeFileClient_ReadContent +Response fileContents = file.ReadContent(); +``` + ### Listing/Traversing through a DataLake Filesystem ```C# Snippet:SampleSnippetDataLakeFileClient_List foreach (PathItem pathItem in filesystem.GetPaths()) From 369409636730ac536d745d78f9acbf85b975c63c Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Wed, 14 Aug 2024 17:41:53 -0400 Subject: [PATCH 09/18] more api export --- .../api/Azure.Storage.Files.DataLake.net6.0.cs | 3 +++ .../api/Azure.Storage.Files.DataLake.netstandard2.0.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index 89592db960d80..3295fc1abbb6a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -753,6 +753,8 @@ public DataLakeMetrics() { } } public static partial class DataLakeModelFactory { + public static Azure.Storage.Files.DataLake.Models.DataLakeFileReadResult DataLakeFileReadResult(System.BinaryData content, Azure.Storage.Files.DataLake.Models.FileDownloadDetails details) { throw null; } + public static Azure.Storage.Files.DataLake.Models.DataLakeFileReadStreamingResult DataLakeFileReadStreamingResult(System.IO.Stream content, Azure.Storage.Files.DataLake.Models.FileDownloadDetails details) { throw null; } public static Azure.Storage.Files.DataLake.Models.DataLakeQueryError DataLakeQueryError(string name = null, string description = null, bool isFatal = false, long position = (long)0) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash) { throw null; } @@ -761,6 +763,7 @@ public static partial class DataLakeModelFactory [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash, System.DateTimeOffset createdOn, string encryptionContext) { throw null; } public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash, System.DateTimeOffset createdOn, string encryptionContext, System.Collections.Generic.IList accessControlList) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadInfo FileDownloadInfo(long contentLength, System.IO.Stream content, byte[] contentHash, Azure.Storage.Files.DataLake.Models.FileDownloadDetails properties) { throw null; } public static Azure.Storage.Files.DataLake.Models.FileSystemInfo FileSystemInfo(Azure.ETag etag, System.DateTimeOffset lastModified) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index 89592db960d80..3295fc1abbb6a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -753,6 +753,8 @@ public DataLakeMetrics() { } } public static partial class DataLakeModelFactory { + public static Azure.Storage.Files.DataLake.Models.DataLakeFileReadResult DataLakeFileReadResult(System.BinaryData content, Azure.Storage.Files.DataLake.Models.FileDownloadDetails details) { throw null; } + public static Azure.Storage.Files.DataLake.Models.DataLakeFileReadStreamingResult DataLakeFileReadStreamingResult(System.IO.Stream content, Azure.Storage.Files.DataLake.Models.FileDownloadDetails details) { throw null; } public static Azure.Storage.Files.DataLake.Models.DataLakeQueryError DataLakeQueryError(string name = null, string description = null, bool isFatal = false, long position = (long)0) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash) { throw null; } @@ -761,6 +763,7 @@ public static partial class DataLakeModelFactory [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash, System.DateTimeOffset createdOn, string encryptionContext) { throw null; } public static Azure.Storage.Files.DataLake.Models.FileDownloadDetails FileDownloadDetails(System.DateTimeOffset lastModified, System.Collections.Generic.IDictionary metadata, string contentRange, Azure.ETag eTag, string contentEncoding, string cacheControl, string contentDisposition, string contentLanguage, System.DateTimeOffset copyCompletionTime, string copyStatusDescription, string copyId, string copyProgress, System.Uri copySource, Azure.Storage.Files.DataLake.Models.CopyStatus copyStatus, Azure.Storage.Files.DataLake.Models.DataLakeLeaseDuration leaseDuration, Azure.Storage.Files.DataLake.Models.DataLakeLeaseState leaseState, Azure.Storage.Files.DataLake.Models.DataLakeLeaseStatus leaseStatus, string acceptRanges, bool isServerEncrypted, string encryptionKeySha256, byte[] contentHash, System.DateTimeOffset createdOn, string encryptionContext, System.Collections.Generic.IList accessControlList) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static Azure.Storage.Files.DataLake.Models.FileDownloadInfo FileDownloadInfo(long contentLength, System.IO.Stream content, byte[] contentHash, Azure.Storage.Files.DataLake.Models.FileDownloadDetails properties) { throw null; } public static Azure.Storage.Files.DataLake.Models.FileSystemInfo FileSystemInfo(Azure.ETag etag, System.DateTimeOffset lastModified) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] From 2dd598ec0ee512313b2934418ad3329d8d38227b Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 11:46:38 -0400 Subject: [PATCH 10/18] Addressed comments --- .../Azure.Storage.Files.DataLake/CHANGELOG.md | 2 +- .../Azure.Storage.Files.DataLake/README.md | 2 + .../samples/Sample01a_HelloWorld.cs | 2 + .../src/DataLakeFileClient.cs | 341 +++++++++--------- 4 files changed, 177 insertions(+), 170 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md index 2fdd1536a5f61..786a5ff1c0da6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/CHANGELOG.md @@ -3,7 +3,7 @@ ## 12.20.0-beta.2 (Unreleased) ### Features Added -- Added feature parity for DataLake download APIs with Blobs #45418 +- Deprecated Read()/ReadAsync() in favor of ReadStreaming()/ReadStreamingAsync() and ReadContent()/ReadContentAsync() for DataLake #45418 ### Breaking Changes diff --git a/sdk/storage/Azure.Storage.Files.DataLake/README.md b/sdk/storage/Azure.Storage.Files.DataLake/README.md index c10c561dc0b1d..e5136a042ff56 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/README.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/README.md @@ -159,11 +159,13 @@ Response fileContents = file.Read(); ### Reading Streaming Data from a DataLake File ```C# Snippet:SampleSnippetDataLakeFileClient_ReadStreaming Response fileContents = file.ReadStreaming(); +Stream readStream = fileContents.Value.Content; ``` ### Reading Content Data from a DataLake File ```C# Snippet:SampleSnippetDataLakeFileClient_ReadContent Response fileContents = file.ReadContent(); +BinaryData readData = fileContents.Value.Content; ``` ### Listing/Traversing through a DataLake Filesystem diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs index ed8ab29134cb0..d9cfb13887d46 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs @@ -363,6 +363,7 @@ public void ReadStreaming() // For large files, it may be faster to call ReadTo() #region Snippet:SampleSnippetDataLakeFileClient_ReadStreaming Response fileContents = file.ReadStreaming(); + Stream readStream = fileContents.Value.Content; #endregion Snippet:SampleSnippetDataLakeFileClient_ReadStreaming using (FileStream stream = File.OpenWrite(downloadPath)) { @@ -416,6 +417,7 @@ public void ReadContent() // For large files, it may be faster to call ReadTo() #region Snippet:SampleSnippetDataLakeFileClient_ReadContent Response fileContents = file.ReadContent(); + BinaryData readData = fileContents.Value.Content; #endregion Snippet:SampleSnippetDataLakeFileClient_ReadContent byte[] data = fileContents.Value.Content.ToArray(); using (FileStream stream = File.OpenWrite(downloadPath)) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 3da9eca92b479..5f56ec89440fa 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -2827,47 +2827,38 @@ internal virtual async Task> FlushInternal( #endregion #region Read Data + + #region Deprecated /// - /// The - /// operation downloads a file from the service, including its metadata - /// and properties. + /// The operation downloads a file from + /// the service, including its metadata and properties. /// /// For more information, see /// /// Get Blob. /// - /// - /// Optional parameters. - /// - /// - /// Optional to propagate - /// notifications that the operation should be cancelled. - /// /// - /// A describing the - /// downloaded file. contains - /// the file's data. + /// A describing the + /// downloaded file. contains + /// the blob's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual Response ReadStreaming( - DataLakeFileReadOptions options = default, - CancellationToken cancellationToken = default) + [EditorBrowsable(EditorBrowsableState.Never)] + public virtual Response Read() { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); try { scope.Start(); - Response response = _blockBlobClient.DownloadStreaming( - options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken); + Response response = _blockBlobClient.DownloadStreaming(); return Response.FromValue( - response.ToDataLakeFileReadStreamingResult(), + response.ToFileDownloadInfo(), response.GetRawResponse()); } catch (Exception ex) @@ -2882,46 +2873,36 @@ public virtual Response ReadStreaming( } /// - /// The - /// operation downloads a file from the service, including its metadata - /// and properties. + /// The operation downloads a file from + /// the service, including its metadata and properties. /// /// For more information, see /// /// Get Blob. /// - /// - /// Optional parameters. - /// - /// - /// Optional to propagate - /// notifications that the operation should be cancelled. - /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual Response ReadContent( - DataLakeFileReadOptions options = default, - CancellationToken cancellationToken = default) + [EditorBrowsable(EditorBrowsableState.Never)] + public virtual async Task> ReadAsync() { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); try { scope.Start(); - Response response = _blockBlobClient.DownloadContent( - options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken); + Response response + = await _blockBlobClient.DownloadStreamingAsync(cancellationToken: CancellationToken.None).ConfigureAwait(false); return Response.FromValue( - response.ToDataLakeFileReadResult(), + response.ToFileDownloadInfo(), response.GetRawResponse()); } catch (Exception ex) @@ -2936,13 +2917,17 @@ public virtual Response ReadContent( } /// - /// The operation downloads a file from + /// The operation downloads a file from /// the service, including its metadata and properties. /// /// For more information, see /// /// Get Blob. /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// /// /// A describing the /// downloaded file. contains @@ -2953,7 +2938,8 @@ public virtual Response ReadContent( /// a failure occurs. /// [EditorBrowsable(EditorBrowsableState.Never)] - public virtual Response Read() + public virtual Response Read( + CancellationToken cancellationToken = default) { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -2961,7 +2947,7 @@ public virtual Response Read() { scope.Start(); - Response response = _blockBlobClient.DownloadStreaming(); + Response response = _blockBlobClient.DownloadStreaming(cancellationToken: cancellationToken); return Response.FromValue( response.ToFileDownloadInfo(), @@ -2979,47 +2965,41 @@ public virtual Response Read() } /// - /// The - /// operation downloads a file from the service, including its metadata - /// and properties. + /// The operation downloads a file from + /// the service, including its metadata and properties. /// /// For more information, see /// /// Get Blob. /// - /// - /// Optional parameters. - /// /// /// Optional to propagate /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual async Task> ReadStreamingAsync( - DataLakeFileReadOptions options = default, + [EditorBrowsable(EditorBrowsableState.Never)] + public virtual async Task> ReadAsync( CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); try { scope.Start(); - Response response = await _blockBlobClient.DownloadStreamingAsync( - options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken) - .ConfigureAwait(false); + Response response + = await _blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).ConfigureAwait(false); return Response.FromValue( - response.ToDataLakeFileReadStreamingResult(), + response.ToFileDownloadInfo(), response.GetRawResponse()); } catch (Exception ex) @@ -3034,7 +3014,7 @@ public virtual async Task> ReadStreami } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3042,39 +3022,58 @@ public virtual async Task> ReadStreami /// /// Get Blob. /// - /// - /// Optional parameters. + /// + /// If provided, only donwload the bytes of the file in the specified + /// range. If not provided, download the entire file. + /// + /// + /// Optional to add conditions on + /// donwloading this file. + /// + /// + /// When set to true and specified together with the , + /// the service returns the MD5 hash for the range, as long as the + /// range is less than or equal to 4 MB in size. If this value is + /// specified without or set to true when the + /// range exceeds 4 MB in size, a + /// is thrown. /// /// /// Optional to propagate /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual async Task> ReadContentAsync( - DataLakeFileReadOptions options = default, - CancellationToken cancellationToken = default) + [EditorBrowsable(EditorBrowsableState.Never)] +#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. + public virtual Response Read( +#pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. + HttpRange range, + DataLakeRequestConditions conditions, + bool rangeGetContentHash, + CancellationToken cancellationToken) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); try { scope.Start(); - Response response = await _blockBlobClient.DownloadContentAsync( - options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken) - .ConfigureAwait(false); + Response response = _blockBlobClient.DownloadStreaming( + range: range, + conditions: conditions.ToBlobRequestConditions(), + rangeGetContentHash: rangeGetContentHash, + cancellationToken: cancellationToken); return Response.FromValue( - response.ToDataLakeFileReadResult(), + response.ToFileDownloadInfo(), response.GetRawResponse()); } catch (Exception ex) @@ -3089,13 +3088,34 @@ public virtual async Task> ReadContentAsync( } /// - /// The operation downloads a file from - /// the service, including its metadata and properties. + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. /// /// For more information, see /// /// Get Blob. /// + /// + /// If provided, only donwload the bytes of the file in the specified + /// range. If not provided, download the entire file. + /// + /// + /// Optional to add conditions on + /// donwloading this file. + /// + /// + /// When set to true and specified together with the , + /// the service returns the MD5 hash for the range, as long as the + /// range is less than or equal to 4 MB in size. If this value is + /// specified without or set to true when the + /// range exceeds 4 MB in size, a + /// is thrown. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// /// /// A describing the /// downloaded file. contains @@ -3106,7 +3126,13 @@ public virtual async Task> ReadContentAsync( /// a failure occurs. /// [EditorBrowsable(EditorBrowsableState.Never)] - public virtual async Task> ReadAsync() +#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. + public virtual async Task> ReadAsync( +#pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. + HttpRange range, + DataLakeRequestConditions conditions, + bool rangeGetContentHash, + CancellationToken cancellationToken) { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -3114,8 +3140,12 @@ public virtual async Task> ReadAsync() { scope.Start(); - Response response - = await _blockBlobClient.DownloadStreamingAsync(cancellationToken: CancellationToken.None).ConfigureAwait(false); + Response response = await _blockBlobClient.DownloadStreamingAsync( + range: range, + conditions: conditions.ToBlobRequestConditions(), + rangeGetContentHash: rangeGetContentHash, + cancellationToken: cancellationToken) + .ConfigureAwait(false); return Response.FromValue( response.ToFileDownloadInfo(), @@ -3133,13 +3163,17 @@ public virtual async Task> ReadAsync() } /// - /// The operation downloads a file from - /// the service, including its metadata and properties. + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. /// /// For more information, see /// /// Get Blob. /// + /// + /// Optional parameters. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -3147,7 +3181,7 @@ public virtual async Task> ReadAsync() /// /// A describing the /// downloaded file. contains - /// the blob's data. + /// the file's data. /// /// /// A will be thrown if @@ -3155,6 +3189,7 @@ public virtual async Task> ReadAsync() /// [EditorBrowsable(EditorBrowsableState.Never)] public virtual Response Read( + DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -3163,7 +3198,9 @@ public virtual Response Read( { scope.Start(); - Response response = _blockBlobClient.DownloadStreaming(cancellationToken: cancellationToken); + Response response = _blockBlobClient.DownloadStreaming( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken); return Response.FromValue( response.ToFileDownloadInfo(), @@ -3181,13 +3218,17 @@ public virtual Response Read( } /// - /// The operation downloads a file from - /// the service, including its metadata and properties. + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. /// /// For more information, see /// /// Get Blob. /// + /// + /// Optional parameters. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -3203,6 +3244,7 @@ public virtual Response Read( /// [EditorBrowsable(EditorBrowsableState.Never)] public virtual async Task> ReadAsync( + DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); @@ -3211,8 +3253,10 @@ public virtual async Task> ReadAsync( { scope.Start(); - Response response - = await _blockBlobClient.DownloadStreamingAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + Response response = await _blockBlobClient.DownloadStreamingAsync( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken) + .ConfigureAwait(false); return Response.FromValue( response.ToFileDownloadInfo(), @@ -3228,9 +3272,10 @@ public virtual async Task> ReadAsync( scope.Dispose(); } } + #endregion Deprecated /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3238,58 +3283,38 @@ public virtual async Task> ReadAsync( /// /// Get Blob. /// - /// - /// If provided, only donwload the bytes of the file in the specified - /// range. If not provided, download the entire file. - /// - /// - /// Optional to add conditions on - /// donwloading this file. - /// - /// - /// When set to true and specified together with the , - /// the service returns the MD5 hash for the range, as long as the - /// range is less than or equal to 4 MB in size. If this value is - /// specified without or set to true when the - /// range exceeds 4 MB in size, a - /// is thrown. + /// + /// Optional parameters. /// /// /// Optional to propagate /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - [EditorBrowsable(EditorBrowsableState.Never)] -#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. - public virtual Response Read( -#pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. - HttpRange range, - DataLakeRequestConditions conditions, - bool rangeGetContentHash, - CancellationToken cancellationToken) + public virtual Response ReadStreaming( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); try { scope.Start(); Response response = _blockBlobClient.DownloadStreaming( - range: range, - conditions: conditions.ToBlobRequestConditions(), - rangeGetContentHash: rangeGetContentHash, + options: options.ToBlobBaseDownloadOptions(), cancellationToken: cancellationToken); return Response.FromValue( - response.ToFileDownloadInfo(), + response.ToDataLakeFileReadStreamingResult(), response.GetRawResponse()); } catch (Exception ex) @@ -3304,7 +3329,7 @@ public virtual Response Read( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3312,59 +3337,38 @@ public virtual Response Read( /// /// Get Blob. /// - /// - /// If provided, only donwload the bytes of the file in the specified - /// range. If not provided, download the entire file. - /// - /// - /// Optional to add conditions on - /// donwloading this file. - /// - /// - /// When set to true and specified together with the , - /// the service returns the MD5 hash for the range, as long as the - /// range is less than or equal to 4 MB in size. If this value is - /// specified without or set to true when the - /// range exceeds 4 MB in size, a - /// is thrown. + /// + /// Optional parameters. /// /// /// Optional to propagate /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - [EditorBrowsable(EditorBrowsableState.Never)] -#pragma warning disable AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. - public virtual async Task> ReadAsync( -#pragma warning restore AZC0002 // DO ensure all service methods, both asynchronous and synchronous, take an optional CancellationToken parameter called cancellationToken. - HttpRange range, - DataLakeRequestConditions conditions, - bool rangeGetContentHash, - CancellationToken cancellationToken) + public virtual Response ReadContent( + DataLakeFileReadOptions options = default, + CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); try { scope.Start(); - Response response = await _blockBlobClient.DownloadStreamingAsync( - range: range, - conditions: conditions.ToBlobRequestConditions(), - rangeGetContentHash: rangeGetContentHash, - cancellationToken: cancellationToken) - .ConfigureAwait(false); + Response response = _blockBlobClient.DownloadContent( + options: options.ToBlobBaseDownloadOptions(), + cancellationToken: cancellationToken); return Response.FromValue( - response.ToFileDownloadInfo(), + response.ToDataLakeFileReadResult(), response.GetRawResponse()); } catch (Exception ex) @@ -3379,7 +3383,7 @@ public virtual async Task> ReadAsync( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3395,31 +3399,31 @@ public virtual async Task> ReadAsync( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - [EditorBrowsable(EditorBrowsableState.Never)] - public virtual Response Read( + public virtual async Task> ReadStreamingAsync( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); try { scope.Start(); - Response response = _blockBlobClient.DownloadStreaming( + Response response = await _blockBlobClient.DownloadStreamingAsync( options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken); + cancellationToken: cancellationToken) + .ConfigureAwait(false); return Response.FromValue( - response.ToFileDownloadInfo(), + response.ToDataLakeFileReadStreamingResult(), response.GetRawResponse()); } catch (Exception ex) @@ -3434,7 +3438,7 @@ public virtual Response Read( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3450,32 +3454,31 @@ public virtual Response Read( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - [EditorBrowsable(EditorBrowsableState.Never)] - public virtual async Task> ReadAsync( + public virtual async Task> ReadContentAsync( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(Read)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); try { scope.Start(); - Response response = await _blockBlobClient.DownloadStreamingAsync( + Response response = await _blockBlobClient.DownloadContentAsync( options: options.ToBlobBaseDownloadOptions(), cancellationToken: cancellationToken) .ConfigureAwait(false); return Response.FromValue( - response.ToFileDownloadInfo(), + response.ToDataLakeFileReadResult(), response.GetRawResponse()); } catch (Exception ex) From 0de888bb170e8219191cd5f18c158f43331aa7e9 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 11:51:33 -0400 Subject: [PATCH 11/18] Readjusted order of methods --- .../src/DataLakeFileClient.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 5f56ec89440fa..034cf32138396 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -3329,7 +3329,7 @@ public virtual Response ReadStreaming( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3345,30 +3345,31 @@ public virtual Response ReadStreaming( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual Response ReadContent( + public virtual async Task> ReadStreamingAsync( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); try { scope.Start(); - Response response = _blockBlobClient.DownloadContent( + Response response = await _blockBlobClient.DownloadStreamingAsync( options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken); + cancellationToken: cancellationToken) + .ConfigureAwait(false); return Response.FromValue( - response.ToDataLakeFileReadResult(), + response.ToDataLakeFileReadStreamingResult(), response.GetRawResponse()); } catch (Exception ex) @@ -3383,7 +3384,7 @@ public virtual Response ReadContent( } /// - /// The + /// The /// operation downloads a file from the service, including its metadata /// and properties. /// @@ -3399,31 +3400,30 @@ public virtual Response ReadContent( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded file. contains + /// A describing the + /// downloaded file. contains /// the file's data. /// /// /// A will be thrown if /// a failure occurs. /// - public virtual async Task> ReadStreamingAsync( + public virtual Response ReadContent( DataLakeFileReadOptions options = default, CancellationToken cancellationToken = default) { - DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); try { scope.Start(); - Response response = await _blockBlobClient.DownloadStreamingAsync( + Response response = _blockBlobClient.DownloadContent( options: options.ToBlobBaseDownloadOptions(), - cancellationToken: cancellationToken) - .ConfigureAwait(false); + cancellationToken: cancellationToken); return Response.FromValue( - response.ToDataLakeFileReadStreamingResult(), + response.ToDataLakeFileReadResult(), response.GetRawResponse()); } catch (Exception ex) From 364d3f2cee67647bec771dc61e59e1b240b8c4c5 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 12:11:02 -0400 Subject: [PATCH 12/18] Typo fix --- .../samples/Sample01a_HelloWorld.cs | 4 ++-- .../Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs index d9cfb13887d46..e067fdab7f9fc 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs @@ -327,7 +327,7 @@ public void Read() } /// - /// Download a DataLake File stream to a file. + /// Download a DataLake File's streaming data to a file. /// [Test] public void ReadStreaming() @@ -381,7 +381,7 @@ public void ReadStreaming() } /// - /// Download a DataLake File content to a file. + /// Download a DataLake File's content data to a file. /// [Test] public void ReadContent() diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 034cf32138396..7506af545b5bc 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -3023,7 +3023,7 @@ public virtual async Task> ReadAsync( /// Get Blob. /// /// - /// If provided, only donwload the bytes of the file in the specified + /// If provided, only download the bytes of the file in the specified /// range. If not provided, download the entire file. /// /// From 79aab44752e2ad4abb966a908ee21a78eec42786 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 12:22:51 -0400 Subject: [PATCH 13/18] tiny sample code refactor --- .../samples/Sample01a_HelloWorld.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs index e067fdab7f9fc..0b554a55b44e6 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs @@ -367,7 +367,7 @@ public void ReadStreaming() #endregion Snippet:SampleSnippetDataLakeFileClient_ReadStreaming using (FileStream stream = File.OpenWrite(downloadPath)) { - fileContents.Value.Content.CopyTo(stream); + readStream.CopyTo(stream); } // Verify the contents @@ -419,7 +419,7 @@ public void ReadContent() Response fileContents = file.ReadContent(); BinaryData readData = fileContents.Value.Content; #endregion Snippet:SampleSnippetDataLakeFileClient_ReadContent - byte[] data = fileContents.Value.Content.ToArray(); + byte[] data = readData.ToArray(); using (FileStream stream = File.OpenWrite(downloadPath)) { stream.Write(data, 0, data.Length); From b137f29c9a8e4a30af2fc2cea6dab0e984889376 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 12:34:39 -0400 Subject: [PATCH 14/18] Comment section typo fix --- .../Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 7506af545b5bc..1a877899aea84 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -3028,7 +3028,7 @@ public virtual async Task> ReadAsync( /// /// /// Optional to add conditions on - /// donwloading this file. + /// downloading this file. /// /// /// When set to true and specified together with the , @@ -3097,12 +3097,12 @@ public virtual Response Read( /// Get Blob. /// /// - /// If provided, only donwload the bytes of the file in the specified + /// If provided, only download the bytes of the file in the specified /// range. If not provided, download the entire file. /// /// /// Optional to add conditions on - /// donwloading this file. + /// downloading this file. /// /// /// When set to true and specified together with the , From adb8e63eb6f2d4f8d930df9acbd138d4eb52156b Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 13:00:42 -0400 Subject: [PATCH 15/18] Added Async samples --- .../samples/Sample01b_HelloWorldAsync.cs | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs index c00d412854193..e24c21ecee457 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs @@ -327,6 +327,111 @@ public async Task ReadAsync() } } + /// + /// Download a DataLake File's streaming data to a file. + /// + [Test] + public async Task ReadStreamingAsync() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string originalPath = CreateTempFile(SampleFileContent); + + // Get a temporary path on disk where we can download the file + string downloadPath = CreateTempPath(); + + // Make StorageSharedKeyCredential to pass to the serviceClient + string storageAccountName = StorageAccountName; + string storageAccountKey = StorageAccountKey; + Uri serviceUri = StorageAccountBlobUri; + StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + + // Create DataLakeServiceClient using StorageSharedKeyCredentials + DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); + + // Get a reference to a filesystem named "sample-filesystem-readasync" and then create it + DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read")); + await filesystem.CreateAsync(); + try + { + // Get a reference to a file named "sample-file" in a filesystem + DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); + + // First upload something the DataLake file so we have something to download + await file.UploadAsync(File.OpenRead(originalPath)); + + // Download the DataLake file's contents and save it to a file + // The ReadStreamingAsync() API downloads a file in a single requests. + // For large files, it may be faster to call ReadToAsync() + Response fileContents = await file.ReadStreamingAsync(); + Stream readStream = fileContents.Value.Content; + using (FileStream stream = File.OpenWrite(downloadPath)) + { + readStream.CopyTo(stream); + } + + // Verify the contents + Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); + } + finally + { + // Clean up after the test when we're finished + await filesystem.DeleteAsync(); + } + } + + /// + /// Download a DataLake File's content data to a file. + /// + [Test] + public async Task ReadContentAsync() + { + // Create a temporary Lorem Ipsum file on disk that we can upload + string originalPath = CreateTempFile(SampleFileContent); + + // Get a temporary path on disk where we can download the file + string downloadPath = CreateTempPath(); + + // Make StorageSharedKeyCredential to pass to the serviceClient + string storageAccountName = StorageAccountName; + string storageAccountKey = StorageAccountKey; + Uri serviceUri = StorageAccountBlobUri; + StorageSharedKeyCredential sharedKeyCredential = new StorageSharedKeyCredential(storageAccountName, storageAccountKey); + + // Create DataLakeServiceClient using StorageSharedKeyCredentials + DataLakeServiceClient serviceClient = new DataLakeServiceClient(serviceUri, sharedKeyCredential); + + // Get a reference to a filesystem named "sample-filesystem-readasync" and then create it + DataLakeFileSystemClient filesystem = serviceClient.GetFileSystemClient(Randomize("sample-filesystem-read")); + await filesystem.CreateAsync(); + try + { + // Get a reference to a file named "sample-file" in a filesystem + DataLakeFileClient file = filesystem.GetFileClient(Randomize("sample-file")); + + // First upload something the DataLake file so we have something to download + await file.UploadAsync(File.OpenRead(originalPath)); + + // Download the DataLake file's contents and save it to a file + // The ReadStreamingAsync() API downloads a file in a single requests. + // For large files, it may be faster to call ReadToAsync() + Response fileContents = await file.ReadContentAsync(); + BinaryData readData = fileContents.Value.Content; + byte[] data = readData.ToArray(); + using (FileStream stream = File.OpenWrite(downloadPath)) + { + stream.Write(data, 0, data.Length); + } + + // Verify the contents + Assert.AreEqual(SampleFileContent, File.ReadAllText(downloadPath)); + } + finally + { + // Clean up after the test when we're finished + await filesystem.DeleteAsync(); + } + } + /// /// Download a DataLake File directly to file. /// From b3978a04170f70df3e5586473ff9bbea9c96147d Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 13:05:38 -0400 Subject: [PATCH 16/18] small comment change --- .../samples/Sample01b_HelloWorldAsync.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs index e24c21ecee457..227cdfaf27a7b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01b_HelloWorldAsync.cs @@ -412,7 +412,7 @@ public async Task ReadContentAsync() await file.UploadAsync(File.OpenRead(originalPath)); // Download the DataLake file's contents and save it to a file - // The ReadStreamingAsync() API downloads a file in a single requests. + // The ReadContentAsync() API downloads a file in a single requests. // For large files, it may be faster to call ReadToAsync() Response fileContents = await file.ReadContentAsync(); BinaryData readData = fileContents.Value.Content; From 6a1b571cacc6183c76e3b2ffcc94e21c0e7d6ff9 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Thu, 15 Aug 2024 14:14:47 -0400 Subject: [PATCH 17/18] removed unused import --- .../Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs index 0b554a55b44e6..f8369e587453a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/samples/Sample01a_HelloWorld.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.IO.Pipes; using System.Linq; using System.Reflection.Metadata.Ecma335; using Azure.Storage.Files.DataLake; From dc37bd69b1b6dba405cf6eafcf8d51f9e47428e1 Mon Sep 17 00:00:00 2001 From: nickliu-msft Date: Mon, 19 Aug 2024 21:15:24 -0400 Subject: [PATCH 18/18] Added all 3 explicit overload methods for ReadContent & ReadStreaming --- .../Azure.Storage.Files.DataLake.net6.0.cs | 8 + ...e.Storage.Files.DataLake.netstandard2.0.cs | 8 + .../src/DataLakeFileClient.cs | 372 ++++++++++++++++++ 3 files changed, 388 insertions(+) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs index 3295fc1abbb6a..84ffe47a07714 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.net6.0.cs @@ -186,10 +186,18 @@ public DataLakeFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCred public virtual System.Threading.Tasks.Task> ReadAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent() { throw null; } public virtual Azure.Response ReadContent(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync() { throw null; } public virtual System.Threading.Tasks.Task> ReadContentAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming() { throw null; } public virtual Azure.Response ReadStreaming(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync() { throw null; } public virtual System.Threading.Tasks.Task> ReadStreamingAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeFileReadToOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, Azure.Storage.StorageTransferOptions transferOptions, System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs index 3295fc1abbb6a..84ffe47a07714 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/api/Azure.Storage.Files.DataLake.netstandard2.0.cs @@ -186,10 +186,18 @@ public DataLakeFileClient(System.Uri fileUri, Azure.Storage.StorageSharedKeyCred public virtual System.Threading.Tasks.Task> ReadAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual System.Threading.Tasks.Task> ReadAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent() { throw null; } public virtual Azure.Response ReadContent(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadContent(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync() { throw null; } public virtual System.Threading.Tasks.Task> ReadContentAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadContentAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming() { throw null; } public virtual Azure.Response ReadStreaming(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual Azure.Response ReadStreaming(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync() { throw null; } public virtual System.Threading.Tasks.Task> ReadStreamingAsync(Azure.Storage.Files.DataLake.Models.DataLakeFileReadOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public virtual System.Threading.Tasks.Task> ReadStreamingAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeFileReadToOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public virtual Azure.Response ReadTo(System.IO.Stream destination, Azure.Storage.Files.DataLake.Models.DataLakeRequestConditions conditions, Azure.Storage.StorageTransferOptions transferOptions, System.Threading.CancellationToken cancellationToken) { throw null; } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 1a877899aea84..ef00d2b9c4ac0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -3274,6 +3274,192 @@ public virtual async Task> ReadAsync( } #endregion Deprecated + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadStreaming() + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadStreaming(); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadStreamingAsync() + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadStreamingAsync() + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadStreaming( + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadStreaming( + cancellationToken: cancellationToken); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadStreamingAsync( + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadStreaming)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadStreamingAsync( + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadStreamingResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + /// /// The /// operation downloads a file from the service, including its metadata @@ -3383,6 +3569,192 @@ public virtual async Task> ReadStreami } } + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadContent() + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadContent(); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadContentAsync() + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadContentAsync() + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual Response ReadContent( + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = _blockBlobClient.DownloadContent( + cancellationToken: cancellationToken); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + + /// + /// The + /// operation downloads a file from the service, including its metadata + /// and properties. + /// + /// For more information, see + /// + /// Get Blob. + /// + /// + /// Optional to propagate + /// notifications that the operation should be cancelled. + /// + /// + /// A describing the + /// downloaded file. contains + /// the file's data. + /// + /// + /// A will be thrown if + /// a failure occurs. + /// + public virtual async Task> ReadContentAsync( + CancellationToken cancellationToken = default) + { + DiagnosticScope scope = ClientConfiguration.ClientDiagnostics.CreateScope($"{nameof(DataLakeFileClient)}.{nameof(ReadContent)}"); + + try + { + scope.Start(); + + Response response = await _blockBlobClient.DownloadContentAsync( + cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Response.FromValue( + response.ToDataLakeFileReadResult(), + response.GetRawResponse()); + } + catch (Exception ex) + { + scope.Failed(ex); + throw; + } + finally + { + scope.Dispose(); + } + } + /// /// The /// operation downloads a file from the service, including its metadata