[storage-file-datalake]Upload stream fix

sdk/storage/storage-blob/package.json

@@ -14,6 +14,7 @@
     "./dist-esm/storage-blob/src/BlobDownloadResponse.js": "./dist-esm/storage-blob/src/BlobDownloadResponse.browser.js",
     "./dist-esm/storage-blob/src/BlobQueryResponse.js": "./dist-esm/storage-blob/src/BlobQueryResponse.browser.js",
     "./dist-esm/storage-blob/src/utils/BufferScheduler.js": "./dist-esm/storage-blob/src/utils/BufferScheduler.browser.js",
+    "./dist-esm/storage-common/src/index.js": "./dist-esm/storage-common/src/index.browser.js",
     "fs": false,
     "os": false,
     "process": false
@@ -65,6 +66,7 @@
     "dist/",
     "dist-esm/storage-blob/src/",
     "dist-esm/storage-internal-avro/src/",
+    "dist-esm/storage-common/src/",
     "typings/latest/storage-blob.d.ts",
     "typings/3.1/storage-blob.d.ts",
     "README.md",

sdk/storage/storage-blob/src/Clients.ts

@@ -10,7 +10,7 @@ import {
   isTokenCredential,
   TokenCredential,
   TransferProgressEvent,
-  URLBuilder,
+  URLBuilder
 } from "@azure/core-http";
 import { PollerLike, PollOperationState } from "@azure/core-lro";
 import { PagedAsyncIterableIterator, PageSettings } from "@azure/core-paging";
@@ -21,7 +21,13 @@ import { BlobDownloadResponse } from "./BlobDownloadResponse";
 import { BlobQueryResponse } from "./BlobQueryResponse";
 import { AnonymousCredential } from "./credentials/AnonymousCredential";
 import { StorageSharedKeyCredential } from "./credentials/StorageSharedKeyCredential";
-import { AppendBlob, Blob as StorageBlob, BlockBlob, Container, PageBlob } from "./generated/src/operations";
+import {
+  AppendBlob,
+  Blob as StorageBlob,
+  BlockBlob,
+  Container,
+  PageBlob
+} from "./generated/src/operations";
 import { StorageClientContext } from "./generated/src/storageClient";
 import {
   AppendBlobAppendBlockFromUrlResponse,
@@ -79,7 +85,7 @@ import {
   PublicAccessType,
   RehydratePriority,
   SequenceNumberActionType,
-  SignedIdentifierModel,
+  SignedIdentifierModel
 } from "./generatedModels";
 import {
   AppendBlobRequestConditions,
@@ -97,18 +103,18 @@ import {
 import {
   PageBlobGetPageRangesDiffResponse,
   PageBlobGetPageRangesResponse,
-  rangeResponseFromModel,
+  rangeResponseFromModel
 } from "./PageBlobRangeResponse";
 import { newPipeline, Pipeline, StoragePipelineOptions } from "./Pipeline";
 import {
   BlobBeginCopyFromUrlPoller,
   BlobBeginCopyFromUrlPollState,
-  CopyPollerBlobClient,
+  CopyPollerBlobClient
 } from "./pollers/BlobStartCopyFromUrlPoller";
 import { Range, rangeToString } from "./Range";
 import { CommonOptions, StorageClient } from "./StorageClient";
 import { Batch } from "./utils/Batch";
-import { BufferScheduler } from "./utils/BufferScheduler";
+import { BufferScheduler } from "../../storage-common/src";
 import {
   BLOCK_BLOB_MAX_BLOCKS,
   BLOCK_BLOB_MAX_STAGE_BLOCK_BYTES,
@@ -118,7 +124,7 @@ import {
   DEFAULT_MAX_DOWNLOAD_RETRY_REQUESTS,
   ETagAny,
   ETagNone,
-  URLConstants,
+  URLConstants
 } from "./utils/constants";
 import { createSpan } from "./utils/tracing";
 import {
@@ -132,10 +138,14 @@ import {
   toBlobTagsString,
   toQuerySerialization,
   toTags,
-  truncatedISO8061Date,
+  truncatedISO8061Date
 } from "./utils/utils.common";
-import { fsCreateReadStream, fsStat, readStreamToLocalFile, streamToBuffer } from "./utils/utils.node";
-
+import {
+  fsCreateReadStream,
+  fsStat,
+  readStreamToLocalFile,
+  streamToBuffer
+} from "./utils/utils.node";
 
 /**
  * Options to configure the {@link BlobClient.beginCopyFromURL} operation.
@@ -2434,7 +2444,7 @@ export interface AppendBlobCreateIfNotExistsOptions extends CommonOptions {
 }
 
 /**
- * Options to configure {@link AppendBlobClient.seal} operation. 
+ * Options to configure {@link AppendBlobClient.seal} operation.
  *
  * @export
  * @interface AppendBlobSealOptions
@@ -2897,16 +2907,10 @@ export class AppendBlobClient extends BlobClient {
    * @returns {Promise<AppendBlobAppendBlockResponse>}
    * @memberof AppendBlobClient
    */
-  public async seal(
-    options: AppendBlobSealOptions = {}
-  ): Promise<AppendBlobAppendBlockResponse> {
-    const { span, spanOptions } = createSpan(
-      "AppendBlobClient-seal",
-      options.tracingOptions
-    );
+  public async seal(options: AppendBlobSealOptions = {}): Promise<AppendBlobAppendBlockResponse> {
+    const { span, spanOptions } = createSpan("AppendBlobClient-seal", options.tracingOptions);
     options.conditions = options.conditions || {};
     try {
-
       return await this.appendBlobContext.seal({
         abortSignal: options.abortSignal,
         appendPositionAccessConditions: options.conditions,

sdk/storage/storage-blob/tsconfig.json

@@ -11,7 +11,13 @@
     "node_modules",
     "./typings/**/*.d.ts",
     "../storage-internal-avro/node_modules",
+    "../storage-common/node_modules",
     "./samples/**"
   ],
-  "include": ["./src/**/*.ts", "./test/**/*.ts", "../storage-internal-avro/**/*.ts"]
+  "include": [
+    "./src/**/*.ts",
+    "./test/**/*.ts",
+    "../storage-internal-avro/**/*.ts",
+    "../storage-common/**/*.ts"
+  ]
 }

sdk/storage/storage-blob/src/utils/PooledBuffer.ts → sdk/storage/storage-common/src/PooledBuffer.ts

@@ -3,16 +3,12 @@
 
 import { BuffersStream } from "./BuffersStream";
 import { Readable } from "stream";
-import { BLOCK_BLOB_MAX_STAGE_BLOCK_BYTES } from "./constants";
 
 /**
  * maxBufferLength is max size of each buffer in the pooled buffers.
  */
-const maxBufferLength = Math.min(
-  // Can't use import as Typescript doesn't recognize "buffer".
-  require("buffer").constants.MAX_LENGTH,
-  BLOCK_BLOB_MAX_STAGE_BLOCK_BYTES
-);
+// Can't use import as Typescript doesn't recognize "buffer".
+const maxBufferLength = require("buffer").constants.MAX_LENGTH;
 
 /**
  * This class provides a buffer container which conceptually has no hard size limit.

sdk/storage/storage-common/src/index.browser.ts

@@ -0,0 +1,4 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+export * from "./BufferScheduler.browser";

sdk/storage/storage-common/src/index.ts

@@ -0,0 +1,4 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+export * from "./BufferScheduler";

sdk/storage/storage-file-datalake/api-extractor.json

@@ -1,6 +1,6 @@
 {
   "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
-  "mainEntryPointFilePath": "typings/latest/src/index.d.ts",
+  "mainEntryPointFilePath": "typings/latest/storage-file-datalake/src/index.d.ts",
   "docModel": {
     "enabled": true
   },

sdk/storage/storage-file-datalake/package.json

@@ -4,7 +4,7 @@
   "description": "Microsoft Azure Storage SDK for JavaScript - DataLake",
   "sdk-type": "client",
   "main": "./dist/index.js",
-  "module": "./dist-esm/src/index.js",
+  "module": "./dist-esm/storage-file-datalake/src/index.js",
   "types": "./typings/latest/storage-file-datalake.d.ts",
   "typesVersions": {
     "<3.6": {
@@ -14,10 +14,11 @@
     }
   },
   "browser": {
-    "./dist-esm/src/index.js": "./dist-esm/src/index.browser.js",
-    "./dist-esm/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/src/credentials/StorageSharedKeyCredential.browser.js",
-    "./dist-esm/src/utils/utils.node.js": "./dist-esm/src/utils/utils.browser.js",
-    "./dist-esm/test/utils/index.js": "./dist-esm/test/utils/index.browser.js",
+    "./dist-esm/storage-file-datalake/src/index.js": "./dist-esm/storage-file-datalake/src/index.browser.js",
+    "./dist-esm/storage-file-datalake/src/credentials/StorageSharedKeyCredential.js": "./dist-esm/storage-file-datalake/src/credentials/StorageSharedKeyCredential.browser.js",
+    "./dist-esm/storage-file-datalake/src/utils/utils.node.js": "./dist-esm/storage-file-datalake/src/utils/utils.browser.js",
+    "./dist-esm/storage-file-datalake/test/utils/index.js": "./dist-esm/storage-file-datalake/test/utils/index.browser.js",
+    "./dist-esm/storage-common/src/index.js": "./dist-esm/storage-common/src/index.browser.js",
     "fs": false,
     "os": false,
     "process": false
@@ -41,7 +42,7 @@
     "execute:samples": "echo skipped until samples automation ready",
     "format": "prettier --write --config ../../../.prettierrc.json --ignore-path ../../../.prettierignore \"src/**/*.ts\" \"test/**/*.ts\" \"*.{js,json}\"",
     "integration-test:browser": "karma start --single-run",
-    "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 120000  dist-esm/test/*.spec.js dist-esm/test/node/*.spec.js",
+    "integration-test:node": "nyc mocha -r esm --require source-map-support/register --reporter ../../../common/tools/mocha-multi-reporter.js --full-trace -t 120000  dist-esm/storage-file-datalake/test/*.spec.js dist-esm/storage-file-datalake/test/node/*.spec.js",
     "integration-test": "npm run integration-test:node && npm run integration-test:browser",
     "lint:fix": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts --fix",
     "lint": "eslint -c ../../.eslintrc.old.json src test samples --ext .ts -f html -o storage-file-share-lintReport.html || exit 0",
@@ -57,7 +58,8 @@
   "files": [
     "BreakingChanges.md",
     "dist/",
-    "dist-esm/src/",
+    "dist-esm/storage-file-datalake/src/",
+    "dist-esm/storage-common/src/",
     "typings/latest/storage-file-datalake.d.ts",
     "typings/3.1/storage-file-datalake.d.ts",
     "README.md",

sdk/storage/storage-file-datalake/rollup.base.config.js

@@ -33,7 +33,7 @@ export function nodeConfig(test = false) {
     "util"
   ];
   const baseConfig = {
-    input: "dist-esm/src/index.js",
+    input: "dist-esm/storage-file-datalake/src/index.js",
     external: depNames.concat(externalNodeBuiltins),
     output: {
       file: "dist/index.js",
@@ -78,9 +78,9 @@ export function nodeConfig(test = false) {
   if (test) {
     // entry point is every test file
     baseConfig.input = [
-      "dist-esm/test/*.spec.js",
-      "dist-esm/test/node/*.spec.js",
-      "dist-esm/src/index.js"
+      "dist-esm/storage-file-datalake/test/*.spec.js",
+      "dist-esm/storage-file-datalake/test/node/*.spec.js",
+      "dist-esm/storage-file-datalake/src/index.js"
     ];
     baseConfig.plugins.unshift(multiEntry());
 
@@ -105,7 +105,7 @@ export function nodeConfig(test = false) {
 
 export function browserConfig(test = false) {
   const baseConfig = {
-    input: "dist-esm/src/index.browser.js",
+    input: "dist-esm/storage-file-datalake/src/index.browser.js",
     output: {
       file: "dist-browser/azure-storage-file-datalake.js",
       banner: banner,
@@ -170,7 +170,7 @@ export function browserConfig(test = false) {
   };
 
   if (test) {
-    baseConfig.input = ["dist-esm/test/*.spec.js", "dist-esm/test/browser/*.spec.js"];
+    baseConfig.input = ["dist-esm/storage-file-datalake/test/*.spec.js", "dist-esm/storage-file-datalake/test/browser/*.spec.js"];
     baseConfig.plugins.unshift(multiEntry({ exports: false }));
     baseConfig.output.file = "dist-test/index.browser.js";
     // mark fs-extra as external

sdk/storage/storage-file-datalake/src/clients.ts

@@ -76,7 +76,7 @@ import {
   BLOCK_BLOB_MAX_BLOCKS,
   ETagAny
 } from "./utils/constants";
-import { BufferScheduler } from "./utils/BufferScheduler";
+import { BufferScheduler } from "../../storage-common/src";
 import { Batch } from "./utils/Batch";
 import { fsStat, fsCreateReadStream } from "./utils/utils.node";
 
@@ -1570,15 +1570,15 @@ export class DataLakeFileClient extends DataLakePathClient {
         stream,
         options.chunkSize,
         options.maxConcurrency,
-        async (buffer: Buffer, offset?: number) => {
-          await this.append(buffer, offset!, buffer.length, {
+        async (body, length, offset) => {
+          await this.append(body, offset!, length, {
             abortSignal: options.abortSignal,
             conditions: options.conditions,
             tracingOptions: { ...options!.tracingOptions, spanOptions }
           });
 
           // Update progress after block is successfully uploaded to server, in case of block trying
-          transferProgress += buffer.length;
+          transferProgress += length;
           if (options.onProgress) {
             options.onProgress({ loadedBytes: transferProgress });
           }

sdk/storage/storage-file-datalake/tsconfig.json

@@ -6,6 +6,11 @@
     "outDir": "./dist-esm",
     "lib": ["dom", "es5", "es6", "es7", "esnext"]
   },
-  "exclude": ["node_modules", "./typings/**/*.d.ts", "./samples/**"],
-  "include": ["./src/**/*.ts", "./test/**/*.ts"]
+  "exclude": [
+    "node_modules",
+    "./typings/**/*.d.ts",
+    "./samples/**",
+    "../storage-common/node_modules"
+  ],
+  "include": ["./src/**/*.ts", "./test/**/*.ts", "../storage-common/**/*.ts"]
 }