diff --git a/packages/middleware-flexible-checksums/package.json b/packages/middleware-flexible-checksums/package.json index 8b7a26c69cd86..83c4cfa1ab5b0 100644 --- a/packages/middleware-flexible-checksums/package.json +++ b/packages/middleware-flexible-checksums/package.json @@ -10,21 +10,20 @@ "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4", "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", "test": "yarn g:vitest run", - "test:integration": "yarn g:vitest run -c vitest.config.integ.ts", "test:watch": "yarn g:vitest watch", - "test:integration:watch": "yarn g:vitest watch -c vitest.config.integ.ts" + "test:integration": "yarn g:vitest run -c vitest.config.integ.ts", + "test:integration:watch": "yarn g:vitest watch -c vitest.config.integ.ts", + "test:e2e": "yarn g:vitest run -c vitest.config.e2e.ts", + "test:e2e:watch": "yarn g:vitest watch -c vitest.config.e2e.ts" }, "main": "./dist-cjs/index.js", "module": "./dist-es/index.js", "browser": { - "./dist-es/getCrc32ChecksumAlgorithmFunction": "./dist-es/getCrc32ChecksumAlgorithmFunction.browser", - "./dist-es/streams/create-read-stream-on-buffer": "./dist-es/streams/create-read-stream-on-buffer.browser" + "./dist-es/getCrc32ChecksumAlgorithmFunction": "./dist-es/getCrc32ChecksumAlgorithmFunction.browser" }, "react-native": { "./dist-es/getCrc32ChecksumAlgorithmFunction": "./dist-es/getCrc32ChecksumAlgorithmFunction.browser", - "./dist-es/streams/create-read-stream-on-buffer": "./dist-es/streams/create-read-stream-on-buffer.browser", - "./dist-cjs/getCrc32ChecksumAlgorithmFunction": "./dist-cjs/getCrc32ChecksumAlgorithmFunction.browser", - "./dist-cjs/streams/create-read-stream-on-buffer": "./dist-cjs/streams/create-read-stream-on-buffer.browser" + "./dist-cjs/getCrc32ChecksumAlgorithmFunction": "./dist-cjs/getCrc32ChecksumAlgorithmFunction.browser" }, "types": "./dist-types/index.d.ts", "author": { diff --git a/packages/middleware-flexible-checksums/src/flexibleChecksumsResponseMiddleware.ts b/packages/middleware-flexible-checksums/src/flexibleChecksumsResponseMiddleware.ts index 0bba7e0b96505..d94f476350006 100644 --- a/packages/middleware-flexible-checksums/src/flexibleChecksumsResponseMiddleware.ts +++ b/packages/middleware-flexible-checksums/src/flexibleChecksumsResponseMiddleware.ts @@ -14,8 +14,6 @@ import { ChecksumAlgorithm } from "./constants"; import { getChecksumAlgorithmListForResponse } from "./getChecksumAlgorithmListForResponse"; import { getChecksumLocationName } from "./getChecksumLocationName"; import { isChecksumWithPartNumber } from "./isChecksumWithPartNumber"; -import { isStreaming } from "./isStreaming"; -import { createReadStreamOnBuffer } from "./streams/create-read-stream-on-buffer"; import { validateChecksumFromResponse } from "./validateChecksumFromResponse"; export interface FlexibleChecksumsResponseMiddlewareConfig { @@ -66,7 +64,6 @@ export const flexibleChecksumsResponseMiddleware = const result = await next(args); const response = result.response as HttpResponse; - let collectedStream: Uint8Array | undefined = undefined; const { requestValidationModeMember, responseAlgorithms } = middlewareConfig; // @ts-ignore Element implicitly has an 'any' type for input[requestValidationModeMember] @@ -84,22 +81,11 @@ export const flexibleChecksumsResponseMiddleware = return result; } - const isStreamingBody = isStreaming(response.body); - - if (isStreamingBody) { - collectedStream = await config.streamCollector(response.body); - response.body = createReadStreamOnBuffer(collectedStream); - } - - await validateChecksumFromResponse(result.response as HttpResponse, { + await validateChecksumFromResponse(response as HttpResponse, { config, responseAlgorithms, logger: context.logger, }); - - if (isStreamingBody && collectedStream) { - response.body = createReadStreamOnBuffer(collectedStream); - } } return result; diff --git a/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts new file mode 100644 index 0000000000000..6dc3b719f30eb --- /dev/null +++ b/packages/middleware-flexible-checksums/src/middleware-flexible-checksums.e2e.spec.ts @@ -0,0 +1,89 @@ +import { S3 } from "@aws-sdk/client-s3"; +import type { HttpRequest, HttpResponse } from "@smithy/types"; +import { headStream } from "@smithy/util-stream"; +import { Readable } from "node:stream"; +import { beforeAll, describe, expect, test as it } from "vitest"; + +import { getIntegTestResources } from "../../../tests/e2e/get-integ-test-resources"; + +describe("S3 checksums", () => { + let s3: S3; + let s3_noChecksum: S3; + let Bucket: string; + let Key: string; + let region: string; + const expected = new Uint8Array([97, 98, 99, 100]); + + beforeAll(async () => { + const integTestResourcesEnv = await getIntegTestResources(); + Object.assign(process.env, integTestResourcesEnv); + + region = process?.env?.AWS_SMOKE_TEST_REGION as string; + Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string; + + s3 = new S3({ region }); + s3_noChecksum = new S3({ + region, + requestChecksumCalculation: "WHEN_REQUIRED", + responseChecksumValidation: "WHEN_REQUIRED", + }); + Key = "middleware-flexible-checksum.txt"; + + s3.middlewareStack.add( + (next) => async (args) => { + const r = await next(args); + const reqHeader = (args.request as HttpRequest).headers["x-amz-sdk-checksum-algorithm"]; + const resHeader = (r.response as HttpResponse).headers["x-amz-checksum-crc32"]; + if (reqHeader) { + expect(reqHeader).toEqual("CRC32"); + } + if (resHeader) { + expect(resHeader).toEqual("7YLNEQ=="); + } + return r; + }, + { + step: "build", + override: true, + name: "assert", + } + ); + + await s3.putObject({ Bucket, Key, Body: "abcd" }); + }); + + it("an object should have checksum by default", async () => { + await s3.getObject({ Bucket, Key }); + }); + + describe("the stream returned by S3::getObject should function interchangeably between ChecksumStream and default streams", () => { + it("when collecting the stream", async () => { + const defaultStream = (await s3_noChecksum.getObject({ Bucket, Key })).Body as Readable; + const checksumStream = (await s3.getObject({ Bucket, Key })).Body as Readable; + + expect(defaultStream.constructor.name).not.toEqual("ChecksumStream"); + expect(checksumStream.constructor.name).toEqual("ChecksumStream"); + + const defaultStreamContents = await headStream(defaultStream, Infinity); + const checksumStreamContents = await headStream(checksumStream, Infinity); + + expect(defaultStreamContents).toEqual(expected); + expect(defaultStream.closed).toBe(true); + expect(checksumStreamContents).toEqual(expected); + expect(checksumStream.closed).toBe(true); + }); + + it("when piping the stream to web stream api", async () => { + const defaultStream = Readable.toWeb((await s3_noChecksum.getObject({ Bucket, Key })).Body as Readable); + const checksumStream = Readable.toWeb((await s3.getObject({ Bucket, Key })).Body as Readable); + + const defaultStreamContents = await headStream(defaultStream as ReadableStream, Infinity); + const checksumStreamContents = await headStream(checksumStream as ReadableStream, Infinity); + + expect(defaultStreamContents).toEqual(expected); + await expect(defaultStream.getReader().closed).resolves.toBe(undefined); + expect(checksumStreamContents).toEqual(expected); + await expect(checksumStream.getReader().closed).resolves.toBe(undefined); + }); + }); +}); diff --git a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.browser.ts b/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.browser.ts deleted file mode 100644 index e55ae6ac33e78..0000000000000 --- a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.browser.ts +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Convert a buffer to a readable stream. - */ -export function createReadStreamOnBuffer(buffer: Uint8Array): ReadableStream { - return new Blob([buffer]).stream(); -} diff --git a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.spec.ts b/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.spec.ts deleted file mode 100644 index ecd2429fce285..0000000000000 --- a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.spec.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { streamCollector } from "@smithy/node-http-handler"; -import { toUtf8 } from "@smithy/util-utf8"; -import { describe, expect, test as it } from "vitest"; - -import { createReadStreamOnBuffer } from "./create-read-stream-on-buffer"; - -describe(createReadStreamOnBuffer.name, () => { - it("converts a buffer into stream of the same contents", async () => { - const buffer = Buffer.from("abcd"); - - const stream = createReadStreamOnBuffer(buffer); - - // changing the buffer here also changes the stream since it has not been collected. - buffer[0] = "z".charCodeAt(0); - - const bytes = await streamCollector(stream); - - // changing the buffer here does not change the stream since it has been collected. - buffer[1] = "z".charCodeAt(0); - - expect(toUtf8(bytes)).not.toEqual("zzcd"); - expect(toUtf8(bytes)).toEqual("zbcd"); - }); -}); diff --git a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.ts b/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.ts deleted file mode 100644 index cec8dedf1b27b..0000000000000 --- a/packages/middleware-flexible-checksums/src/streams/create-read-stream-on-buffer.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Readable, Transform } from "stream"; - -/** - * Convert a buffer to a readable stream. - */ -export function createReadStreamOnBuffer(buffer: Uint8Array): Readable { - const stream = new Transform(); - stream.push(buffer); - stream.push(null); - return stream; -} diff --git a/packages/middleware-flexible-checksums/vitest.config.e2e.ts b/packages/middleware-flexible-checksums/vitest.config.e2e.ts new file mode 100644 index 0000000000000..92073c6cfcf00 --- /dev/null +++ b/packages/middleware-flexible-checksums/vitest.config.e2e.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.e2e.spec.ts"], + environment: "node", + }, +});