From df99fc33a43982e1c59000721a535f6fe77a3c23 Mon Sep 17 00:00:00 2001 From: George Fu Date: Fri, 23 Sep 2022 11:26:58 -0400 Subject: [PATCH] feat(endpoint): endpoints 2.0 existing package changes (#3947) * feat(endpoint): endpoints 2.0 existing package changes * feat(endpoint): enable middleware stack identification * feat(endpoint): update existing packages for endpoints v2 * feat(endpoint): address PR style comments * feat(endpoint): fixes to types * feat(endpoint): additional fixes from testing * feat(endpoint): s3 presigned post fixes for endpoints 2.0 * feat(endpoint): unit test fixes * feat(endpoint): type fixes * feat(endpoint): types and docs updates * feat(endpoint): s3 bucket name forcePathStyle * feat(endpoint): type fix * feat(endpoint): java code format * feat(endpoint): type fix Co-authored-by: AllanZhengYP --- .../aws/typescript/codegen/AddS3Config.java | 160 +++++++++--------- lib/lib-storage/package.json | 2 + lib/lib-storage/src/Upload.ts | 27 ++- .../endpointsConfig/resolveEndpointsConfig.ts | 5 +- packages/middleware-endpoint/package.json | 2 + .../adaptors/getEndpointFromInstructions.ts | 77 +++++++++ .../middleware-endpoint/src/adaptors/index.ts | 2 + .../src/adaptors/toEndpointV1.ts | 14 ++ .../src/endpointMiddleware.ts | 57 ++++++- .../src/getEndpointPlugin.ts | 26 +-- packages/middleware-endpoint/src/index.ts | 1 + .../src/resolveEndpointConfig.ts | 35 ++-- .../src/service-customizations/index.ts | 1 + .../src/service-customizations/s3.ts | 50 ++++++ packages/middleware-endpoint/src/types.ts | 2 +- .../middleware-sdk-s3/src/configuration.ts | 31 ++++ packages/middleware-sdk-s3/src/index.ts | 1 + .../src/write-get-object-response-endpoint.ts | 2 +- .../src/configuration.ts | 10 +- packages/middleware-serde/src/serdePlugin.ts | 10 +- .../src/serializerMiddleware.ts | 16 +- packages/middleware-signing/package.json | 2 +- .../src/configuration.spec.ts | 15 +- .../middleware-signing/src/configurations.ts | 72 +++++--- packages/middleware-signing/src/middleware.ts | 9 +- .../middleware-stack/src/MiddlewareStack.ts | 20 ++- .../src/createPresignedPost.ts | 22 ++- .../src/getSignedUrl.spec.ts | 5 +- .../s3-request-presigner/src/getSignedUrl.ts | 23 ++- packages/types/src/auth.ts | 13 +- packages/types/src/endpoint.ts | 6 +- packages/types/src/middleware.ts | 14 ++ packages/types/src/serde.ts | 2 +- .../src/resolveEndpoint.spec.ts | 8 +- .../util-endpoints/src/types/RuleSetObject.ts | 5 +- .../util-middleware/src/normalizeProvider.ts | 3 + 36 files changed, 566 insertions(+), 184 deletions(-) create mode 100644 packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts create mode 100644 packages/middleware-endpoint/src/adaptors/index.ts create mode 100644 packages/middleware-endpoint/src/adaptors/toEndpointV1.ts create mode 100644 packages/middleware-endpoint/src/service-customizations/index.ts create mode 100644 packages/middleware-endpoint/src/service-customizations/s3.ts create mode 100644 packages/middleware-sdk-s3/src/configuration.ts diff --git a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java index 46b9255fa4e0..57928c4a618a 100644 --- a/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java +++ b/codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AddS3Config.java @@ -72,8 +72,8 @@ public final class AddS3Config implements TypeScriptIntegration { ); private static final String CRT_NOTIFICATION = "

Note: To supply the Multi-region Access Point (MRAP) to Bucket," - + " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n" - + "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues

"; + + " you need to install the \"@aws-sdk/signature-v4-crt\" package to your project dependencies. \n" + + "For more information, please go to https://github.com/aws/aws-sdk-js-v3#known-issues

"; @Override public Model preprocessModel(Model model, TypeScriptSettings settings) { @@ -118,17 +118,19 @@ public void addConfigInterfaceFields( return; } writer.writeDocs("Whether to escape request path when signing the request.") - .write("signingEscapePath?: boolean;\n"); + .write("signingEscapePath?: boolean;\n"); writer.writeDocs( "Whether to override the request region with the region inferred from requested resource's ARN." - + " Defaults to false.") - .addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName) - .write("useArnRegion?: boolean | Provider;"); + + " Defaults to false.") + .addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName) + .write("useArnRegion?: boolean | Provider;"); } @Override - public Map> getRuntimeConfigWriters(TypeScriptSettings settings, Model model, - SymbolProvider symbolProvider, LanguageTarget target) { + public Map> getRuntimeConfigWriters( + TypeScriptSettings settings, Model model, + SymbolProvider symbolProvider, LanguageTarget target + ) { if (!isS3(settings.getService(model))) { return Collections.emptyMap(); } @@ -140,18 +142,18 @@ public Map> getRuntimeConfigWriters(TypeScrip writer.write("false"); }, "signerConstructor", writer -> { writer.addDependency(AwsDependency.SIGNATURE_V4_MULTIREGION) - .addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion", - AwsDependency.SIGNATURE_V4_MULTIREGION.packageName) - .write("SignatureV4MultiRegion"); + .addImport("SignatureV4MultiRegion", "SignatureV4MultiRegion", + AwsDependency.SIGNATURE_V4_MULTIREGION.packageName) + .write("SignatureV4MultiRegion"); }); case NODE: return MapUtils.of("useArnRegion", writer -> { writer.addDependency(AwsDependency.NODE_CONFIG_PROVIDER) - .addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName) - .addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE) - .addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS", - AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName) - .write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)"); + .addImport("loadConfig", "loadNodeConfig", AwsDependency.NODE_CONFIG_PROVIDER.packageName) + .addDependency(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE) + .addImport("NODE_USE_ARN_REGION_CONFIG_OPTIONS", "NODE_USE_ARN_REGION_CONFIG_OPTIONS", + AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.packageName) + .write("loadNodeConfig(NODE_USE_ARN_REGION_CONFIG_OPTIONS)"); }); default: return Collections.emptyMap(); @@ -161,67 +163,67 @@ public Map> getRuntimeConfigWriters(TypeScrip @Override public List getClientPlugins() { return ListUtils.of( - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName", - HAS_MIDDLEWARE) - .servicePredicate((m, s) -> isS3(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader", - HAS_MIDDLEWARE) - .operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject")) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions", - HAS_MIDDLEWARE) - .operationPredicate( - (m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s)) - && isS3(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, - "WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE) - .operationPredicate((m, s, o) -> isS3(s) - && o.getId().getName(s).equals("WriteGetObjectResponse")) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue", - HAS_MIDDLEWARE) - .servicePredicate((m, s) -> isS3(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE) - .operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS) - && isS3(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint", - HAS_MIDDLEWARE) - .operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket") - && isS3(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3", - HAS_CONFIG) - .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s)) - .build(), - /* - * BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client. - * The second applies the middleware to bucket endpoint operations. - */ - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint", - HAS_CONFIG) - .servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s)) - .build(), - RuntimeClientPlugin.builder() - .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint", - HAS_MIDDLEWARE) - .operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s)) - && isS3(s) - && !isEndpointsV2Service(s) - && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS)) - .build() + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "ValidateBucketName", + HAS_MIDDLEWARE) + .servicePredicate((m, s) -> isS3(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "CheckContentLengthHeader", + HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> isS3(s) && o.getId().getName(s).equals("PutObject")) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "throw200Exceptions", + HAS_MIDDLEWARE) + .operationPredicate( + (m, s, o) -> EXCEPTIONS_OF_200_OPERATIONS.contains(o.getId().getName(s)) + && isS3(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, + "WriteGetObjectResponseEndpoint", HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> isS3(s) + && o.getId().getName(s).equals("WriteGetObjectResponse")) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.ADD_EXPECT_CONTINUE.dependency, "AddExpectContinue", + HAS_MIDDLEWARE) + .servicePredicate((m, s) -> isS3(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.SSEC_MIDDLEWARE.dependency, "Ssec", HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> containsInputMembers(m, o, SSEC_INPUT_KEYS) + && isS3(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.LOCATION_CONSTRAINT.dependency, "LocationConstraint", + HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> o.getId().getName(s).equals("CreateBucket") + && isS3(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.S3_MIDDLEWARE.dependency, "S3", + HAS_CONFIG) + .servicePredicate((m, s) -> isS3(s) && isEndpointsV2Service(s)) + .build(), + /* + * BUCKET_ENDPOINT_MIDDLEWARE needs two separate plugins. The first resolves the config in the client. + * The second applies the middleware to bucket endpoint operations. + */ + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint", + HAS_CONFIG) + .servicePredicate((m, s) -> isS3(s) && !isEndpointsV2Service(s)) + .build(), + RuntimeClientPlugin.builder() + .withConventions(AwsDependency.BUCKET_ENDPOINT_MIDDLEWARE.dependency, "BucketEndpoint", + HAS_MIDDLEWARE) + .operationPredicate((m, s, o) -> !NON_BUCKET_ENDPOINT_OPERATIONS.contains(o.getId().getName(s)) + && isS3(s) + && !isEndpointsV2Service(s) + && containsInputMembers(m, o, BUCKET_ENDPOINT_INPUT_KEYS)) + .build() ); } @@ -232,8 +234,8 @@ private static boolean containsInputMembers( ) { OperationIndex operationIndex = OperationIndex.of(model); return operationIndex.getInput(operationShape) - .filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains)) - .isPresent(); + .filter(input -> input.getMemberNames().stream().anyMatch(expectedMemberNames::contains)) + .isPresent(); } private static boolean isS3(Shape serviceShape) { diff --git a/lib/lib-storage/package.json b/lib/lib-storage/package.json index 33aa42ec9e28..0947fb9a2bef 100644 --- a/lib/lib-storage/package.json +++ b/lib/lib-storage/package.json @@ -24,6 +24,7 @@ }, "license": "Apache-2.0", "dependencies": { + "@aws-sdk/middleware-endpoint": "*", "@aws-sdk/smithy-client": "*", "buffer": "5.6.0", "events": "3.3.0", @@ -37,6 +38,7 @@ "devDependencies": { "@aws-sdk/abort-controller": "*", "@aws-sdk/client-s3": "*", + "@aws-sdk/types": "*", "@tsconfig/recommended": "1.0.1", "@types/node": "^14.11.2", "concurrently": "7.0.0", diff --git a/lib/lib-storage/src/Upload.ts b/lib/lib-storage/src/Upload.ts index 05f9c5f17b45..13062010979f 100644 --- a/lib/lib-storage/src/Upload.ts +++ b/lib/lib-storage/src/Upload.ts @@ -13,8 +13,14 @@ import { Tag, UploadPartCommand, } from "@aws-sdk/client-s3"; +import { + EndpointParameterInstructionsSupplier, + getEndpointFromInstructions, + toEndpointV1, +} from "@aws-sdk/middleware-endpoint"; import { HttpRequest } from "@aws-sdk/protocol-http"; import { extendedEncodeURIComponent } from "@aws-sdk/smithy-client"; +import { Endpoint } from "@aws-sdk/types"; import { EventEmitter } from "events"; import { byteLength } from "./bytelength"; @@ -101,7 +107,8 @@ export class Upload extends EventEmitter { this.isMultiPart = false; const params = { ...this.params, Body: dataPart.data }; - const requestHandler = this.client.config.requestHandler; + const clientConfig = this.client.config; + const requestHandler = clientConfig.requestHandler; const eventEmitter: EventEmitter | null = requestHandler instanceof EventEmitter ? requestHandler : null; const uploadEventListener = (event: ProgressEvent) => { this.bytesUploadedSoFar = event.loaded; @@ -120,14 +127,20 @@ export class Upload extends EventEmitter { eventEmitter.on("xhr.upload.progress", uploadEventListener); } - const [putResult, endpoint] = await Promise.all([ - this.client.send(new PutObjectCommand(params)), - this.client.config?.endpoint?.(), - ]); + const resolved = await Promise.all([this.client.send(new PutObjectCommand(params)), clientConfig?.endpoint?.()]); + const putResult = resolved[0]; + let endpoint: Endpoint = resolved[1]; + + if (!endpoint) { + endpoint = toEndpointV1( + await getEndpointFromInstructions(params, PutObjectCommand as EndpointParameterInstructionsSupplier, { + ...clientConfig, + }) + ); + } if (!endpoint) { - // TODO(endpointsv2): handle endpoint v2 - throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()".'); + throw new Error('Could not resolve endpoint from S3 "client.config.endpoint()" nor EndpointsV2.'); } if (eventEmitter !== null) { diff --git a/packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts b/packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts index d8c2dc65f23b..5a8f2cba2c86 100644 --- a/packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts +++ b/packages/config-resolver/src/endpointsConfig/resolveEndpointsConfig.ts @@ -5,7 +5,8 @@ import { getEndpointFromRegion } from "./utils/getEndpointFromRegion"; export interface EndpointsInputConfig { /** - * The fully qualified endpoint of the webservice. This is only required when using a custom endpoint (for example, when using a local version of S3). + * The fully qualified endpoint of the webservice. This is only required when using + * a custom endpoint (for example, when using a local version of S3). */ endpoint?: string | Endpoint | Provider; @@ -56,7 +57,7 @@ export const resolveEndpointsConfig = ( endpoint: endpoint ? normalizeProvider(typeof endpoint === "string" ? urlParser(endpoint) : endpoint) : () => getEndpointFromRegion({ ...input, useDualstackEndpoint, useFipsEndpoint }), - isCustomEndpoint: endpoint ? true : false, + isCustomEndpoint: !!endpoint, useDualstackEndpoint, }; }; diff --git a/packages/middleware-endpoint/package.json b/packages/middleware-endpoint/package.json index f4d0bbfe33aa..99728e8a8eb4 100644 --- a/packages/middleware-endpoint/package.json +++ b/packages/middleware-endpoint/package.json @@ -5,6 +5,7 @@ "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'", "build:cjs": "tsc -p tsconfig.cjs.json", "build:es": "tsc -p tsconfig.es.json", + "build:include:deps": "lerna run --scope $npm_package_name --include-dependencies build", "build:types": "tsc -p tsconfig.types.json", "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4", "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", @@ -19,6 +20,7 @@ }, "license": "Apache-2.0", "dependencies": { + "@aws-sdk/protocol-http": "*", "@aws-sdk/signature-v4": "*", "@aws-sdk/types": "*", "@aws-sdk/util-config-provider": "*", diff --git a/packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts b/packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts new file mode 100644 index 000000000000..95557686beb9 --- /dev/null +++ b/packages/middleware-endpoint/src/adaptors/getEndpointFromInstructions.ts @@ -0,0 +1,77 @@ +import { EndpointParameters, EndpointV2, HandlerExecutionContext } from "@aws-sdk/types"; + +import { EndpointResolvedConfig } from "../resolveEndpointConfig"; +import { EndpointParameterInstructions } from "../types"; + +export type EndpointParameterInstructionsSupplier = Partial<{ + getEndpointParameterInstructions(): EndpointParameterInstructions; +}>; + +/** + * This step in the endpoint resolution process is exposed as a function + * to allow packages such as signers, lib-upload, etc. to get + * the V2 Endpoint associated to an instance of some api operation command + * without needing to send it or resolve its middleware stack. + * + * @private + * @param commandInput - the input of the Command in question. + * @param instructionsSupplier - this is typically a Command constructor. A static function supplying the + * endpoint parameter instructions will exist for commands in services + * having an endpoints ruleset trait. + * @param clientConfig - config of the service client. + * @param context - optional context. + */ +export const getEndpointFromInstructions = async < + T extends EndpointParameters, + CommandInput extends Record, + Config extends Record +>( + commandInput: CommandInput, + instructionsSupplier: EndpointParameterInstructionsSupplier, + clientConfig: Partial> & Config, + context?: HandlerExecutionContext +): Promise => { + const endpointParams: EndpointParameters = {}; + const instructions: EndpointParameterInstructions = + (instructionsSupplier.getEndpointParameterInstructions || (() => null))() || {}; + + if (typeof clientConfig.endpointProvider !== "function") { + throw new Error("config.endpointProvider is not set."); + } + + for (const [name, instruction] of Object.entries(instructions)) { + switch (instruction.type) { + case "staticContextParams": + endpointParams[name] = instruction.value; + break; + case "contextParams": + endpointParams[name] = commandInput[instruction.name] as string | boolean; + break; + case "clientContextParams": + case "builtInParams": + endpointParams[name] = await createConfigProvider(instruction.name, clientConfig)(); + break; + default: + throw new Error("Unrecognized endpoint parameter instruction: " + JSON.stringify(instruction)); + } + } + + const endpoint: EndpointV2 = clientConfig.endpointProvider!(endpointParams as T, context); + + return endpoint; +}; + +/** + * Normalize some key of the client config to an async provider. + * @private + */ +const createConfigProvider = >(configKey: string, config: Config) => { + const configProvider = async () => { + const configValue: unknown = config[configKey]; + if (typeof configValue === "function") { + return configValue(); + } + return configValue; + }; + return configProvider; +}; diff --git a/packages/middleware-endpoint/src/adaptors/index.ts b/packages/middleware-endpoint/src/adaptors/index.ts new file mode 100644 index 000000000000..17752da20113 --- /dev/null +++ b/packages/middleware-endpoint/src/adaptors/index.ts @@ -0,0 +1,2 @@ +export * from "./getEndpointFromInstructions"; +export * from "./toEndpointV1"; diff --git a/packages/middleware-endpoint/src/adaptors/toEndpointV1.ts b/packages/middleware-endpoint/src/adaptors/toEndpointV1.ts new file mode 100644 index 000000000000..8aaa8cb37863 --- /dev/null +++ b/packages/middleware-endpoint/src/adaptors/toEndpointV1.ts @@ -0,0 +1,14 @@ +import { Endpoint, EndpointV2 } from "@aws-sdk/types"; +import { parseUrl } from "@aws-sdk/url-parser"; + +export const toEndpointV1 = (endpoint: string | Endpoint | EndpointV2): Endpoint => { + if (typeof endpoint === "object") { + if ("url" in endpoint) { + // v2 + return parseUrl(endpoint.url); + } + // v1 + return endpoint; + } + return parseUrl(endpoint); +}; diff --git a/packages/middleware-endpoint/src/endpointMiddleware.ts b/packages/middleware-endpoint/src/endpointMiddleware.ts index 4c6bd6f44c32..ee376ef63539 100644 --- a/packages/middleware-endpoint/src/endpointMiddleware.ts +++ b/packages/middleware-endpoint/src/endpointMiddleware.ts @@ -1,4 +1,8 @@ import { + AuthScheme, + EndpointParameters, + EndpointV2, + HandlerExecutionContext, MetadataBearer, SerializeHandler, SerializeHandlerArguments, @@ -6,19 +10,56 @@ import { SerializeMiddleware, } from "@aws-sdk/types"; -import { EndpointParameterInstruction } from "./types"; +import { getEndpointFromInstructions } from "./adaptors/getEndpointFromInstructions"; +import { EndpointResolvedConfig } from "./resolveEndpointConfig"; +import { s3Customizations } from "./service-customizations"; +import { EndpointParameterInstructions } from "./types"; -export const endpointMiddleware = (options: { - config: any; // TODO(endpointsV2): should be ResolvedEndpointConfig interface - instruction: EndpointParameterInstruction; +export type PreviouslyResolvedServiceId = { + serviceId?: string; +}; + +/** + * @private + */ +export const endpointMiddleware = ({ + config, + instructions, +}: { + config: EndpointResolvedConfig & PreviouslyResolvedServiceId; + instructions: EndpointParameterInstructions; }): SerializeMiddleware => { - return (next: SerializeHandler): SerializeHandler => + return ( + next: SerializeHandler, + context: HandlerExecutionContext + ): SerializeHandler => async (args: SerializeHandlerArguments): Promise> => { - // TODO(endpointsV2): resolve async providers from client config and static values according to - // instruction from input to populate endpoint parameters + const endpoint: EndpointV2 = await getEndpointFromInstructions( + args.input, + { + getEndpointParameterInstructions() { + return instructions; + }, + }, + { ...config }, + context + ); + + context.endpointV2 = endpoint; + context.authSchemes = endpoint.properties?.authSchemes; + + const authScheme: AuthScheme = context.authSchemes?.[0]; + if (authScheme) { + context["signing_region"] = authScheme.signingScope; + context["signing_service"] = authScheme.signingName; + } + + if (config.serviceId === "S3") { + await s3Customizations(config, instructions, args, context); + } + return next({ ...args, }); }; }; - diff --git a/packages/middleware-endpoint/src/getEndpointPlugin.ts b/packages/middleware-endpoint/src/getEndpointPlugin.ts index f06d9c7a76de..6a8200169f7e 100644 --- a/packages/middleware-endpoint/src/getEndpointPlugin.ts +++ b/packages/middleware-endpoint/src/getEndpointPlugin.ts @@ -1,24 +1,28 @@ -import { Pluggable, SerializeHandlerOptions } from "@aws-sdk/types"; +import { serializerMiddlewareOption } from "@aws-sdk/middleware-serde"; +import { EndpointParameters, Pluggable, RelativeMiddlewareOptions, SerializeHandlerOptions } from "@aws-sdk/types"; -import { endpointMiddleware } from "./endpointMiddleware"; -import { EndpointParameterInstruction } from "./types"; +import { endpointMiddleware, PreviouslyResolvedServiceId } from "./endpointMiddleware"; +import { EndpointResolvedConfig } from "./resolveEndpointConfig"; +import { EndpointParameterInstructions } from "./types"; -export const endpointMiddlewareOptions: SerializeHandlerOptions = { +export const endpointMiddlewareOptions: SerializeHandlerOptions & RelativeMiddlewareOptions = { step: "serialize", tags: ["ENDPOINT_PARAMETERS", "ENDPOINT_V2", "ENDPOINT"], - name: "endpointMiddleware", + name: "endpointV2Middleware", override: true, + relation: "before", + toMiddleware: serializerMiddlewareOption.name!, }; -export const getEndpointPlugin = ( - config: any, //TODO(endpointsV2): should be ResolvedEndpointConfig interface - instruction: EndpointParameterInstruction +export const getEndpointPlugin = ( + config: EndpointResolvedConfig & PreviouslyResolvedServiceId, + instructions: EndpointParameterInstructions ): Pluggable => ({ applyToStack: (clientStack) => { - clientStack.add( - endpointMiddleware({ + clientStack.addRelativeTo( + endpointMiddleware({ config, - instruction, + instructions, }), endpointMiddlewareOptions ); diff --git a/packages/middleware-endpoint/src/index.ts b/packages/middleware-endpoint/src/index.ts index 4a51073f1bd1..f89653edeb1e 100644 --- a/packages/middleware-endpoint/src/index.ts +++ b/packages/middleware-endpoint/src/index.ts @@ -1,3 +1,4 @@ +export * from "./adaptors"; export * from "./endpointMiddleware"; export * from "./getEndpointPlugin"; export * from "./resolveEndpointConfig"; diff --git a/packages/middleware-endpoint/src/resolveEndpointConfig.ts b/packages/middleware-endpoint/src/resolveEndpointConfig.ts index 848a64e2a516..4c9bf0070624 100644 --- a/packages/middleware-endpoint/src/resolveEndpointConfig.ts +++ b/packages/middleware-endpoint/src/resolveEndpointConfig.ts @@ -1,14 +1,17 @@ import { Endpoint, EndpointParameters, EndpointV2, Logger, Provider, UrlParser } from "@aws-sdk/types"; import { normalizeProvider } from "@aws-sdk/util-middleware"; +import { toEndpointV1 } from "./adaptors/toEndpointV1"; + /** * Endpoint config interfaces and resolver for Endpoint v2. They live in separate package to allow per-service onboarding. - * When all services onboard the endpoint v2, the resolver in config-resolver package can be removed. + * When all services onboard Endpoint v2, the resolver in config-resolver package can be removed. * This interface includes all the endpoint parameters with built-in bindings of "AWS::*" and "SDK::*" */ export interface EndpointInputConfig { /** - * The fully qualified endpoint of the webservice. This is only required when using a custom endpoint (for example, when using a local version of S3). + * The fully qualified endpoint of the webservice. This is only required when using + * a custom endpoint (for example, when using a local version of S3). */ endpoint?: string | Endpoint | Provider | EndpointV2 | Provider; @@ -38,12 +41,16 @@ interface PreviouslyResolved logger?: Logger; } +/** + * This supercedes the similarly named EndpointsResolvedConfig (no parametric types) + * from resolveEndpointsConfig.ts in @aws-sdk/config-resolver. + */ export interface EndpointResolvedConfig { /** * Resolved value for input {@link EndpointsInputConfig.endpoint} * @deprecated Use {@link EndpointResolvedConfig.endpointProvider} instead */ - endpoint?: Provider; + endpoint: Provider; endpointProvider: (params: T, context?: { logger?: Logger }) => EndpointV2; @@ -76,24 +83,18 @@ export const resolveEndpointConfig = => { const tls = input.tls ?? true; const { endpoint } = input; - const resolvedEndpoint = - endpoint != null - ? typeof endpoint === "function" - ? async () => { - return toEndpointV1(await endpoint()); - } - : normalizeProvider(toEndpointV1(endpoint)) - : undefined; - const isCustomEndpoint = input.endpoint ? true : false; + + const customEndpointProvider = + endpoint != null ? async () => toEndpointV1(await normalizeProvider(endpoint)()) : undefined; + + const isCustomEndpoint = !!endpoint; + return { ...input, - endpoint: resolvedEndpoint, + endpoint: customEndpointProvider, tls, isCustomEndpoint, useDualstackEndpoint: normalizeProvider(input.useDualstackEndpoint ?? false), useFipsEndpoint: normalizeProvider(input.useFipsEndpoint ?? false), - // endpointProvider, - }; + } as T & EndpointResolvedConfig

; }; - -declare const toEndpointV1: (endpoint: string | Endpoint | EndpointV2) => Endpoint; // TODO(endpointsV2) implementation diff --git a/packages/middleware-endpoint/src/service-customizations/index.ts b/packages/middleware-endpoint/src/service-customizations/index.ts new file mode 100644 index 000000000000..e8919ca9b14d --- /dev/null +++ b/packages/middleware-endpoint/src/service-customizations/index.ts @@ -0,0 +1 @@ +export * from './s3' \ No newline at end of file diff --git a/packages/middleware-endpoint/src/service-customizations/s3.ts b/packages/middleware-endpoint/src/service-customizations/s3.ts new file mode 100644 index 000000000000..7fcd0040e60a --- /dev/null +++ b/packages/middleware-endpoint/src/service-customizations/s3.ts @@ -0,0 +1,50 @@ +import { EndpointParameters, HandlerExecutionContext, SerializeHandlerArguments } from "@aws-sdk/types"; + +import { getEndpointFromInstructions } from "../adaptors"; +import { PreviouslyResolvedServiceId } from "../endpointMiddleware"; +import { EndpointResolvedConfig } from "../resolveEndpointConfig"; +import { EndpointParameterInstructions } from "../types"; + +/** + * Bucket name DNS compatibility customization. + */ +export const s3Customizations = async ( + config: EndpointResolvedConfig & PreviouslyResolvedServiceId, + instructions: EndpointParameterInstructions, + args: SerializeHandlerArguments, + context: HandlerExecutionContext +) => { + const endpoint = context.endpointV2; + const bucket: string | undefined = args.input?.Bucket || void 0; + + if (!endpoint || !bucket) { + return; + } + + if (!isDnsCompatibleBucketName(bucket) || bucket.indexOf(".") !== -1) { + context.endpointV2 = await getEndpointFromInstructions( + args.input, + { + getEndpointParameterInstructions: () => instructions, + }, + { ...config, forcePathStyle: true } + ); + } +}; + +const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$/; +const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; +const DOTS_PATTERN = /\.\./; +export const DOT_PATTERN = /\./; +export const S3_HOSTNAME_PATTERN = /^(.+\.)?s3(-fips)?(\.dualstack)?[.-]([a-z0-9-]+)\./; + +/** + * Determines whether a given string is DNS compliant per the rules outlined by + * S3. Length, capitaization, and leading dot restrictions are enforced by the + * DOMAIN_PATTERN regular expression. + * @internal + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html + */ +export const isDnsCompatibleBucketName = (bucketName: string): boolean => + DOMAIN_PATTERN.test(bucketName) && !IP_ADDRESS_PATTERN.test(bucketName) && !DOTS_PATTERN.test(bucketName); diff --git a/packages/middleware-endpoint/src/types.ts b/packages/middleware-endpoint/src/types.ts index e106aec5eae5..3c2447ab549b 100644 --- a/packages/middleware-endpoint/src/types.ts +++ b/packages/middleware-endpoint/src/types.ts @@ -1,4 +1,4 @@ -export interface EndpointParameterInstruction { +export interface EndpointParameterInstructions { [name: string]: | BuiltInParamInstruction | ClientContextParamInstruction diff --git a/packages/middleware-sdk-s3/src/configuration.ts b/packages/middleware-sdk-s3/src/configuration.ts new file mode 100644 index 000000000000..c0ca3b38dcce --- /dev/null +++ b/packages/middleware-sdk-s3/src/configuration.ts @@ -0,0 +1,31 @@ +/** + * All endpoint parameters with built-in bindings of AWS::S3::* + */ +export interface S3InputConfig { + /** + * Whether to force path style URLs for S3 objects + * (e.g., https://s3.amazonaws.com// instead of https://.s3.amazonaws.com/ + */ + forcePathStyle?: boolean; + /** + * Whether to use the S3 Transfer Acceleration endpoint by default + */ + useAccelerateEndpoint?: boolean; + /** + * Whether multi-region access points (MRAP) should be disabled. + */ + disableMultiregionAccessPoints?: boolean; +} + +export interface S3ResolvedConfig { + forcePathStyle: boolean; + useAccelerateEndpoint: boolean; + disableMultiregionAccessPoints: boolean; +} + +export const resolveS3Config = (input: T & S3InputConfig): T & S3ResolvedConfig => ({ + ...input, + forcePathStyle: input.forcePathStyle ?? false, + useAccelerateEndpoint: input.useAccelerateEndpoint ?? false, + disableMultiregionAccessPoints: input.disableMultiregionAccessPoints ?? false, +}); diff --git a/packages/middleware-sdk-s3/src/index.ts b/packages/middleware-sdk-s3/src/index.ts index 6053179bc317..95e25976746b 100644 --- a/packages/middleware-sdk-s3/src/index.ts +++ b/packages/middleware-sdk-s3/src/index.ts @@ -1,4 +1,5 @@ export * from "./check-content-length-header"; +export * from "./configuration"; export * from "./throw-200-exceptions"; export * from "./validate-bucket-name"; export * from "./write-get-object-response-endpoint"; diff --git a/packages/middleware-sdk-s3/src/write-get-object-response-endpoint.ts b/packages/middleware-sdk-s3/src/write-get-object-response-endpoint.ts index b7a7eb3de558..800fe62d197d 100644 --- a/packages/middleware-sdk-s3/src/write-get-object-response-endpoint.ts +++ b/packages/middleware-sdk-s3/src/write-get-object-response-endpoint.ts @@ -14,7 +14,7 @@ import { type PreviouslyResolved = { region: Provider; - isCustomEndpoint: boolean; + isCustomEndpoint?: boolean; disableHostPrefix: boolean; runtime: string; }; diff --git a/packages/middleware-sdk-transcribe-streaming/src/configuration.ts b/packages/middleware-sdk-transcribe-streaming/src/configuration.ts index e1944cff450e..7344a3410eae 100644 --- a/packages/middleware-sdk-transcribe-streaming/src/configuration.ts +++ b/packages/middleware-sdk-transcribe-streaming/src/configuration.ts @@ -1,12 +1,12 @@ import { SignatureV4 as BaseSignatureV4 } from "@aws-sdk/signature-v4"; -import { Provider, RequestHandler, RequestSigner } from "@aws-sdk/types"; +import { AuthScheme, Provider, RequestHandler, RequestSigner } from "@aws-sdk/types"; import { SignatureV4 } from "./signer"; export interface WebSocketInputConfig {} interface PreviouslyResolved { - signer: Provider; + signer: (authScheme: AuthScheme) => Promise; requestHandler: RequestHandler; } @@ -14,7 +14,7 @@ export interface WebSocketResolvedConfig { /** * Resolved value of input config {@link AwsAuthInputConfig.signer} */ - signer: Provider; + signer: (authScheme: AuthScheme) => Promise; /** * The HTTP handler to use. Fetch in browser and Https in Nodejs. */ @@ -28,8 +28,8 @@ export const resolveWebSocketConfig = ( ? input : { ...input, - signer: async () => { - const signerObj = await input.signer(); + signer: async (authScheme: AuthScheme) => { + const signerObj = await input.signer(authScheme); if (validateSigner(signerObj)) { return new SignatureV4({ signer: signerObj }); } diff --git a/packages/middleware-serde/src/serdePlugin.ts b/packages/middleware-serde/src/serdePlugin.ts index 722b5efe22d3..8a7f111d3310 100644 --- a/packages/middleware-serde/src/serdePlugin.ts +++ b/packages/middleware-serde/src/serdePlugin.ts @@ -1,5 +1,6 @@ import { DeserializeHandlerOptions, + Endpoint, EndpointBearer, MetadataBearer, MiddlewareStack, @@ -7,6 +8,7 @@ import { RequestSerializer, ResponseDeserializer, SerializeHandlerOptions, + UrlParser, } from "@aws-sdk/types"; import { deserializerMiddleware } from "./deserializerMiddleware"; @@ -26,12 +28,18 @@ export const serializerMiddlewareOption: SerializeHandlerOptions = { override: true, }; +// Type the modifies the EndpointBearer to make it compatible with Endpoints 2.0 change. +// Must be removed after all clients has been onboard the Endpoints 2.0 +export type V1OrV2Endpoint = T & { + urlParser?: UrlParser; +}; + export function getSerdePlugin< InputType extends object, SerDeContext extends EndpointBearer, OutputType extends MetadataBearer >( - config: SerDeContext, + config: V1OrV2Endpoint, serializer: RequestSerializer, deserializer: ResponseDeserializer ): Pluggable { diff --git a/packages/middleware-serde/src/serializerMiddleware.ts b/packages/middleware-serde/src/serializerMiddleware.ts index 11c9e933c2a8..9d3fd9367bfd 100644 --- a/packages/middleware-serde/src/serializerMiddleware.ts +++ b/packages/middleware-serde/src/serializerMiddleware.ts @@ -8,14 +8,26 @@ import { SerializeMiddleware, } from "@aws-sdk/types"; +import type { V1OrV2Endpoint } from "./serdePlugin"; + export const serializerMiddleware = ( - options: RuntimeUtils, + options: V1OrV2Endpoint, serializer: RequestSerializer ): SerializeMiddleware => (next: SerializeHandler, context: HandlerExecutionContext): SerializeHandler => async (args: SerializeHandlerArguments): Promise> => { - const request = await serializer(args.input, options); + const endpoint = + context.endpointV2?.url && options.urlParser + ? async () => options.urlParser!(context.endpointV2!.url as URL) + : options.endpoint; + + if (!endpoint) { + throw new Error("No valid endpoint provider available."); + } + + const request = await serializer(args.input, { ...options, endpoint }); + return next({ ...args, request, diff --git a/packages/middleware-signing/package.json b/packages/middleware-signing/package.json index f70a706428e3..2a932256667c 100644 --- a/packages/middleware-signing/package.json +++ b/packages/middleware-signing/package.json @@ -9,7 +9,7 @@ "build:types": "tsc -p tsconfig.types.json", "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4", "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", - "test": "jest" + "test": "jest --passWithNoTests" }, "main": "./dist-cjs/index.js", "module": "./dist-es/index.js", diff --git a/packages/middleware-signing/src/configuration.spec.ts b/packages/middleware-signing/src/configuration.spec.ts index 6734ac6dd6b4..7af531b2b420 100644 --- a/packages/middleware-signing/src/configuration.spec.ts +++ b/packages/middleware-signing/src/configuration.spec.ts @@ -3,6 +3,13 @@ import { HttpRequest } from "@aws-sdk/protocol-http"; import { resolveAwsAuthConfig, resolveSigV4AuthConfig } from "./configurations"; describe("AuthConfig", () => { + const authScheme = { + name: "sigv4", + signingScope: "UNIT_TEST_REGION", + signingName: "UNIT_TEST_SERVICE_NAME", + properties: {}, + }; + describe("resolveAwsAuthConfig", () => { const inputParams = { credentialDefaultProvider: () => () => Promise.resolve({ accessKeyId: "key", secretAccessKey: "secret" }), @@ -24,7 +31,7 @@ describe("AuthConfig", () => { it("should memoize custom credential provider", async () => { const { signer: signerProvider } = resolveAwsAuthConfig(inputParams); - const signer = await signerProvider(); + const signer = await signerProvider(authScheme); const request = new HttpRequest({}); const repeats = 10; for (let i = 0; i < repeats; i++) { @@ -47,7 +54,7 @@ describe("AuthConfig", () => { .mockResolvedValue({ accessKeyId: "key", secretAccessKey: "secret" }), }; const { signer: signerProvider } = resolveAwsAuthConfig(input); - const signer = await signerProvider(); + const signer = await signerProvider(authScheme); const request = new HttpRequest({}); const repeats = 10; for (let i = 0; i < repeats; i++) { @@ -75,7 +82,7 @@ describe("AuthConfig", () => { it("should memoize custom credential provider", async () => { const { signer: signerProvider } = resolveSigV4AuthConfig(inputParams); - const signer = await signerProvider(); + const signer = await signerProvider(authScheme); const request = new HttpRequest({}); const repeats = 10; for (let i = 0; i < repeats; i++) { @@ -98,7 +105,7 @@ describe("AuthConfig", () => { .mockResolvedValue({ accessKeyId: "key", secretAccessKey: "secret" }), }; const { signer: signerProvider } = resolveSigV4AuthConfig(input); - const signer = await signerProvider(); + const signer = await signerProvider(authScheme); const request = new HttpRequest({}); const repeats = 10; for (let i = 0; i < repeats; i++) { diff --git a/packages/middleware-signing/src/configurations.ts b/packages/middleware-signing/src/configurations.ts index 5f376cc79a6c..77cf74bb39fd 100644 --- a/packages/middleware-signing/src/configurations.ts +++ b/packages/middleware-signing/src/configurations.ts @@ -1,6 +1,7 @@ import { memoize } from "@aws-sdk/property-provider"; import { SignatureV4, SignatureV4CryptoInit, SignatureV4Init } from "@aws-sdk/signature-v4"; import { + AuthScheme, Credentials, HashConstructor, Logger, @@ -10,6 +11,7 @@ import { RegionInfoProvider, RequestSigner, } from "@aws-sdk/types"; +import { normalizeProvider } from "@aws-sdk/util-middleware"; // 5 minutes buffer time the refresh the credential before it really expires const CREDENTIAL_EXPIRE_WINDOW = 300000; @@ -27,7 +29,7 @@ export interface AwsAuthInputConfig { /** * The signer to use when signing requests. */ - signer?: RequestSigner | Provider; + signer?: RequestSigner | ((authScheme?: AuthScheme) => Promise); /** * Whether to escape request path when signing the request. @@ -62,7 +64,7 @@ export interface SigV4AuthInputConfig { /** * The signer to use when signing requests. */ - signer?: RequestSigner | Provider; + signer?: RequestSigner | ((authScheme?: AuthScheme) => Promise); /** * Whether to escape request path when signing the request. @@ -78,7 +80,7 @@ export interface SigV4AuthInputConfig { interface PreviouslyResolved { credentialDefaultProvider: (input: any) => MemoizedProvider; region: string | Provider; - regionInfoProvider: RegionInfoProvider; + regionInfoProvider?: RegionInfoProvider; signingName?: string; serviceId: string; sha256: HashConstructor; @@ -104,7 +106,7 @@ export interface AwsAuthResolvedConfig { /** * Resolved value for input config {@link AwsAuthInputConfig.signer} */ - signer: Provider; + signer: (authScheme?: AuthScheme) => Promise; /** * Resolved value for input config {@link AwsAuthInputConfig.signingEscapePath} */ @@ -124,18 +126,19 @@ export const resolveAwsAuthConfig = ( ? normalizeCredentialProvider(input.credentials) : input.credentialDefaultProvider(input as any); const { signingEscapePath = true, systemClockOffset = input.systemClockOffset || 0, sha256 } = input; - let signer: Provider; + let signer: (authScheme?: AuthScheme) => Promise; if (input.signer) { - //if signer is supplied by user, normalize it to a function returning a promise for signer. + // if signer is supplied by user, normalize it to a function returning a promise for signer. signer = normalizeProvider(input.signer); - } else { - //construct a provider inferring signing from region. + } else if (input.regionInfoProvider) { + // This branch is for endpoints V1. + // construct a provider inferring signing from region. signer = () => normalizeProvider(input.region)() .then( async (region) => [ - (await input.regionInfoProvider(region, { + (await input.regionInfoProvider!(region, { useFipsEndpoint: await input.useFipsEndpoint(), useDualstackEndpoint: await input.useDualstackEndpoint(), })) || {}, @@ -144,11 +147,11 @@ export const resolveAwsAuthConfig = ( ) .then(([regionInfo, region]) => { const { signingRegion, signingService } = regionInfo; - //update client's singing region and signing service config if they are resolved. - //signing region resolving order: user supplied signingRegion -> endpoints.json inferred region -> client region + // update client's singing region and signing service config if they are resolved. + // signing region resolving order: user supplied signingRegion -> endpoints.json inferred region -> client region input.signingRegion = input.signingRegion || signingRegion || region; - //signing name resolving order: - //user supplied signingName -> endpoints.json inferred (credential scope -> model arnNamespace) -> model service id + // signing name resolving order: + // user supplied signingName -> endpoints.json inferred (credential scope -> model arnNamespace) -> model service id input.signingName = input.signingName || signingService || input.serviceId; const params: SignatureV4Init & SignatureV4CryptoInit = { @@ -159,9 +162,37 @@ export const resolveAwsAuthConfig = ( sha256, uriEscapePath: signingEscapePath, }; - const signerConstructor = input.signerConstructor || SignatureV4; - return new signerConstructor(params); + const SignerCtor = input.signerConstructor || SignatureV4; + return new SignerCtor(params); }); + } else { + // This branch is for endpoints V2. + // Handle endpoints v2 that resolved per-command + // TODO: need total refactor for reference auth architecture. + signer = async (authScheme?: AuthScheme) => { + if (!authScheme) { + throw new Error("Unexpected empty auth scheme config"); + } + const signingRegion = authScheme.signingScope; + const signingService = authScheme.signingName; + // update client's singing region and signing service config if they are resolved. + // signing region resolving order: user supplied signingRegion -> endpoints.json inferred region -> client region + input.signingRegion = input.signingRegion || signingRegion; + // signing name resolving order: + // user supplied signingName -> endpoints.json inferred (credential scope -> model arnNamespace) -> model service id + input.signingName = input.signingName || signingService || input.serviceId; + + const params: SignatureV4Init & SignatureV4CryptoInit = { + ...input, + credentials: normalizedCreds, + region: input.signingRegion, + service: input.signingName, + sha256, + uriEscapePath: signingEscapePath, + }; + const SignerCtor = input.signerConstructor || SignatureV4; + return new SignerCtor(params); + }; } return { @@ -183,7 +214,7 @@ export const resolveSigV4AuthConfig = ( const { signingEscapePath = true, systemClockOffset = input.systemClockOffset || 0, sha256 } = input; let signer: Provider; if (input.signer) { - //if signer is supplied by user, normalize it to a function returning a promise for signer. + // if signer is supplied by user, normalize it to a function returning a promise for signer. signer = normalizeProvider(input.signer); } else { signer = normalizeProvider( @@ -196,7 +227,6 @@ export const resolveSigV4AuthConfig = ( }) ); } - return { ...input, systemClockOffset, @@ -206,14 +236,6 @@ export const resolveSigV4AuthConfig = ( }; }; -const normalizeProvider = (input: T | Provider): Provider => { - if (typeof input === "object") { - const promisified = Promise.resolve(input); - return () => promisified; - } - return input as Provider; -}; - const normalizeCredentialProvider = ( credentials: Credentials | Provider ): MemoizedProvider => { diff --git a/packages/middleware-signing/src/middleware.ts b/packages/middleware-signing/src/middleware.ts index 8f007b19215a..06f8189066a3 100644 --- a/packages/middleware-signing/src/middleware.ts +++ b/packages/middleware-signing/src/middleware.ts @@ -1,5 +1,7 @@ import { HttpRequest, HttpResponse } from "@aws-sdk/protocol-http"; import { + AuthScheme, + EndpointV2, FinalizeHandler, FinalizeHandlerArguments, FinalizeHandlerOutput, @@ -20,7 +22,12 @@ export const awsAuthMiddleware = (next: FinalizeHandler, context: HandlerExecutionContext): FinalizeHandler => async function (args: FinalizeHandlerArguments): Promise> { if (!HttpRequest.isInstance(args.request)) return next(args); - const signer = await options.signer(); + + // TODO(identityandauth): call authScheme resolver + const authScheme: AuthScheme | undefined = (context.endpointV2)?.properties?.authSchemes?.[0]; + + const signer = await options.signer(authScheme); + const output = await next({ ...args, request: await signer.sign(args.request, { diff --git a/packages/middleware-stack/src/MiddlewareStack.ts b/packages/middleware-stack/src/MiddlewareStack.ts index 71c982bf1362..2be61a15d056 100644 --- a/packages/middleware-stack/src/MiddlewareStack.ts +++ b/packages/middleware-stack/src/MiddlewareStack.ts @@ -94,8 +94,9 @@ export const constructStack = (): M /** * Get a final list of middleware in the order of being executed in the resolved handler. + * @param debug - don't throw, getting info only. */ - const getMiddlewareList = (): Array> => { + const getMiddlewareList = (debug = false): Array> => { const normalizedAbsoluteEntries: Normalized, Input, Output>[] = []; const normalizedRelativeEntries: Normalized, Input, Output>[] = []; const normalizedEntriesNameMap: Record, Input, Output>> = {}; @@ -124,6 +125,9 @@ export const constructStack = (): M if (entry.toMiddleware) { const toMiddleware = normalizedEntriesNameMap[entry.toMiddleware]; if (toMiddleware === undefined) { + if (debug) { + return; + } throw new Error( `${entry.toMiddleware} is not found when adding ${entry.name || "anonymous"} middleware ${entry.relation} ${ entry.toMiddleware @@ -146,10 +150,10 @@ export const constructStack = (): M wholeList.push(...expendedMiddlewareList); return wholeList; }, [] as MiddlewareEntry[]); - return mainChain.map((entry) => entry.middleware); + return mainChain; }; - const stack = { + const stack: MiddlewareStack = { add: (middleware: MiddlewareType, options: HandlerOptions & AbsoluteLocation = {}) => { const { name, override } = options; const entry: AbsoluteMiddlewareEntry = { @@ -237,11 +241,19 @@ export const constructStack = (): M applyToStack: cloneTo, + identify: (): string[] => { + return getMiddlewareList(true).map((mw: MiddlewareEntry) => { + return mw.name + ": " + (mw.tags || []).join(","); + }); + }, + resolve: ( handler: DeserializeHandler, context: HandlerExecutionContext ): Handler => { - for (const middleware of getMiddlewareList().reverse()) { + for (const middleware of getMiddlewareList() + .map((entry) => entry.middleware) + .reverse()) { handler = middleware(handler as Handler, context) as any; } return handler as Handler; diff --git a/packages/s3-presigned-post/src/createPresignedPost.ts b/packages/s3-presigned-post/src/createPresignedPost.ts index 3edeb123294b..e2d1223e60d6 100644 --- a/packages/s3-presigned-post/src/createPresignedPost.ts +++ b/packages/s3-presigned-post/src/createPresignedPost.ts @@ -1,4 +1,9 @@ -import { S3Client } from "@aws-sdk/client-s3"; +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { + EndpointParameterInstructionsSupplier, + getEndpointFromInstructions, + toEndpointV1, +} from "@aws-sdk/middleware-endpoint"; import { createScope, getSigningKey } from "@aws-sdk/signature-v4"; import { HashConstructor, SourceData } from "@aws-sdk/types"; import { formatUrl } from "@aws-sdk/util-format-url"; @@ -80,8 +85,19 @@ export const createPresignedPost = async ( const signingKey = await getSigningKey(sha256, clientCredentials, shortDate, clientRegion, "s3"); const signature = await hmac(sha256, signingKey, encodedPolicy); - const endpoint = await client.config.endpoint(); - if (!client.config.bucketEndpoint) { + let endpoint = await client.config?.endpoint?.(); + let isEndpointV2 = false; + + if (!endpoint) { + isEndpointV2 = true; + endpoint = toEndpointV1( + await getEndpointFromInstructions({ Bucket, Key }, PutObjectCommand as EndpointParameterInstructionsSupplier, { + ...client.config, + }) + ); + } + + if (endpoint && !client.config.isCustomEndpoint && !isEndpointV2) { endpoint.path = `/${Bucket}`; } diff --git a/packages/s3-request-presigner/src/getSignedUrl.spec.ts b/packages/s3-request-presigner/src/getSignedUrl.spec.ts index cad7709a9b0a..4a9e55e892c0 100644 --- a/packages/s3-request-presigner/src/getSignedUrl.spec.ts +++ b/packages/s3-request-presigner/src/getSignedUrl.spec.ts @@ -158,7 +158,10 @@ describe("getSignedUrl", () => { it("should throw if presign request with MRAP ARN and disableMultiregionAccessPoints option", () => { const mockPresigned = "a presigned url"; mockPresign.mockReturnValue(mockPresigned); - const client = new S3Client({ ...clientParams, disableMultiregionAccessPoints: true }); + const client = new S3Client({ + ...clientParams, + disableMultiregionAccessPoints: true, + }); const command = new GetObjectCommand({ Bucket: "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap", Key: "Key", diff --git a/packages/s3-request-presigner/src/getSignedUrl.ts b/packages/s3-request-presigner/src/getSignedUrl.ts index 93201e56a047..aedf3decde7c 100644 --- a/packages/s3-request-presigner/src/getSignedUrl.ts +++ b/packages/s3-request-presigner/src/getSignedUrl.ts @@ -1,6 +1,7 @@ +import { EndpointParameterInstructionsSupplier, getEndpointFromInstructions } from "@aws-sdk/middleware-endpoint"; import { HttpRequest } from "@aws-sdk/protocol-http"; import { Client, Command } from "@aws-sdk/smithy-client"; -import { BuildMiddleware, MetadataBearer, RequestPresigningArguments } from "@aws-sdk/types"; +import { BuildMiddleware, EndpointV2, MetadataBearer, RequestPresigningArguments } from "@aws-sdk/types"; import { formatUrl } from "@aws-sdk/util-format-url"; import { S3RequestPresigner } from "./presigner"; @@ -14,7 +15,24 @@ export const getSignedUrl = async < command: Command, options: RequestPresigningArguments = {} ): Promise => { - const s3Presigner = new S3RequestPresigner({ ...client.config }); + let s3Presigner: S3RequestPresigner; + + if (typeof client.config.endpointProvider === "function") { + const endpointV2: EndpointV2 = await getEndpointFromInstructions( + command.input as Record, + command.constructor as EndpointParameterInstructionsSupplier, + client.config + ); + const authScheme = endpointV2.properties?.authSchemes?.[0]; + s3Presigner = new S3RequestPresigner({ + ...client.config, + signingName: authScheme?.signingName, + region: async () => authScheme?.signingScope, + }); + } else { + s3Presigner = new S3RequestPresigner(client.config); + } + const presignInterceptMiddleware: BuildMiddleware = (next, context) => async (args) => { const { request } = args; @@ -31,6 +49,7 @@ export const getSignedUrl = async < signingRegion: options.signingRegion ?? context["signing_region"], signingService: options.signingService ?? context["signing_service"], }); + return { // Intercept the middleware stack by returning fake response response: {}, diff --git a/packages/types/src/auth.ts b/packages/types/src/auth.ts index be4a74f27bfc..528f28a4121c 100644 --- a/packages/types/src/auth.ts +++ b/packages/types/src/auth.ts @@ -2,6 +2,17 @@ * Authentication schemes represent a way that the service will authenticate the customer’s identity. */ export interface AuthScheme { - name: string; // eg. SigV4 + /** + * @example "v4" for SigV4 + */ + name: string; + /** + * @example "s3" + */ + signingName: string; + /** + * @example "us-east-1" + */ + signingScope: string; properties: Record; } diff --git a/packages/types/src/endpoint.ts b/packages/types/src/endpoint.ts index 7e274b2a8fef..d5263185bd66 100644 --- a/packages/types/src/endpoint.ts +++ b/packages/types/src/endpoint.ts @@ -1,3 +1,5 @@ +import { AuthScheme } from "./auth"; + export interface EndpointPartition { name: string; dnsSuffix: string; @@ -56,7 +58,9 @@ export type EndpointObjectProperty = export interface EndpointV2 { url: URL; - properties?: Record; + properties?: { + authSchemes?: AuthScheme[]; + } & Record; headers?: Record; } diff --git a/packages/types/src/middleware.ts b/packages/types/src/middleware.ts index 9167673f836a..efb55f6ac3d1 100644 --- a/packages/types/src/middleware.ts +++ b/packages/types/src/middleware.ts @@ -1,3 +1,4 @@ +import { EndpointV2 } from "./endpoint"; import { Logger } from "./logger"; import { UserAgent } from "./util"; @@ -353,6 +354,13 @@ export interface MiddlewareStack ex from: MiddlewareStack ): MiddlewareStack; + /** + * Returns a list of the current order of middleware in the stack. + * This does not execute the middleware functions, nor does it + * provide a reference to the stack itself. + */ + identify(): string[]; + /** * Builds a single handler function from zero or more middleware classes and * a core handler. The core handler is meant to send command objects to AWS @@ -387,6 +395,12 @@ export interface HandlerExecutionContext { */ userAgent?: UserAgent; + /** + * Resolved by the endpointMiddleware function of @aws-sdk/middleware-endpoint + * in the serialization stage. + */ + endpointV2?: EndpointV2; + [key: string]: any; } diff --git a/packages/types/src/serde.ts b/packages/types/src/serde.ts index d41dc04c3207..5b0956795607 100644 --- a/packages/types/src/serde.ts +++ b/packages/types/src/serde.ts @@ -6,7 +6,7 @@ import { Decoder, Encoder, Provider } from "./util"; * Interface for object requires an Endpoint set. */ export interface EndpointBearer { - /* TODO(endpointsv2) */ + /* TODO(endpointsv2) post-release */ // Keep using Endpoint V1 interface in serde // After all services have onboard EndpointV2, we need to migrate it to Endpoint V2 interface too. endpoint: Provider; diff --git a/packages/util-endpoints/src/resolveEndpoint.spec.ts b/packages/util-endpoints/src/resolveEndpoint.spec.ts index 70dd60873fba..d65777e7d293 100644 --- a/packages/util-endpoints/src/resolveEndpoint.spec.ts +++ b/packages/util-endpoints/src/resolveEndpoint.spec.ts @@ -20,17 +20,17 @@ describe(resolveEndpoint.name, () => { const mockRules = []; const mockRuleSetParameters: Record = { [boolParamKey]: { - type: "boolean", + type: "Boolean", }, [stringParamKey]: { - type: "string", + type: "String", }, [requiredParamKey]: { - type: "string", + type: "String", required: true, }, [paramWithDefaultKey]: { - type: "string", + type: "String", default: "paramWithDefaultValue", }, }; diff --git a/packages/util-endpoints/src/types/RuleSetObject.ts b/packages/util-endpoints/src/types/RuleSetObject.ts index 6907e92a501d..013e466f89ac 100644 --- a/packages/util-endpoints/src/types/RuleSetObject.ts +++ b/packages/util-endpoints/src/types/RuleSetObject.ts @@ -6,16 +6,17 @@ export type DeprecatedObject = { }; export type ParameterObject = { - type: "string" | "boolean"; + type: "String" | "Boolean"; default?: string | boolean; required?: boolean; documentation?: string; + builtIn?: string; deprecated?: DeprecatedObject; }; export type RuleSetObject = { version: string; - serviceId: string; + serviceId?: string; parameters: Record; rules: RuleSetRules; }; diff --git a/packages/util-middleware/src/normalizeProvider.ts b/packages/util-middleware/src/normalizeProvider.ts index db52572f61c9..8f31973d1e11 100644 --- a/packages/util-middleware/src/normalizeProvider.ts +++ b/packages/util-middleware/src/normalizeProvider.ts @@ -1,5 +1,8 @@ import { Provider } from "@aws-sdk/types"; +/** + * @returns a provider function for the input value if it isn't already one. + */ export const normalizeProvider = (input: T | Provider): Provider => { if (typeof input === "function") return input as Provider; const promisified = Promise.resolve(input);