Add More Throttlers for Historian and Alfred Endpoints (#14682)

Add tenant-level and cluster-level throttler for: 1. create summary in historian 2. get summary in historian 3. get session in alfred Add cluster-level throttler for: 1. create doc in alfred 2. get deltas in alfred
microsoft · Mar 30, 2023 · 278d803 · 278d803
1 parent 1796f60
commit 278d803
Show file tree

Hide file tree

Showing 33 changed files with 981 additions and 224 deletions.
diff --git a/server/BREAKING.md b/server/BREAKING.md
@@ -21,7 +21,7 @@ export class AlfredResources implements core.IResources {
         public webSocketLibrary: string,
         public orderManager: core.IOrdererManager,
         public tenantManager: core.ITenantManager,
-        public restTenantThrottler: core.IThrottler,
+        public restTenantThrottlers: Map<string, core.IThrottler>,
         public restClusterThrottlers: Map<string, core.IThrottler>,
         public socketConnectTenantThrottler: core.IThrottler,
         public socketConnectClusterThrottler: core.IThrottler,
@@ -51,7 +51,7 @@ export class AlfredResourcesFactory implements core.IResourcesFactory<AlfredReso
             webSocketLibrary,
             orderManager,
             tenantManager,
-            restTenantThrottler,
+            restTenantThrottlers,
             restClusterThrottler,
             socketConnectTenantThrottler,
             socketConnectClusterThrottler

diff --git a/server/charts/historian/templates/historian-configmap.yaml b/server/charts/historian/templates/historian-configmap.yaml
@@ -37,12 +37,55 @@ data:
             "endpoint": "{{ .Values.historian.error.endpoint }}"
         },
         "throttling": {
-            "maxRequestsPerMs": {{ .Values.historian.throttling.maxRequestsPerMs }},
-            "maxRequestBurst": {{ .Values.historian.throttling.maxRequestBurst }},
-            "minCooldownIntervalInMs": {{ .Values.historian.throttling.minCooldownIntervalInMs }},
-            "minThrottleIntervalInMs": {{ .Values.historian.throttling.minThrottleIntervalInMs }},
-            "maxInMemoryCacheSize": {{ .Values.historian.throttling.maxInMemoryCacheSize }},
-            "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.maxInMemoryCacheAgeInMs }}
+            "restCallsPerTenant": {
+                "generalRestCall": {
+                    "maxPerMs": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.maxPerMs }},
+                    "maxBurst": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.maxBurst }},
+                    "minCooldownIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.minCooldownIntervalInMs }},
+                    "minThrottleIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.minThrottleIntervalInMs }},
+                    "maxInMemoryCacheSize": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.maxInMemoryCacheSize }},
+                    "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.maxInMemoryCacheAgeInMs }},
+                    "enableEnhancedTelemetry": {{ .Values.historian.throttling.restCallsPerTenant.generalRestCall.enableEnhancedTelemetry }}
+                },
+                "getSummary": {
+                    "maxPerMs": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.maxPerMs }},
+                    "maxBurst": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.maxBurst }},
+                    "minCooldownIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.minCooldownIntervalInMs }},
+                    "minThrottleIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.minThrottleIntervalInMs }},
+                    "maxInMemoryCacheSize": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.maxInMemoryCacheSize }},
+                    "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.maxInMemoryCacheAgeInMs }},
+                    "enableEnhancedTelemetry": {{ .Values.historian.throttling.restCallsPerTenant.getSummary.enableEnhancedTelemetry }}
+                },
+                "createSummary": {
+                    "maxPerMs": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.maxPerMs }},
+                    "maxBurst": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.maxBurst }},
+                    "minCooldownIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.minCooldownIntervalInMs }},
+                    "minThrottleIntervalInMs": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.minThrottleIntervalInMs }},
+                    "maxInMemoryCacheSize": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.maxInMemoryCacheSize }},
+                    "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.maxInMemoryCacheAgeInMs }},
+                    "enableEnhancedTelemetry": {{ .Values.historian.throttling.restCallsPerTenant.createSummary.enableEnhancedTelemetry }}
+                }
+            },
+            "restCallsPerCluster": {
+                "getSummary": {
+                    "maxPerMs": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.maxPerMs }},
+                    "maxBurst": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.maxBurst }},
+                    "minCooldownIntervalInMs": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.minCooldownIntervalInMs }},
+                    "minThrottleIntervalInMs": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.minThrottleIntervalInMs }},
+                    "maxInMemoryCacheSize": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.maxInMemoryCacheSize }},
+                    "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.maxInMemoryCacheAgeInMs }},
+                    "enableEnhancedTelemetry": {{ .Values.historian.throttling.restCallsPerCluster.getSummary.enableEnhancedTelemetry }}
+                },
+                "createSummary": {
+                    "maxPerMs": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.maxPerMs }},
+                    "maxBurst": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.maxBurst }},
+                    "minCooldownIntervalInMs": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.minCooldownIntervalInMs }},
+                    "minThrottleIntervalInMs": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.minThrottleIntervalInMs }},
+                    "maxInMemoryCacheSize": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.maxInMemoryCacheSize }},
+                    "maxInMemoryCacheAgeInMs": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.maxInMemoryCacheAgeInMs }},
+                    "enableEnhancedTelemetry": {{ .Values.historian.throttling.restCallsPerCluster.createSummary.enableEnhancedTelemetry }}
+                }
+            }
         },
         "restGitService": {
             "disableGitCache": {{ .Values.historian.restGitService.disableGitCache }}

diff --git a/server/historian/packages/historian-base/src/app.ts b/server/historian/packages/historian-base/src/app.ts
@@ -25,7 +25,8 @@ import { getDocumentIdFromRequest, getTenantIdFromRequest } from "./utils";
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
+	restClusterThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ) {
@@ -71,7 +72,14 @@ export function create(
 	app.use(cors());
 	app.use(bindCorrelationId(asyncLocalStorage));
 
-	const apiRoutes = routes.create(config, tenantService, throttler, cache, asyncLocalStorage);
+	const apiRoutes = routes.create(
+		config,
+		tenantService,
+		restTenantThrottlers,
+		restClusterThrottlers,
+		cache,
+		asyncLocalStorage,
+	);
 	app.use(apiRoutes.git.blobs);
 	app.use(apiRoutes.git.refs);
 	app.use(apiRoutes.git.tags);

diff --git a/server/historian/packages/historian-base/src/routes/git/blobs.ts b/server/historian/packages/historian-base/src/routes/git/blobs.ts
@@ -16,20 +16,24 @@ import * as nconf from "nconf";
 import winston from "winston";
 import { ICache, ITenantService } from "../../services";
 import * as utils from "../utils";
+import { Constants } from "../../utils";
 
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ): Router {
 	const router: Router = Router();
 
-	const commonThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
+	const tenantThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
 		throttleIdPrefix: (req) => getParam(req.params, "tenantId"),
-		throttleIdSuffix: utils.Constants.throttleIdSuffix,
+		throttleIdSuffix: Constants.historianRestThrottleIdSuffix,
 	};
+	const restTenantGeneralThrottler = restTenantThrottlers.get(
+		Constants.generalRestCallThrottleIdPrefix,
+	);
 
 	async function createBlob(
 		tenantId: string,
@@ -69,8 +73,8 @@ export function create(
 	 */
 	router.get(
 		"/repos/ping",
-		throttle(throttler, winston, {
-			...commonThrottleOptions,
+		throttle(restTenantGeneralThrottler, winston, {
+			...tenantThrottleOptions,
 			throttleIdPrefix: "ping",
 		}),
 		async (request, response) => {
@@ -81,7 +85,7 @@ export function create(
 	router.post(
 		"/repos/:ignored?/:tenantId/git/blobs",
 		utils.validateRequestParams("tenantId"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const blobP = createBlob(
 				request.params.tenantId,
@@ -98,7 +102,7 @@ export function create(
 	router.get(
 		"/repos/:ignored?/:tenantId/git/blobs/:sha",
 		utils.validateRequestParams("tenantId", "sha"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const useCache = !("disableCache" in request.query);
 			const blobP = getBlob(
@@ -117,7 +121,7 @@ export function create(
 	router.get(
 		"/repos/:ignored?/:tenantId/git/blobs/raw/:sha",
 		utils.validateRequestParams("tenantId", "sha"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const useCache = !("disableCache" in request.query);
 

diff --git a/server/historian/packages/historian-base/src/routes/git/commits.ts b/server/historian/packages/historian-base/src/routes/git/commits.ts
@@ -16,20 +16,24 @@ import * as nconf from "nconf";
 import winston from "winston";
 import { ICache, ITenantService } from "../../services";
 import * as utils from "../utils";
+import { Constants } from "../../utils";
 
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ): Router {
 	const router: Router = Router();
 
-	const commonThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
+	const tenantThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
 		throttleIdPrefix: (req) => getParam(req.params, "tenantId"),
-		throttleIdSuffix: utils.Constants.throttleIdSuffix,
+		throttleIdSuffix: Constants.historianRestThrottleIdSuffix,
 	};
+	const restTenantGeneralThrottler = restTenantThrottlers.get(
+		Constants.generalRestCallThrottleIdPrefix,
+	);
 
 	async function createCommit(
 		tenantId: string,
@@ -67,7 +71,7 @@ export function create(
 	router.post(
 		"/repos/:ignored?/:tenantId/git/commits",
 		utils.validateRequestParams("tenantId"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const commitP = createCommit(
 				request.params.tenantId,
@@ -82,7 +86,7 @@ export function create(
 	router.get(
 		"/repos/:ignored?/:tenantId/git/commits/:sha",
 		utils.validateRequestParams("tenantId", "sha"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const useCache = !("disableCache" in request.query);
 			const commitP = getCommit(

diff --git a/server/historian/packages/historian-base/src/routes/git/refs.ts b/server/historian/packages/historian-base/src/routes/git/refs.ts
@@ -20,20 +20,24 @@ import * as nconf from "nconf";
 import winston from "winston";
 import { ICache, ITenantService } from "../../services";
 import * as utils from "../utils";
+import { Constants } from "../../utils";
 
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ): Router {
 	const router: Router = Router();
 
-	const commonThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
+	const tenantThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
 		throttleIdPrefix: (req) => getParam(req.params, "tenantId"),
-		throttleIdSuffix: utils.Constants.throttleIdSuffix,
+		throttleIdSuffix: Constants.historianRestThrottleIdSuffix,
 	};
+	const restTenantGeneralThrottler = restTenantThrottlers.get(
+		Constants.generalRestCallThrottleIdPrefix,
+	);
 
 	async function getRefs(tenantId: string, authorization: string): Promise<git.IRef[]> {
 		const service = await utils.createGitService(
@@ -106,7 +110,7 @@ export function create(
 
 	router.get(
 		"/repos/:ignored?/:tenantId/git/refs",
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const refsP = getRefs(request.params.tenantId, request.get("Authorization"));
 			utils.handleResponse(refsP, response, false);
@@ -115,7 +119,7 @@ export function create(
 
 	router.get(
 		"/repos/:ignored?/:tenantId/git/refs/*",
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const refP = getRef(
 				request.params.tenantId,
@@ -128,7 +132,7 @@ export function create(
 
 	router.post(
 		"/repos/:ignored?/:tenantId/git/refs",
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const refP = createRef(
 				request.params.tenantId,
@@ -142,7 +146,7 @@ export function create(
 	router.patch(
 		"/repos/:ignored?/:tenantId/git/refs/*",
 		utils.validateRequestParams("tenantId", 0),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const refP = updateRef(
 				request.params.tenantId,
@@ -157,7 +161,7 @@ export function create(
 	router.delete(
 		"/repos/:ignored?/:tenantId/git/refs/*",
 		utils.validateRequestParams("tenantId", 0),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const refP = deleteRef(
 				request.params.tenantId,

diff --git a/server/historian/packages/historian-base/src/routes/git/tags.ts b/server/historian/packages/historian-base/src/routes/git/tags.ts
@@ -16,20 +16,24 @@ import * as nconf from "nconf";
 import winston from "winston";
 import { ICache, ITenantService } from "../../services";
 import * as utils from "../utils";
+import { Constants } from "../../utils";
 
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ): Router {
 	const router: Router = Router();
 
-	const commonThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
+	const tenantThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
 		throttleIdPrefix: (req) => getParam(req.params, "tenantId"),
-		throttleIdSuffix: utils.Constants.throttleIdSuffix,
+		throttleIdSuffix: Constants.historianRestThrottleIdSuffix,
 	};
+	const restTenantGeneralThrottler = restTenantThrottlers.get(
+		Constants.generalRestCallThrottleIdPrefix,
+	);
 
 	async function createTag(
 		tenantId: string,
@@ -62,7 +66,7 @@ export function create(
 	router.post(
 		"/repos/:ignored?/:tenantId/git/tags",
 		utils.validateRequestParams("tenantId"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const tagP = createTag(
 				request.params.tenantId,
@@ -76,7 +80,7 @@ export function create(
 	router.get(
 		"/repos/:ignored?/:tenantId/git/tags/*",
 		utils.validateRequestParams("tenantId", 0),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const tagP = getTag(
 				request.params.tenantId,

diff --git a/server/historian/packages/historian-base/src/routes/git/trees.ts b/server/historian/packages/historian-base/src/routes/git/trees.ts
@@ -16,20 +16,24 @@ import * as nconf from "nconf";
 import winston from "winston";
 import { ICache, ITenantService } from "../../services";
 import * as utils from "../utils";
+import { Constants } from "../../utils";
 
 export function create(
 	config: nconf.Provider,
 	tenantService: ITenantService,
-	throttler: IThrottler,
+	restTenantThrottlers: Map<string, IThrottler>,
 	cache?: ICache,
 	asyncLocalStorage?: AsyncLocalStorage<string>,
 ): Router {
 	const router: Router = Router();
 
-	const commonThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
+	const tenantThrottleOptions: Partial<IThrottleMiddlewareOptions> = {
 		throttleIdPrefix: (req) => getParam(req.params, "tenantId"),
-		throttleIdSuffix: utils.Constants.throttleIdSuffix,
+		throttleIdSuffix: Constants.historianRestThrottleIdSuffix,
 	};
+	const restTenantGeneralThrottler = restTenantThrottlers.get(
+		Constants.generalRestCallThrottleIdPrefix,
+	);
 
 	async function createTree(
 		tenantId: string,
@@ -68,7 +72,7 @@ export function create(
 	router.post(
 		"/repos/:ignored?/:tenantId/git/trees",
 		utils.validateRequestParams("tenantId"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const treeP = createTree(
 				request.params.tenantId,
@@ -82,7 +86,7 @@ export function create(
 	router.get(
 		"/repos/:ignored?/:tenantId/git/trees/:sha",
 		utils.validateRequestParams("tenantId", "sha"),
-		throttle(throttler, winston, commonThrottleOptions),
+		throttle(restTenantGeneralThrottler, winston, tenantThrottleOptions),
 		(request, response, next) => {
 			const useCache = !("disableCache" in request.query);
 			const treeP = getTree(