From d47a54de56105f19c77243dc27aedf9cadfe960b Mon Sep 17 00:00:00 2001 From: Luis Mastrangelo Date: Mon, 5 Feb 2024 22:20:49 -0300 Subject: [PATCH 1/4] Include dependencies to support Prometheus metrics Signed-off-by: Luis Mastrangelo --- package-lock.json | 120 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 4 +- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2d83b8d4..b1dfab01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { - "name": "ethereum-sourcify", - "version": "1.0.0", + "name": "@hashgraph/hedera-sourcify", + "version": "0.3.0-SNAPSHOT", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "ethereum-sourcify", - "version": "1.0.0", + "name": "@hashgraph/hedera-sourcify", + "version": "0.3.0-SNAPSHOT", "license": "MIT", "dependencies": { "@ethereum-sourcify/bytecode-utils": "*", @@ -25,6 +25,7 @@ "express": "^4.17.1", "express-fileupload": "^1.4.0", "express-openapi-validator": "^5.0.4", + "express-prom-bundle": "^7.0.0", "express-session": "^1.17.1", "http-status-codes": "^2.1.4", "ipfs-http-client": "^56.0.3", @@ -32,6 +33,7 @@ "json-refs": "^3.0.15", "memorystore": "^1.6.7", "node-fetch": "2.6.6", + "prom-client": "^15.1.0", "puppeteer": "^20.7.4", "serve-index": "^1.9.1", "solc": "^0.8.17", @@ -5638,6 +5640,14 @@ "@octokit/openapi-types": "^12.11.0" } }, + "node_modules/@opentelemetry/api": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.7.0.tgz", + "integrity": "sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "license": "BSD-3-Clause" @@ -5999,8 +6009,9 @@ } }, "node_modules/@types/express": { - "version": "4.17.17", - "license": "MIT", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -7130,6 +7141,11 @@ "file-uri-to-path": "1.0.0" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "node_modules/bl": { "version": "5.1.0", "license": "MIT", @@ -10706,6 +10722,23 @@ "version": "6.2.1", "license": "MIT" }, + "node_modules/express-prom-bundle": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-7.0.0.tgz", + "integrity": "sha512-VwVaCyGBGHkHdecpTqRdW1Jm2fXK8weCUKjGjNWorc9g4M+cZ3xoj+N9uQzfRWfIPXJG5QOaiAziOIalQzMwgA==", + "dependencies": { + "@types/express": "^4.17.21", + "express": "^4.18.2", + "on-finished": "^2.3.0", + "url-value-parser": "^2.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "prom-client": ">=15.0.0" + } + }, "node_modules/express-session": { "version": "1.17.3", "license": "MIT", @@ -20163,6 +20196,18 @@ "node": ">=0.4.0" } }, + "node_modules/prom-client": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.0.tgz", + "integrity": "sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw==", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/promise-inflight": { "version": "1.0.1", "dev": true, @@ -22963,6 +23008,14 @@ "tcomb": "^3.0.0" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/teeny-request": { "version": "7.1.1", "dev": true, @@ -23937,6 +23990,14 @@ "dev": true, "license": "MIT" }, + "node_modules/url-value-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.2.0.tgz", + "integrity": "sha512-yIQdxJpgkPamPPAPuGdS7Q548rLhny42tg8d4vyTNzFqvOnwqrgHXvgehT09U7fwrzxi3RxCiXjoNUNnNOlQ8A==", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/urlgrey": { "version": "1.0.0", "dev": true, @@ -57784,6 +57845,11 @@ "@octokit/openapi-types": "^12.11.0" } }, + "@opentelemetry/api": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.7.0.tgz", + "integrity": "sha512-AdY5wvN0P2vXBi3b29hxZgSFvdhdxPB9+f0B6s//P9Q8nibRWeA3cHm8UmLpio9ABigkVHJ5NMPk+Mz8VCCyrw==" + }, "@protobufjs/aspromise": { "version": "1.1.2" }, @@ -58063,7 +58129,9 @@ } }, "@types/express": { - "version": "4.17.17", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", "requires": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -58801,6 +58869,11 @@ "file-uri-to-path": "1.0.0" } }, + "bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "bl": { "version": "5.1.0", "requires": { @@ -61210,6 +61283,17 @@ } } }, + "express-prom-bundle": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/express-prom-bundle/-/express-prom-bundle-7.0.0.tgz", + "integrity": "sha512-VwVaCyGBGHkHdecpTqRdW1Jm2fXK8weCUKjGjNWorc9g4M+cZ3xoj+N9uQzfRWfIPXJG5QOaiAziOIalQzMwgA==", + "requires": { + "@types/express": "^4.17.21", + "express": "^4.18.2", + "on-finished": "^2.3.0", + "url-value-parser": "^2.0.0" + } + }, "express-session": { "version": "1.17.3", "requires": { @@ -67655,6 +67739,15 @@ "progress": { "version": "2.0.3" }, + "prom-client": { + "version": "15.1.0", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.0.tgz", + "integrity": "sha512-cCD7jLTqyPdjEPBo/Xk4Iu8jxjuZgZJ3e/oET3L+ZwOuap/7Cw3dH/TJSsZKs1TQLZ2IHpIlRAKw82ef06kmMw==", + "requires": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + } + }, "promise-inflight": { "version": "1.0.1", "dev": true @@ -69526,6 +69619,14 @@ "tcomb": "^3.0.0" } }, + "tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "requires": { + "bintrees": "1.0.2" + } + }, "teeny-request": { "version": "7.1.1", "dev": true, @@ -70157,6 +70258,11 @@ "version": "0.1.0", "dev": true }, + "url-value-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/url-value-parser/-/url-value-parser-2.2.0.tgz", + "integrity": "sha512-yIQdxJpgkPamPPAPuGdS7Q548rLhny42tg8d4vyTNzFqvOnwqrgHXvgehT09U7fwrzxi3RxCiXjoNUNnNOlQ8A==" + }, "urlgrey": { "version": "1.0.0", "dev": true, diff --git a/package.json b/package.json index 8c450f4f..280186d1 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "express": "^4.17.1", "express-fileupload": "^1.4.0", "express-openapi-validator": "^5.0.4", + "express-prom-bundle": "^7.0.0", "express-session": "^1.17.1", "http-status-codes": "^2.1.4", "ipfs-http-client": "^56.0.3", @@ -76,6 +77,7 @@ "json-refs": "^3.0.15", "memorystore": "^1.6.7", "node-fetch": "2.6.6", + "prom-client": "^15.1.0", "puppeteer": "^20.7.4", "serve-index": "^1.9.1", "solc": "^0.8.17", @@ -131,4 +133,4 @@ "text-summary" ] } -} \ No newline at end of file +} From a214e3b7b74bd19523754feaf8ddf4031b4440c4 Mon Sep 17 00:00:00 2001 From: Luis Mastrangelo Date: Tue, 6 Feb 2024 19:24:01 -0300 Subject: [PATCH 2/4] Setup Prometheus metrics Signed-off-by: Luis Mastrangelo --- src/server/server.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/server/server.ts b/src/server/server.ts index a303bff8..43230a54 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -28,6 +28,9 @@ import { initDeprecatedRoutes } from "./deprecated.routes"; import { getAddress } from "ethers"; import { logger } from "../common/loggerLoki"; import { setLibSourcifyLogger } from "@ethereum-sourcify/lib-sourcify"; + +import promBundle from "express-prom-bundle"; + // eslint-disable-next-line @typescript-eslint/no-var-requires const fileUpload = require("express-fileupload"); @@ -175,6 +178,20 @@ export class Server { next(); }); + // Add the options to the prometheus middleware most option are for http_request_duration_seconds histogram metric + const metricsMiddleware = promBundle({ + includeMethod: true, + includePath: true, + includeStatusCode: true, + includeUp: true, + promClient: { + collectDefaultMetrics: { + } + } + }); + // add the prometheus middleware to all routes + this.app.use(metricsMiddleware); + // Session API endpoints require non "*" origins because of the session cookies const sessionPaths = [ "/session", // all paths /session/verify /session/input-files etc. From b0ce8691c1026cbad4338e1abfe69458d456967f Mon Sep 17 00:00:00 2001 From: Luis Mastrangelo Date: Fri, 9 Feb 2024 16:17:51 -0300 Subject: [PATCH 3/4] Include event in metrics Signed-off-by: Luis Mastrangelo --- src/server/server.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/server/server.ts b/src/server/server.ts index 43230a54..4ef3af30 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -30,6 +30,9 @@ import { logger } from "../common/loggerLoki"; import { setLibSourcifyLogger } from "@ethereum-sourcify/lib-sourcify"; import promBundle from "express-prom-bundle"; +import { SourcifyEventManager } from "../common/SourcifyEventManager/SourcifyEventManager"; + +import client from "prom-client"; // eslint-disable-next-line @typescript-eslint/no-var-requires const fileUpload = require("express-fileupload"); @@ -192,6 +195,19 @@ export class Server { // add the prometheus middleware to all routes this.app.use(metricsMiddleware); + const counter = new client.Counter({ + name: 'event_count', + help: 'events that happened during verification labeled with: event and chainId', + labelNames: ['event', 'chainId'], + }); + + SourcifyEventManager.on("*", [ + (event: string, argument: any) => { + const chainId = argument.chainId; + counter.labels({ event, chainId }).inc(1); + }, + ]); + // Session API endpoints require non "*" origins because of the session cookies const sessionPaths = [ "/session", // all paths /session/verify /session/input-files etc. From c199f6b1ac2933b9389e6632c604f3e860405611 Mon Sep 17 00:00:00 2001 From: Luis Mastrangelo Date: Fri, 9 Feb 2024 18:24:00 -0300 Subject: [PATCH 4/4] Use the `sourcify_` prefix for all metrics Signed-off-by: Luis Mastrangelo --- src/server/server.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server/server.ts b/src/server/server.ts index 4ef3af30..98897352 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -187,8 +187,10 @@ export class Server { includePath: true, includeStatusCode: true, includeUp: true, + httpDurationMetricName: "sourcify_http_request_duration_seconds", promClient: { collectDefaultMetrics: { + prefix: "sourcify_", } } }); @@ -196,9 +198,9 @@ export class Server { this.app.use(metricsMiddleware); const counter = new client.Counter({ - name: 'event_count', - help: 'events that happened during verification labeled with: event and chainId', - labelNames: ['event', 'chainId'], + name: "sourcify_event_count", + help: "events that happened during verification labeled with: event and chainId", + labelNames: ["event", "chainId"], }); SourcifyEventManager.on("*", [