Expose metrics endpoint in nodejs functions (#17917)

components/function-controller/internal/controllers/serverless/build_resources.go

@@ -18,7 +18,7 @@ var (
 	istioSidecarInjectFalse = map[string]string{
 		"sidecar.istio.io/inject": "false",
 	}
-	svcTargetPort = intstr.FromInt(8080) // https://github.com/kubeless/runtimes/blob/master/stable/nodejs/kubeless.js#L28
+	svcTargetPort = intstr.FromInt(8080)
 )
 
 func buildRepoFetcherEnvVars(instance *serverlessv1alpha2.Function, gitOptions git.Options) []corev1.EnvVar {

components/function-controller/internal/controllers/serverless/system_state.go

@@ -56,6 +56,14 @@ func (s *systemState) functionLabels() map[string]string {
 	return labels.Merge(functionLabels, internalLabels)
 }
 
+func (s *systemState) functionAnnotations() map[string]string {
+	return map[string]string{
+		"prometheus.io/port": "80",
+		"prometheus.io/path": "/metrics",
+		"prometheus.io/scrape": "true",
+	}
+}
+
 func (s *systemState) buildImageAddress(registryAddress string) string {
 	var imageTag string
 	isGitType := s.instance.TypeOf(serverlessv1alpha2.FunctionTypeGit)
@@ -598,6 +606,7 @@ func (s *systemState) buildService() corev1.Service {
 			Name:      s.instance.GetName(),
 			Namespace: s.instance.GetNamespace(),
 			Labels:    s.functionLabels(),
+			Annotations: s.functionAnnotations(),
 		},
 		Spec: corev1.ServiceSpec{
 			Ports: []corev1.ServicePort{{

components/function-runtimes/nodejs/lib/metrics.js

@@ -0,0 +1,60 @@
+const opentelemetry = require('@opentelemetry/api');
+const { MeterProvider } = require('@opentelemetry/sdk-metrics');
+const { PrometheusExporter } = require('@opentelemetry/exporter-prometheus');
+const { Resource } = require( '@opentelemetry/resources');
+const { SemanticResourceAttributes } = require( '@opentelemetry/semantic-conventions');
+
+let exporter;
+
+function setupMetrics(serviceName){
+
+    const resource = new Resource({
+        [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
+    });
+
+
+    const myServiceMeterProvider = new MeterProvider({
+    resource,
+    });
+
+    exporter = new PrometheusExporter({ preventServerStart: true})
+
+    myServiceMeterProvider.addMetricReader(exporter);
+
+    opentelemetry.metrics.setGlobalMeterProvider(myServiceMeterProvider);
+
+}
+
+function createFunctionCallsTotalCounter(name){
+  const meter =  opentelemetry.metrics.getMeter(name)
+  return meter.createCounter('function_calls_total',{
+    description: 'Number of calls to user function',
+  }); 
+}
+
+
+function createFunctionFailuresTotalCounter(name){
+  const meter =  opentelemetry.metrics.getMeter(name)
+  return meter.createCounter('function_failures_total',{
+    description: 'Number of exceptions in user function',
+  });  
+}
+
+function createFunctionDurationHistogram(name){
+  const meter =  opentelemetry.metrics.getMeter(name)
+  return meter.createHistogram("function_duration_miliseconds",{
+    description: 'Duration of user function in miliseconds',
+  });  
+}
+
+const getMetrics = (req, res) => {
+  exporter.getMetricsRequestHandler(req, res);
+};
+
+module.exports = {
+    setupMetrics,
+    createFunctionCallsTotalCounter,
+    createFunctionFailuresTotalCounter,
+    createFunctionDurationHistogram,
+    getMetrics,
+}

components/function-runtimes/nodejs/nodejs16/package.json

@@ -14,6 +14,8 @@
     "@opentelemetry/propagator-b3": "^1.2.0",
     "@opentelemetry/sdk-trace-node": "^1.2.0",
     "@opentelemetry/tracing": "^0.18.0",
+    "@opentelemetry/sdk-metrics": "^1.15.1",
+    "@opentelemetry/exporter-prometheus": "^0.41.1",
     "axios": "0.26.1",
     "co": "~4.6.0",
     "express": ">=4.0.0",

components/function-runtimes/nodejs/nodejs18/package.json

@@ -14,6 +14,8 @@
     "@opentelemetry/propagator-b3": "^1.9.1",
     "@opentelemetry/sdk-trace-node": "^1.9.1",
     "@opentelemetry/tracing": "^0.18.0",
+    "@opentelemetry/sdk-metrics": "^1.15.1",
+    "@opentelemetry/exporter-prometheus": "^0.41.1",
     "axios": "^1.3.3",
     "co": "^4.6.0",
     "express": ">=4.18.2",

components/function-runtimes/nodejs/server.js

@@ -6,6 +6,7 @@ const process = require("process");
 const morgan = require("morgan");
 
 const { setupTracer, startNewSpan } = require('./lib/tracer')
+const { getMetrics, setupMetrics, createFunctionDurationHistogram, createFunctionCallsTotalCounter, createFunctionFailuresTotalCounter  } = require('./lib/metrics')
 
 
 // To catch unhandled exceptions thrown by user code async callbacks,
@@ -22,11 +23,17 @@ const defaultFunctioneName = serviceName.substring(0, serviceName.lastIndexOf("-
 const functionName = process.env.FUNC_NAME || defaultFunctioneName;
 const bodySizeLimit = Number(process.env.REQ_MB_LIMIT || '1');
 const funcPort = Number(process.env.FUNC_PORT || '8080');
-const tracer = setupTracer([serviceName, serviceNamespace].join('.'));
+
+const otelServiceName = [serviceName, serviceNamespace].join('.')
+const tracer = setupTracer(otelServiceName);
+setupMetrics(otelServiceName);
+
+const callsTotalCounter = createFunctionCallsTotalCounter(otelServiceName);
+const failuresTotalCounter = createFunctionFailuresTotalCounter(otelServiceName);
+const durationHistogram = createFunctionDurationHistogram(otelServiceName);
 
 //require express must be called AFTER tracer was setup!!!!!!
 const express = require("express");
-const { type } = require('os');
 const app = express();
 
 
@@ -66,20 +73,32 @@ app.use(bodyParser.raw({limit: `${bodySizeLimit}mb`, type: () => true}))
 app.use(helper.handleTimeOut);
 
 app.get("/healthz", (req, res) => {
-    res.status(200).send("")
+    res.status(200).send("OK")
+})
+
+app.get("/metrics", (req, res) => {
+    getMetrics(req, res)
 })
 
+app.get('/favicon.ico', (req, res) => res.status(204));
+
 // Generic route -- all http requests go to the user function.
-app.all("*", (req, res) => {
+app.all("*", (req, res, next) => {
+
+
     res.header('Access-Control-Allow-Origin', '*');
     if (req.method === 'OPTIONS') {
         // CORS preflight support (Allow any method or header requested)
         res.header('Access-Control-Allow-Methods', req.headers['access-control-request-method']);
         res.header('Access-Control-Allow-Headers', req.headers['access-control-request-headers']);
         res.end();
     } else {
+
+        callsTotalCounter.add(1)
+        const startTime = new Date().getTime()
 
         if (!userFunction) {
+            failuresTotalCounter.add(1)
             res.status(500).send("User function not loaded");
             return;
         }
@@ -135,6 +154,7 @@ app.all("*", (req, res) => {
                 })
                 .catch((err) => {
                     helper.handleError(err, span, sendResponse)
+                    failuresTotalCounter.add(1);
                 })
                 .finally(()=>{
                     span.end();
@@ -144,9 +164,14 @@ app.all("*", (req, res) => {
             }
         } catch (err) {
             helper.handleError(err, span, sendResponse)
+            failuresTotalCounter.add(1);
         } finally {
             span.end();
         }
+
+        const endTime = new Date().getTime()
+        const executionTime = endTime - startTime;
+        durationHistogram.record(executionTime);
     }
 });
 

resources/serverless/values.yaml

@@ -81,28 +81,28 @@ global:
       directory: "prod"
     function_controller:
       name: "function-controller"
-      version: "PR-17927"
+      version: "PR-17917"
       directory: "dev"
     function_webhook:
       name: "function-webhook"
-      version: "v20230628-61d97068"
-      directory: "prod"
+      version: "PR-17917"
+      directory: "dev"
     function_build_init:
       name: "function-build-init"
-      version: "v20230628-61d97068"
-      directory: "prod"
+      version: "PR-17917"
+      directory: "dev"
     function_registry_gc:
       name: "function-registry-gc"
       version: "v20230628-61d97068"
       directory: "prod"
     function_runtime_nodejs16:
       name: "function-runtime-nodejs16"
-      version: "v20230720-e1268109"
-      directory: "prod"
+      version: "PR-17917"
+      directory: "dev"
     function_runtime_nodejs18:
       name: "function-runtime-nodejs18"
-      version: "v20230720-e1268109"
-      directory: "prod"
+      version: "PR-17917"
+      directory: "dev"
     function_runtime_python39:
       name: "function-runtime-python39"
       version: "v20230720-e1268109"