diff --git a/charts/hedera-network/config-files/otel-collector-config.yaml b/charts/hedera-network/config-files/otel-collector-config.yaml index 791127bd0..962a9ff00 100644 --- a/charts/hedera-network/config-files/otel-collector-config.yaml +++ b/charts/hedera-network/config-files/otel-collector-config.yaml @@ -2,68 +2,59 @@ receivers: otlp: protocols: grpc: - hostmetrics: - collection_interval: "15s" - scrapers: - cpu: - disk: - load: - filesystem: - memory: - network: - paging: - processes: + http: + + # Collect node metrics + prometheus: + config: + scrape_configs: + - job_name: 'node-metrics-scraper' # network-node metrics + scrape_interval: {{ .otelDefaults.receivers.prometheus.scrapeInterval }} + static_configs: + - targets: {{ .otelDefaults.receivers.prometheus.scrapeTargets }} + +processors: + batch: exporters: prometheus: - endpoint: "0.0.0.0:8889" + endpoint: "0.0.0.0:8889" # note: network-node-svc exposes this port for all prometheus metrics instead of node's port 9999 const_labels: source: p-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="p-otel-collector"} - {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }} + + {{- if eq .otelDefaults.exporters.prometheusRemoteWrite.enable "true" }} prometheusremotewrite: - endpoint: "{{ .otelDefaults.prometheusRemoteWrite.endpoint }}" + endpoint: "{{ .otelDefaults.exporters.prometheusRemoteWrite.endpoint }}" tls: - insecure: true + {{- .otelDefaults.exporters.prometheusRemoteWrite.tls | toYaml | nindent 6 }} external_labels: source: prw-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="prw-otel-collector"} {{- end }} -# logging: -# verbosity: detailed -# sampling_initial: 5 + otlp: + endpoint: "{{ .otelDefaults.exporters.otlp.endpoint }}" + {{- if .otelDefaults.exporters.otlp.headers }} + headers: + {{- .otelDefaults.exporters.otlp.headers | toYaml | nindent 6 }} + {{- end }} + tls: + {{- .otelDefaults.exporters.otlp.tls | toYaml | nindent 6 }} -# jaeger: -# endpoint: jaeger-all-in-one:14250 -# tls: -# insecure: true -# -# otlp: -# endpoint: tempo:4317 -# tls: -# insecure: true - -processors: - batch: extensions: health_check: service: -# telemetry: -# logs: -# level: "debug" -# development: true extensions: [health_check] pipelines: - #traces: - # receivers: [otlp] - # processors: [batch] - # exporters: [logging, otlp, jaeger] + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] metrics: - receivers: [otlp, hostmetrics] + receivers: [prometheus] processors: [batch] - exporters: - - prometheus - {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }} - - prometheusremotewrite - {{- end }} -# exporters: [logging, prometheus{{ if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}, prometheusremotewrite{{ end }}] + exporters: + - prometheus + {{- if eq .otelDefaults.exporters.prometheusRemoteWrite.enable "true" }} + - prometheusremotewrite + {{- end }} diff --git a/charts/hedera-network/templates/services/network-node-svc.yaml b/charts/hedera-network/templates/services/network-node-svc.yaml index 9757a9192..8dc400988 100644 --- a/charts/hedera-network/templates/services/network-node-svc.yaml +++ b/charts/hedera-network/templates/services/network-node-svc.yaml @@ -11,23 +11,24 @@ spec: selector: app: network-{{ $nodeConfig.name }} ports: + # GOSSIP port is used by nodes for gossip protocol - name: gossip protocol: TCP port: 50111 # gossip port targetPort: 50111 + # GRPC-NON-TLS port is used for services API - name: grpc-non-tls protocol: TCP port: 50211 # non-tls grpc client port targetPort: 50211 + # GRPC-TLS port - name: grpc-tls protocol: TCP port: 50212 # tls grpc client port targetPort: 50212 - - name: otel-metrics - protocol: TCP - port: 8888 - targetPort: 8888 - - name: prometheus + # Node metrics port + # Rather than exposing the node's metrics port 9999 directly, we expose otel-collect's prometheus port here. + - name: prometheus # otel-collector's prometheus exporter port protocol: TCP port: 8889 targetPort: 8889 diff --git a/charts/hedera-network/templates/sidecars/_otel-collector.tpl b/charts/hedera-network/templates/sidecars/_otel-collector.tpl index 6935db96c..9944a2ac0 100644 --- a/charts/hedera-network/templates/sidecars/_otel-collector.tpl +++ b/charts/hedera-network/templates/sidecars/_otel-collector.tpl @@ -8,26 +8,26 @@ securityContext: {{- include "fullstack.root.security.context" . | nindent 4 }} ports: - - name: otel-health - containerPort: 13133 - protocol: TCP - - name: otel-metrics - containerPort: 8888 - protocol: TCP - - name: otel-otlp + - name: otlp # otel port defined in otel-collector config containerPort: 4317 protocol: TCP - - name: prometheus + - name: prometheus # prometheus exporter port as specified in otel-collector-config.yaml containerPort: 8889 protocol: TCP - {{- with default $defaults.livenessProbe $otel.livenessProbe }} + - name: health # for otel-collector liveness check + containerPort: 13133 + protocol: TCP + - name: metrics # default metrics port exposed by the otel-collector itself + containerPort: 8888 + protocol: TCP livenessProbe: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with default $defaults.readinessProbe $otel.readinessProbe }} + httpGet: + path: / + port: health readinessProbe: - {{- toYaml . | nindent 4 }} - {{- end }} + httpGet: + path: / + port: health volumeMounts: - name: otel-collector-volume mountPath: /etc/otelcol-contrib/config.yaml diff --git a/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml b/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml index 241a3db43..35d2893f8 100644 --- a/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml +++ b/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml @@ -10,8 +10,6 @@ spec: matchLabels: fullstack.hedera.com/type: network-node-svc endpoints: - - port: otel-metrics - interval: 5s - - port: prometheus - interval: 5s + - port: prometheus # must match the prometheus port-name in network-node-svc.yaml + interval: 10s # ideally it should be higher than the node-metrics-scraper interval set in otel-collector-config.yaml {{- end }} diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml index 3c332de11..9cd932812 100644 --- a/charts/hedera-network/values.yaml +++ b/charts/hedera-network/values.yaml @@ -161,18 +161,24 @@ defaults: repository: "otel/opentelemetry-collector-contrib" tag: "0.72.0" pullPolicy: "IfNotPresent" - livenessProbe: - httpGet: - path: / - port: otel-health - readinessProbe: - httpGet: - path: / - port: otel-health resources: {} - prometheusRemoteWrite: - enable: "false" - endpoint: "http://prometheus.default.svc:9090/api/v1/write" + receivers: + prometheus: + scrapeTargets: [ 0.0.0.0:9999 ] # hedera node metrics are exposed at port 9090 + scrapeInterval: 5s + exporters: + otlp: + endpoint: tempo:4317 + tls: + insecure: true + prometheus: + tls: + insecure: true + prometheusRemoteWrite: + enable: "false" + endpoint: "http://prometheus.default.svc:9090/api/v1/write" + tls: + insecure: true # This configures the minio tenant subchart # Reference for configuration: https://github.com/minio/operator/blob/master/helm/tenant/values.yaml diff --git a/dev/Makefile b/dev/Makefile index e804fda54..7460cf59d 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -92,6 +92,12 @@ deploy-network: deploy-chart kubectl get pods -o wide && \ echo ">> Waiting for network-node pods to be active (first deployment takes ~10m)...." && \ kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.com/type=network-node --timeout=900s + echo "" && \ + echo ">> Service Information...." && \ + echo "" && \ + kubectl get svc -o wide && \ + echo ">> Waiting for pods to be up (timeout 600s)" && \ + kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l fullstack.hedera.com/type=network-node --timeout=600s .PHONY: destroy-network destroy-network: destroy-test-container destroy-chart @@ -151,7 +157,7 @@ destroy-gateway-api: #source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_haproxy_ingress source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api -######################################### Prometheus ################################# +######################################### Telemetry ################################# .PHONY: fetch-prometheus-operator-bundle fetch-prometheus-operator-bundle: source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && fetch-prometheus-operator-bundle @@ -164,14 +170,31 @@ deploy-prometheus-operator: fetch-prometheus-operator-bundle destroy-prometheus-operator: source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy-prometheus-operator +.PHONY: deploy-grafana-tempo +deploy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy_grafana_tempo && expose_grafana + +.PHONY: destroy-grafana-tempo +destroy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_grafana && destroy_grafana_tempo + .PHONY: deploy-prometheus deploy-prometheus: deploy-prometheus-operator - source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus && expose_prometheus .PHONY: destroy-prometheus destroy-prometheus: - -source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy-prometheus - make destroy-prometheus-operator + -source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_prometheus && destroy-prometheus + -${MAKE} destroy-prometheus-operator + +.PHONY: deploy-telemetry-stack +deploy-telemetry-stack: deploy-prometheus deploy-grafana-tempo + +.PHONY: destroy-telemetry-stack +destroy-telemetry-stack: + # Note: - prefix ensures errors are ignored and continues + -${MAKE} destroy-prometheus + -${MAKE} destroy-grafana-tempo ######################################### MinIO ################################# .PHONY: deploy-minio-operator diff --git a/dev/scripts/telemetry.sh b/dev/scripts/telemetry.sh index 5fd765c5e..5e41e1d7c 100644 --- a/dev/scripts/telemetry.sh +++ b/dev/scripts/telemetry.sh @@ -61,7 +61,7 @@ function deploy-prometheus() { kubectl create -f "${PROMETHEUS_RBAC_YAML}" sleep 10 kubectl create -f "${PROMETHEUS_YAML}" - echo "Waiting for prometheus to be active..." + echo "Waiting for prometheus to be active (timeout 300s)..." kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=prometheus -n default --timeout 300s } @@ -82,7 +82,8 @@ function deploy-prometheus-example-app() { echo "PROMETHEUS_EXAMPLE_APP_YAML: ${PROMETHEUS_EXAMPLE_APP_YAML}" echo "-----------------------------------------------------------------------------------------------------" kubectl create -f "${PROMETHEUS_EXAMPLE_APP_YAML}" - kubectl wait --for=condition=Ready pods -l app=prometheus-example-app -n default --timeout 60s + echo "Waiting for prometheus example app to be active (timeout 300s)..." + kubectl wait --for=condition=Ready pods -l app=prometheus-example-app -n default --timeout 300s } function destroy-prometheus-example-app() { @@ -94,3 +95,80 @@ function destroy-prometheus-example-app() { local status="$?" [[ "${status}" = 0 ]] && sleep 10 } + +function expose_prometheus() { + export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=prometheus,app.kubernetes.io/instance=prometheus" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace default port-forward $POD_NAME 9090 & + echo "Prometheus is exposed from ${POD_NAME} to port 9090" +} + +function unexpose_prometheus() { + export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=prometheus,app.kubernetes.io/instance=prometheus" -o jsonpath="{.items[0].metadata.name}") + export PID=$(ps aux | grep "port-forward ${POD_NAME}" | sed -n 2p | awk '{ print $2 }') + [[ -z "${PID}" ]] && echo "No Prometheus port-forward PID is found" && return 0 + + if [[ "${PID}" ]]; then + echo "" + echo "Un-exposing Prometheus: ${POD_NAME} for PID: ${PID}" + echo "-----------------------------------------------------------------------------------------------------" + kill "${PID}" &>/dev/null || true + fi +} + +function deploy_grafana_tempo() { + echo "" + echo "Deploying Grafana" + echo "-----------------------------------------------------------------------------------------------------" + helm repo add grafana https://grafana.github.io/helm-charts + helm repo update + helm upgrade --install tempo grafana/tempo + echo "Waiting for tempo to be active (timeout 300s)..." + kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l "app.kubernetes.io/name=tempo,app.kubernetes.io/instance=tempo" --timeout=300s + + helm upgrade -f "${TELEMETRY_DIR}/grafana/grafana-values.yaml" --install grafana grafana/grafana + echo "Waiting for grafana to be active (timeout 300s)..." + kubectl wait --for=jsonpath='{.status.phase}'=Running pod -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana" --timeout=300s +} + +function destroy_grafana_tempo() { + echo "" + echo "Destroying Grafana" + echo "-----------------------------------------------------------------------------------------------------" + helm delete grafana + helm delete tempo +} + +function expose_grafana() { + export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace default port-forward $POD_NAME 3000 & + echo "Grafana is exposed from ${POD_NAME} to port 3000" +} + +function unexpose_grafana() { + export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=grafana" -o jsonpath="{.items[0].metadata.name}") + export PID=$(ps aux | grep "port-forward ${POD_NAME}" | sed -n 2p | awk '{ print $2 }') + [[ -z "${PID}" ]] && echo "No Grafana port-forward PID is found" && return 0 + + if [[ "${PID}" ]]; then + echo "" + echo "Un-exposing Grafana: ${POD_NAME} for PID: ${PID}" + echo "-----------------------------------------------------------------------------------------------------" + kill "${PID}" &>/dev/null || true + fi +} + +function deploy_tracing_example_app() { + echo "" + echo "Deploying Example Tracing App" + echo "-----------------------------------------------------------------------------------------------------" + kubectl create -f "${TELEMETRY_DIR}/grafana/example-tracing-app.yaml" + echo "Waiting for tracing example app to be active (timeout 300s)..." + kubectl wait --for=condition=Ready pods -l app=xk6-tracing -n default --timeout 300s +} + +function destroy_tracing_example_app() { + echo "" + echo "Destroying Example Tracing App" + echo "-----------------------------------------------------------------------------------------------------" + kubectl delete -f "${TELEMETRY_DIR}/grafana/example-tracing-app.yaml" +} diff --git a/dev/telemetry/Makefile b/dev/telemetry/Makefile new file mode 100644 index 000000000..a718d2ccf --- /dev/null +++ b/dev/telemetry/Makefile @@ -0,0 +1,83 @@ +# Force the use of bash as the shell for more features +SHELL=/bin/bash + +# Ensure we can catch error to run cleanup when multiple make commands are run in sequence. +# Here we tell make to run all scripts as one-shell and also set 'pipefail' and 'errexit' flags. +# https://stackoverflow.com/questions/28597794/how-can-i-clean-up-after-an-error-in-a-makefile +SHELLOPTS:=$(if $(SHELLOPTS),$(SHELLOPTS):)pipefail:errexit +.ONESHELL: + +SCRIPTS_DIR=$(PWD)/../scripts +TELEMETRY_SCRIPT="telemetry.sh" + +.PHONY: fetch-prometheus-operator-bundle +fetch-prometheus-operator-bundle: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && fetch-prometheus-operator-bundle + +.PHONY: deploy-prometheus-operator +deploy-prometheus-operator: fetch-prometheus-operator-bundle + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus-operator + +.PHONY: destroy-prometheus-operator +destroy-prometheus-operator: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy-prometheus-operator + +.PHONY: deploy-grafana-tempo +deploy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy_grafana_tempo && expose_grafana + +.PHONY: destroy-grafana-tempo +destroy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_grafana && destroy_grafana_tempo + +.PHONY: deploy-prometheus +deploy-prometheus: deploy-prometheus-operator + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus && expose_prometheus + +.PHONY: destroy-prometheus +destroy-prometheus: + -source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_prometheus && destroy-prometheus + make destroy-prometheus-operator + +.PHONY: deploy-telemetry-stack +deploy-telemetry-stack: deploy-prometheus deploy-grafana-tempo + +.PHONY: destroy-telemetry-stack +destroy-telemetry-stack: destroy-prometheus destroy-grafana-tempo + +# Prometheus example app +.PHONY: deploy-prometheus-example-app +deploy-prometheus-example-app: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy-prometheus-example-app + +.PHONY: destroy-prometheus-example-app +destroy-prometheus-example-app: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy-prometheus-example-app + +.PHONY: deploy-grafana-tempo +deploy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy_grafana_tempo && expose_grafana + +.PHONY: destroy-grafana-tempo +destroy-grafana-tempo: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && unexpose_grafana && destroy_grafana_tempo + +# Tracing example app +.PHONY: deploy-tracing-example-app +deploy-tracing-example-app: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && deploy_tracing_example_app + +.PHONY: destroy-tracing-example-app +destroy-tracing-example-app: + source "${SCRIPTS_DIR}/${TELEMETRY_SCRIPT}" && destroy_tracing_example_app + +.PHONY: deploy-all +deploy-all: deploy-telemetry-stack deploy-prometheus-example-app deploy-tracing-example-app + +.PHONY: destroy-all +destroy-all: + # Note: - prefix ensures errors are ignored and continues + -${MAKE} destroy-telemetry-stack + -${MAKE} destroy-prometheus-example-app + -${MAKE} destroy-tracing-example-app + diff --git a/dev/telemetry/README.md b/dev/telemetry/README.md new file mode 100644 index 000000000..99f7b8591 --- /dev/null +++ b/dev/telemetry/README.md @@ -0,0 +1,22 @@ +# Setup Telemetry +This folder contains helper files to setup a grafana, tempo and prometheus instances locally. + + +## Commands +- Deploy prometheus + - `make deploy-prometheus` +- Deploy prometheus example app + - `make deploy-prometheus-example-app` +- Deploy grafana+tempo + - `make deploy-grafana-tempo` +- Deploy tracing example + - `make deploy-tracing-example-app` + +The reverse operations are prefixed with `destroy` (e.g. `destroy-grafana-tempo`) + +## Manual Test +- From `dev` folder deploy the network `make deploy-network`. +- From this folder run `make deploy-all` to deploy the telemetry stack (grafana + tempo + prometheus ) along with example apps. +- Browse `http://localhost:9090/tsdb-status` and ensure "Head Stats" metrics status are non-zero. +- Browse `http://localhost:3000/explore` and ensure tracing are visible. +- Run `make destroy-all` to clean up telemetry stack and example apps. diff --git a/dev/telemetry/grafana/dashboards/NMT.json b/dev/telemetry/grafana/dashboards/NMT.json new file mode 100644 index 000000000..e77d8ebd0 --- /dev/null +++ b/dev/telemetry/grafana/dashboards/NMT.json @@ -0,0 +1,742 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "description": "Plot displaying number of inotify events handled by NMT ICS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Total" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "nmt_ics_metrics_inotify_event_count", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{node_alias}} - Total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_inotify_event_hit_count", + "hide": false, + "legendFormat": "{{node_alias}} - Hits", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_inotify_event_miss_count", + "hide": false, + "legendFormat": "{{node_alias}} - Miss", + "range": true, + "refId": "C" + } + ], + "title": "Inotify Events", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "targetBlank": true, + "title": "View Trace", + "url": "http://localhost:3001/explore?left=%7B%22datasource%22%3A%22PD952A3C4CFED24BF%22%2C%22queries%22%3A%5B%7B%22query%22%3A%22${__data.fields.traceID}%22%2C%22queryType%22%3A%22traceql%22%7D%5D%7D" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "areaM2" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "limit": 20, + "queryType": "nativeSearch", + "refId": "A", + "serviceName": "nmt-ics", + "spanName": "processEvent" + } + ], + "title": "Process Event Traces", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "Start time" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_cmd_execution_count", + "legendFormat": "{{node_alias}} - Execution", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_cmd_execution_success_count", + "hide": false, + "legendFormat": "{{node_alias}} - Success", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_cmd_execution_error_count", + "hide": false, + "legendFormat": "{{node_alias}} - Error", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_internal_error_count", + "hide": false, + "legendFormat": "{{node_alias}} - Internal errors", + "range": true, + "refId": "D" + } + ], + "title": "Command Execution", + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Trace name" + }, + "properties": [ + { + "id": "custom.width", + "value": 139 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Start time" + }, + "properties": [ + { + "id": "custom.width", + "value": 164 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trace ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 312 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "limit": 1500, + "queryType": "nativeSearch", + "refId": "A", + "serviceName": "nmt-ics", + "spanName": "processEvent" + } + ], + "title": "Process Event Traces", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration(s)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"nmt_ics_metrics_alive_for_ms\", exported_job=\"nmt-ics\", instance=\"otel-collector:8889\", job=\"otel-collector\", label1=\"value1\", nmt_product=\"ics\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P38774DBC5AF30CBB" + }, + "editorMode": "builder", + "expr": "nmt_ics_metrics_alive_for_secs", + "legendFormat": "{{node_alias}} - Alive Duration(sec)", + "range": true, + "refId": "A" + } + ], + "title": "Heartbeats", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 20, + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "8.5.16", + "targets": [ + { + "datasource": { + "type": "tempo", + "uid": "PD952A3C4CFED24BF" + }, + "key": "Q-36b42c7f-8b26-4733-9665-0c976c6b15d9-0", + "limit": 1500, + "queryType": "nativeSearch", + "refId": "A", + "serviceName": "nmt-ics", + "spanName": "heartbeat" + } + ], + "title": "Heartbeat Traces", + "type": "table" + } + ], + "refresh": "5s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "NMT", + "uid": "ClaqZQb4z", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/dev/telemetry/grafana/dashboards/important-stats-prom.json b/dev/telemetry/grafana/dashboards/important-stats-prom.json new file mode 100644 index 000000000..ae365217b --- /dev/null +++ b/dev/telemetry/grafana/dashboards/important-stats-prom.json @@ -0,0 +1,2276 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 40, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Network Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "avg(platform_secC2C{environment=\"$Networks\", type=\"mean\"})", + "legendFormat": "{{__name__}} - {{ type }}", + "range": true, + "refId": "A" + } + ], + "title": "${Networks} TPS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.3.2-67a213dc85", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "avg(platform_secC2C{environment=\"$Networks\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "secC2C avg all", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_secC2C{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "SecC2C - Max", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(.*)(internalNodeWrites_per_s_smartContractKvStore)", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_secC2C{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "{{__name__}} - {{type}}", + "range": true, + "refId": "A" + } + ], + "title": "SecC2C Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_transH_per_sec{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "TransH_Per_sec Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_dupEvPercent{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "dupEvPercent Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_trans_per_sec{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "trans_per_sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_trans_per_event{environment=\"$Networks\", type=\"mean\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "trans_per_event", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_sigStateHash{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "sigStateHash Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "app_writeQueueSizeRecordStream{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "writeQueueSizeRecordStream Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_conns{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "conns Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_cpuLoadSys{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_bytes_per_trans{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "bytes_per_trans Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_PlatformStatus{environment=\"$Networks\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "PlatformStatus Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_q4{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Q4 Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_rounds_per_sec{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "rounds_per_sec Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_memFree{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "memFree Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "platform_events_per_sec{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "events_per_sec Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_secTransH{environment=\"$Networks\", type=\"max\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "secTransH Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_sec_per_sync2{environment=\"$Networks\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "sec_per_sync2 Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 49, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_sec_per_sync4{environment=\"$Networks\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "sec_per_sync4 Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_sec_per_sync3{environment=\"$Networks\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "sec_per_sync3 Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 51, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "editorMode": "code", + "expr": "internal_sec_per_sync{environment=\"$Networks\"}", + "legendFormat": "node{{node}}", + "range": true, + "refId": "A" + } + ], + "title": "sec_per_sync Max", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "grafanacloud-swirldslabspreproduction-prom", + "value": "grafanacloud-swirldslabspreproduction-prom" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "/^.*-prom/", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "dev", + "value": "dev" + }, + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "definition": "label_values(environment)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Networks", + "options": [], + "query": { + "query": "label_values(environment)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "4", + "value": "4" + }, + "datasource": { + "type": "prometheus", + "uid": "${Datasource}" + }, + "definition": "label_values(node)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Node", + "options": [], + "query": { + "query": "label_values(node)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Node Software Important (Prometheus)", + "uid": "node-software-important-prom", + "version": 6, + "weekStart": "" +} diff --git a/dev/telemetry/grafana/example-tracing-app.yaml b/dev/telemetry/grafana/example-tracing-app.yaml new file mode 100644 index 000000000..7231b25e2 --- /dev/null +++ b/dev/telemetry/grafana/example-tracing-app.yaml @@ -0,0 +1,45 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + ingress.kubernetes.io/ssl-redirect: "false" + name: ingress + namespace: default +spec: + rules: + - http: + paths: + - backend: + service: + name: grafana + port: + number: 80 + path: / + pathType: Prefix +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: xk6-tracing + namespace: default +spec: + minReadySeconds: 10 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: xk6-tracing + name: xk6-tracing + template: + metadata: + labels: + app: xk6-tracing + name: xk6-tracing + spec: + containers: + - env: + - name: ENDPOINT + value: tempo:4317 + image: ghcr.io/grafana/xk6-client-tracing:v0.0.2 + imagePullPolicy: IfNotPresent + name: xk6-tracing \ No newline at end of file diff --git a/dev/telemetry/grafana/grafana-values.yaml b/dev/telemetry/grafana/grafana-values.yaml new file mode 100644 index 000000000..89444b8f0 --- /dev/null +++ b/dev/telemetry/grafana/grafana-values.yaml @@ -0,0 +1,32 @@ +env: + GF_AUTH_ANONYMOUS_ENABLED: true + GF_AUTH_ANONYMOUS_ORG_ROLE: 'Admin' + GF_AUTH_DISABLE_LOGIN_FORM: true + +datasources: + datasources.yaml: + apiVersion: 1 + + datasources: + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3100 + basicAuth: false + isDefault: true + version: 1 + editable: false + apiVersion: 1 + uid: tempo + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false + apiVersion: 1 + uid: prometheus diff --git a/dev/telemetry/prometheus/README.md b/dev/telemetry/prometheus/README.md deleted file mode 100644 index 77ae1e957..000000000 --- a/dev/telemetry/prometheus/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Setup Prometheus -This folder contains helper files to setup a prometheus instance locally. - -## Commands -- Deploy prometheus operator - - `make deploy-prometheus-operator` -- Deploy prometheus - - `make deploy-prometheus` -- Deploy prometheus example app - - `make deploy-prometheus-example-app` -- Deploy all - `make deploy-all` -- Destroy all - `make destroy-all` - -## Manual Test -- From `dev` folder deploy the network `make deploy-network` -- From this folder run `make deploy-all` -- export prometheus svc port `kubectl port-forward svc/prometheus 9090:9090` -- browse `http://localhost:9090/tsdb-status` and ensure status are non-zero diff --git a/dev/telemetry/prometheus/example-app.yaml b/dev/telemetry/prometheus/example-app.yaml index 93b5cd4b6..3d6c84e6d 100644 --- a/dev/telemetry/prometheus/example-app.yaml +++ b/dev/telemetry/prometheus/example-app.yaml @@ -16,7 +16,7 @@ spec: - name: prometheus-example-app image: fabxc/instrumented_app ports: - - name: otel-metrics # use same name as used by otel-collector sidecar in hedera-network network-node + - name: app-port containerPort: 8080 --- kind: Service @@ -24,11 +24,11 @@ apiVersion: v1 metadata: name: prometheus-example-app labels: - # use the same label used by hedera-network network-node-svc + # use the same label used in network-node-svc.yaml fullstack.hedera.com/type: network-node-svc spec: selector: app: prometheus-example-app ports: - - name: otel-metrics + - name: prometheus # use same prometheus port-name as in network-node-svc.yaml port: 8080