From 5eba40212a8e0a4fd4d18fe873fc618d0f05545c Mon Sep 17 00:00:00 2001 From: "Roberto C. Morano" Date: Tue, 23 Aug 2022 18:20:44 +0200 Subject: [PATCH] feat(testdrive): added cardano2dgraph simple example --- testdrive/cardano2dgraph/README.md | 45 +++++ testdrive/cardano2dgraph/k8s/dgraph.yaml | 99 ++++++++++ .../k8s/kustomize-haproxy/.gitignore | 1 + .../k8s/kustomize-haproxy/helm-values.yaml | 11 ++ .../k8s/kustomize-haproxy/kustomization.yaml | 12 ++ testdrive/cardano2dgraph/k8s/oura.yaml | 101 +++++++++++ .../k8s/payload-transformer.yaml | 170 ++++++++++++++++++ testdrive/cardano2dgraph/skaffold.yaml | 24 +++ 8 files changed, 463 insertions(+) create mode 100644 testdrive/cardano2dgraph/README.md create mode 100644 testdrive/cardano2dgraph/k8s/dgraph.yaml create mode 100644 testdrive/cardano2dgraph/k8s/kustomize-haproxy/.gitignore create mode 100644 testdrive/cardano2dgraph/k8s/kustomize-haproxy/helm-values.yaml create mode 100644 testdrive/cardano2dgraph/k8s/kustomize-haproxy/kustomization.yaml create mode 100644 testdrive/cardano2dgraph/k8s/oura.yaml create mode 100644 testdrive/cardano2dgraph/k8s/payload-transformer.yaml create mode 100644 testdrive/cardano2dgraph/skaffold.yaml diff --git a/testdrive/cardano2dgraph/README.md b/testdrive/cardano2dgraph/README.md new file mode 100644 index 00000000..ea2d6642 --- /dev/null +++ b/testdrive/cardano2dgraph/README.md @@ -0,0 +1,45 @@ +# Cardano => dgraph Testdrive + +## Introduction + +This is a reference implementation to show how _Oura_ can be leveradged to read from a Cardano relay node into an output events via HTTP to a remote endpoint, in this case a dgraph instance passing through a payload transformer that makes oura events fit dgraph expected payload. + +Note the payload transformer [(comcast/eel)](https://github.com/Comcast/eel) runs multiple replicas (load balanced using haproxy) to achieve higher throughput, and although it manages its own buffer and handle failed requests to dgraph and tries to resubmit them, events can be lost in case any of these replicas are unexpectedly restarted. + +## Prerequisites + +- K8s Cluster +- kubectl +- Skaffold +- kustomize + +## Deployment + +Create a k8s namespace for the testdrive: + +``` +kubectl create namespace cardano2dgraph +``` + +Deploy the resources: + +``` +skaffold run --namespace cardano2dgraph +``` + +Check events going thru: +``` +kubectl logs -n cardano2dgraph -f -l app=oura-2-dgraph-etl +``` + +## Access dgraph + +Expose `dgraph` api (will sit in http://localhost:8081): +``` +kubectl port-forward -n cardano2dgraph svc/dgraph-public 8081:8080 +``` + +Optionally, use [dgraph/ratel](https://github.com/dgraph-io/ratel) webui to access `dgraph` (access http://localhost:8001 and use http://localhost:8081 as dgraph endpoint): +``` +docker run --rm -d -p 8001:8000 dgraph/ratel:latest +``` diff --git a/testdrive/cardano2dgraph/k8s/dgraph.yaml b/testdrive/cardano2dgraph/k8s/dgraph.yaml new file mode 100644 index 00000000..3eac4351 --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/dgraph.yaml @@ -0,0 +1,99 @@ +# This is the service that should be used by the clients of Dgraph to talk to the cluster. +apiVersion: v1 +kind: Service +metadata: + name: dgraph-public + labels: + app: dgraph +spec: + type: ClusterIP + ports: + - port: 5080 + targetPort: 5080 + name: grpc-zero + - port: 6080 + targetPort: 6080 + name: http-zero + - port: 8080 + targetPort: 8080 + name: http-alpha + - port: 9080 + targetPort: 9080 + name: grpc-alpha + selector: + app: dgraph +--- +# This StatefulSet runs 1 pod with one Zero container and one Alpha container. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: dgraph +spec: + serviceName: "dgraph" + replicas: 1 + selector: + matchLabels: + app: dgraph + template: + metadata: + labels: + app: dgraph + spec: + containers: + - name: zero + image: dgraph/dgraph:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5080 + name: grpc-zero + - containerPort: 6080 + name: http-zero + volumeMounts: + - name: datadir + mountPath: /dgraph + command: + - bash + - "-c" + - | + set -ex + dgraph zero --survive filesystem --my=$(hostname -f):5080 + - name: alpha + image: dgraph/dgraph:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + name: http-alpha + - containerPort: 9080 + name: grpc-alpha + volumeMounts: + - name: datadir + mountPath: /dgraph + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - bash + - "-c" + - | + set -ex + dgraph alpha --survive filesystem --my=$(hostname -f):7080 --zero dgraph-0.dgraph.${POD_NAMESPACE}.svc.cluster.local:5080 --security whitelist=0.0.0.0/0 + terminationGracePeriodSeconds: 60 + volumes: + - name: datadir + persistentVolumeClaim: + claimName: datadir + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 5Gi diff --git a/testdrive/cardano2dgraph/k8s/kustomize-haproxy/.gitignore b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/.gitignore new file mode 100644 index 00000000..ebf1d3dc --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/.gitignore @@ -0,0 +1 @@ +charts diff --git a/testdrive/cardano2dgraph/k8s/kustomize-haproxy/helm-values.yaml b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/helm-values.yaml new file mode 100644 index 00000000..96c96a80 --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/helm-values.yaml @@ -0,0 +1,11 @@ +controller: + extraArgs: + - --namespace-whitelist=cardano2dgraph + service: + type: ClusterIP + enablePorts: + http: true + https: false + stat: false +defaultBackend: + enabled: false diff --git a/testdrive/cardano2dgraph/k8s/kustomize-haproxy/kustomization.yaml b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/kustomization.yaml new file mode 100644 index 00000000..83b19e56 --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/kustomize-haproxy/kustomization.yaml @@ -0,0 +1,12 @@ +helmCharts: +- name: kubernetes-ingress + repo: https://haproxytech.github.io/helm-charts + version: 1.21.1 + releaseName: init0 + namespace: cardano2dgraph + valuesFile: helm-values.yaml + +#resources: +#- ../dgraph.yaml +#- ../oura.yaml +#- ../payload-transformer.yaml diff --git a/testdrive/cardano2dgraph/k8s/oura.yaml b/testdrive/cardano2dgraph/k8s/oura.yaml new file mode 100644 index 00000000..3dfe8c14 --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/oura.yaml @@ -0,0 +1,101 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: oura +data: + disabled: |- + [source.intersect] + type = "Tip" + + [source.intersect] + type = "Origin" + + [source.intersect] + type = "Fallbacks" + value = [ + [4492799, "f8084c61b6a238acec985b59310b6ecec49c0ab8352249afd7268da5cff2a457"] + [4490688, "aa83acbf5904c0edfe4d79b3689d3d00fcfc553cf360fd2229b98d464c28e9de"] # epoch 208 first slot + [4490687, "f8084c61b6a238acec985b59310b6ecec49c0ab8352249afd7268da5cff2a457"] # epoch 207 last slot + ] + + daemon.toml: |- + [source] + type = "N2N" + address = ["Tcp", "europe.relays-new.cardano-mainnet.iohk.io:3001"] + magic = "mainnet" + + [source.intersect] + type = "Origin" + + [source.mapper] + include_transaction_details = true + + [[filters]] + type = "Fingerprint" + + [sink] + type = "Webhook" + url = "http://init0-kubernetes-ingress/v1/events" + timeout = 3000 + max_retries = 30 + backoff_delay = 5000 + [sink.headers] + Host = "oura-2-dgraph-etl.local" + + [cursor] + type = "File" + path = "/var/oura/cursor" + + [metrics] + address = "0.0.0.0:9186" + endpoint = "/metrics" + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: oura +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oura + labels: + app: oura +spec: + replicas: 1 + selector: + matchLabels: + app: oura + template: + metadata: + labels: + app: oura + spec: + containers: + - name: main + image: ghcr.io/txpipe/oura:v1.6.0 + env: + - name: "RUST_LOG" + value: "warn" + args: + - "daemon" + volumeMounts: + - mountPath: /etc/oura + name: oura-config + - mountPath: /var/oura + name: oura-var + volumes: + - name: oura-config + configMap: + name: oura + - name: oura-var + persistentVolumeClaim: + claimName: oura diff --git a/testdrive/cardano2dgraph/k8s/payload-transformer.yaml b/testdrive/cardano2dgraph/k8s/payload-transformer.yaml new file mode 100644 index 00000000..ee667e65 --- /dev/null +++ b/testdrive/cardano2dgraph/k8s/payload-transformer.yaml @@ -0,0 +1,170 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: handlers +data: + default.json: |- + { + "Version": "1.0", + "Name": "oura-to-dgraph", + "Info": "Transforms and forwards oura's webhook events/payload to dgraph", + "Active": true, + "Match": null, + "IsMatchByExample": false, + "TerminateOnMatch": true, + "Transformation": { + "{{/set}}": "{{/}}" + }, + "IsTransformationByExample": false, + "Path": "", + "Verb": "POST", + "Protocol": "http", + "Endpoint": "http://dgraph-public:8080/mutate?commitNow=true" + } + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config +data: + config.json: |- + { + "Name": "Default", + "AppName": "eel", + "Endpoint": "http://localhost:8088", + "MaxAttempts": 30, + "InitialDelay": 30, + "InitialBackoff": 999, + "Pad": 0, + "BackoffMethod": "Exponential", + "MaxMessageSize": 5120000, + "MessageQueueTimeout": 1000, + "MessageQueueDepth": 500, + "WorkerPoolSize": { + "": 150 + }, + "HttpTimeout": 1500, + "ResponseHeaderTimeout": 1000, + "MaxIdleConnsPerHost": 100, + "DuplicateTimeout": 0, + "HttpTransactionHeader": "X-B3-TraceId", + "HttpTenantHeader": "Xrs-Tenant-Id", + "HttpPartnerHeader": "Partner-Id", + "HttpDebugHeader": "X-Debug", + "CustomProperties": { + "key": "value" + }, + "LogStats": false, + "TopicPath": "{{/topic}}", + "CloseIdleConnectionIntervalSec": 0, + "Version": "1.0", + "HandlerConfigPath": "", + "TraceLogParams": { + "Active": false, + "FileName": "event.trace.log", + "LogIncoming": true, + "LogOutgoing": false, + "LogParams": { + "payload": "{{/}}" + } + }, + "AllowPartner": false, + "DefaultPartner": "" + } + + plugins.json: |- + [ + { + "Type" : "WEBHOOK", + "Name" : "WEBHOOK", + "AutoStart" : true, + "RestartOk": false, + "ExitOnErr": true, + "Parameters" : { + "EventPort": 8080, + "EventProxyPath": "/v1/events", + "EventProcPath": "/v1/sync/events" + } + }, + { + "Type" : "STDIN", + "Name" : "STDIN", + "AutoStart" : false, + "RestartOk": false, + "ExitOnErr": false, + "Parameters" : {} + } + ] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oura-2-dgraph-etl + labels: + app: oura-2-dgraph-etl +spec: + replicas: 5 + selector: + matchLabels: + app: oura-2-dgraph-etl + template: + metadata: + labels: + app: oura-2-dgraph-etl + spec: + containers: + - name: main + image: gimbalabs/eel:latest + command: ["/bin/sh", "-c", "mkdir -p /config/config-handlers/tenant1 && cp /handlers/default.json /config/config-handlers/tenant1 && ./bin/eel ${EEL_PARAMS} | grep after_transformation"] + env: + - name: EEL_PARAMS + value: "-loglevel debug -path /config" + ports: + - containerPort: 8080 + resources: {} + volumeMounts: + - mountPath: /config/config-eel + name: config + - mountPath: /handlers + name: handlers + volumes: + - name: config + configMap: + name: config + - name: handlers + configMap: + name: handlers +--- +apiVersion: v1 +kind: Service +metadata: + name: oura-2-dgraph-etl +spec: + selector: + app: oura-2-dgraph-etl + ports: + - protocol: TCP + port: 5000 + targetPort: 8080 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: oura-2-dgraph-etl + annotations: + kubernetes.io/ingress.class: "haproxy" +spec: + rules: + - host: oura-2-dgraph-etl.local + http: + paths: + - backend: + service: + name: oura-2-dgraph-etl + port: + number: 5000 + path: / + pathType: Prefix diff --git a/testdrive/cardano2dgraph/skaffold.yaml b/testdrive/cardano2dgraph/skaffold.yaml new file mode 100644 index 00000000..3a07f28f --- /dev/null +++ b/testdrive/cardano2dgraph/skaffold.yaml @@ -0,0 +1,24 @@ +apiVersion: skaffold/v2beta26 +kind: Config +metadata: + name: endpoint +deploy: + kubectl: + manifests: + - k8s/payload-transformer.yaml + - k8s/dgraph.yaml + kustomize: + buildArgs: ["--enable-helm"] + paths: + - k8s/kustomize-haproxy +--- +apiVersion: skaffold/v2beta26 +kind: Config +metadata: + name: oura +requires: + - configs: [elastic] +deploy: + kubectl: + manifests: + - k8s/oura.yaml