feat: add minio tenant helm chart conditionally

charts/hedera-network/Chart.yaml

@@ -25,3 +25,12 @@ appVersion: "0.6.0"
 
 # This is range of versions of Kubernetes server that is supported by this chart.
 kubeVersion: ">=1.25.0"
+
+# helm dependency update
+dependencies:
+  - name: tenant
+    alias: minio-server
+    version: 5.0.7
+    repository: https://operator.min.io/
+    # TODO:  is this a good enough name, if we want to group enable/disable the uploader sidecars as a group
+    condition: cloud.minio.enable

charts/hedera-network/templates/_helpers.tpl

@@ -32,3 +32,8 @@ privileged: true
 {{- $tag := default .defaults.image.tag (.image).tag | default .Chart.AppVersion -}}
 {{ $reg }}/{{ $repo }}:{{ $tag }}
 {{- end }}
+
+{{- define "minio.configEnv" -}}
+export MINIO_ROOT_USER={{ include "minio.accessKey" . }}
+export MINIO_ROOT_PASSWORD={{ include "minio.secretKey" . }}
+{{- end -}}

charts/hedera-network/templates/network-node-statefulset.yaml

@@ -42,7 +42,7 @@ spec:
       - name: init-container
         image: {{ include "fullstack.container.image" (dict "image" $root.image "Chart" $.Chart "defaults" $root) }}
         imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $root.image "defaults" $root) }}
-        command: [ "/bin/sh", "-c", "mkdir /opt/hgcapp/hedera-backups /opt/hgcapp/accountbalance /opt/hgcapp/events /opt/hgcapp/recordstream /opt/hgcapp/sidecar /opt/hgcapp/recordstream/uploader-stats/"  ]
+        command: [ "/bin/sh", "-c", "mkdir -p /opt/hgcapp/hedera-backups /opt/hgcapp/accountBalances /opt/hgcapp/events /opt/hgcapp/recordStreams /opt/hgcapp/sidecar /opt/hgcapp/recordStreams/uploader-stats/" ]
         securityContext:
           {{- include "fullstack.hedera.security.context" . | nindent 10 }}
         volumeMounts:
@@ -74,6 +74,8 @@ spec:
         "otelCollector" $otelCollector
         "cloud" $.Values.cloud
         "chart" $.Chart
+        "node" $node
         "defaults" $defaults
+        "minioserver" (index $.Values "minio-server")
         ) | nindent 6 }}
 {{ end }}

charts/hedera-network/templates/secrets/uploder-mirror-secrets.yaml

@@ -1,10 +1,20 @@
+{{- $minio_accessKey := randAlpha 10  -}}
+{{- $minio_secretKey := randAlpha 10  -}}
+{{- $minio_config_env := printf "export MINIO_ROOT_USER=%s\nexport MINIO_ROOT_PASSWORD=%s" $minio_accessKey $minio_secretKey -}}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secrets
+type: Opaque
+data:
+  config.env: {{ $minio_config_env | b64enc }}
+---
 apiVersion: v1
 kind: Secret
 metadata:
   name: uploader-mirror-secrets
 type: Opaque
 data:
-  S3_ACCESS_KEY: ""
-  S3_SECRET_KEY: ""
-  GCS_ACCESS_KEY: ""
-  GCS_SECRET_KEY: ""
+  S3_ACCESS_KEY: {{ $minio_accessKey | b64enc }}
+  S3_SECRET_KEY: {{ $minio_secretKey | b64enc }}

charts/hedera-network/templates/sidecars/_account-balance-uploader.tpl

@@ -3,6 +3,8 @@
 {{- $defaults := .defaults | required "context must include 'defaults'!" }}
 {{- $cloud := .cloud | required "context must include 'cloud'!" -}}
 {{- $chart := .chart | required "context must include 'chart'!" -}}
+{{- $nodeId := .nodeId -}}
+{{- $minioserver := .minioserver -}}
 - name: {{ default "account-balance-uploader" $balanceUploader.nameOverride }}
   image: {{ include "fullstack.container.image" (dict "image" $balanceUploader.image "Chart" $chart "defaults" $defaults ) }}
   imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $balanceUploader.image "defaults" $defaults) }}
@@ -14,10 +16,13 @@
     - /usr/local/bin/mirror.py
     - --linux
     - --watch-directory
-    - /opt/hgcapp/accountbalance
+    - /opt/hgcapp/accountBalances
+    - --s3-endpoint
+    - http://{{ $minioserver.tenant.name }}-hl:9000
   volumeMounts:
     - name: hgcapp-storage
-      mountPath: /opt/hgcapp/
+      mountPath: /opt/hgcapp/accountBalances
+      subPath: accountBalances/balance{{ $nodeId }}
   env:
     - name: DEBUG
       value: {{ default $defaults.config.debug ($balanceUploader.config).debug | quote }}
@@ -42,13 +47,13 @@
     - name: SIG_PRIORITIZE
       value: {{ default $defaults.config.signature.prioritize (($balanceUploader.config).signature).prioritize | quote }}
     - name: BUCKET_PATH
-      value: "/accountbalance"
+      value: "accountbalance"
     - name: BUCKET_NAME
       value: {{ $cloud.buckets.streamBucket | quote }}
     - name: S3_ENABLE
-      value: {{ $cloud.s3.enable | quote }}
+      value: "true"
     - name: GCS_ENABLE
-      value: {{ $cloud.gcs.enable | quote }}
+      value: "false"
   envFrom:
     - secretRef:
         name: uploader-mirror-secrets

charts/hedera-network/templates/sidecars/_event-stream-uploader.tpl

@@ -3,6 +3,8 @@
 {{- $defaults := .defaults | required "context must include 'defaults'!" }}
 {{- $cloud := .cloud | required "context must include 'cloud'!" -}}
 {{- $chart := .chart | required "context must include 'chart'!" -}}
+{{- $nodeId := .nodeId -}}
+{{- $minioserver := .minioserver -}}
 - name: {{ default "event-stream-uploader" $eventStream.nameOverride }}
   image: {{ include "fullstack.container.image" (dict "image" $eventStream.image "Chart" $chart "defaults" $defaults) }}
   imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $eventStream.image "defaults" $defaults) }}
@@ -15,9 +17,13 @@
     - --linux
     - --watch-directory
     - /opt/hgcapp/events
+    - --debug
+    - --s3-endpoint
+    - http://{{ $minioserver.tenant.name }}-hl:9000
   volumeMounts:
     - name: hgcapp-storage
-      mountPath: /opt/hgcapp/
+      mountPath: /opt/hgcapp/events
+      subPath: events/balance{{ $nodeId }}
   env:
     - name: DEBUG
       value: {{ default $defaults.config.debug ($eventStream.config).debug | quote}}
@@ -46,9 +52,9 @@
     - name: BUCKET_NAME
       value: {{ $cloud.buckets.streamBucket | quote }}
     - name: S3_ENABLE
-      value: {{ $cloud.s3.enable | quote }}
+      value: "true"
     - name: GCS_ENABLE
-      value: {{ $cloud.gcs.enable | quote }}
+      value: "false"
   envFrom:
     - secretRef:
         name: uploader-mirror-secrets

charts/hedera-network/templates/sidecars/_record-stream-uploader.tpl

@@ -3,6 +3,8 @@
 {{- $defaults := .defaults | required "context must include 'defaults'!" }}
 {{- $cloud := .cloud | required "context must include 'cloud'!" -}}
 {{- $chart := .chart | required "context must include 'chart'!" -}}
+{{- $minioserver := .minioserver -}}
+{{- $nodeId := .nodeId -}}
 - name: {{ default "record-stream-uploader" $recordStream.nameOverride }}
   image: {{ include "fullstack.container.image" (dict "image" $recordStream.image "Chart" $chart "defaults" $defaults) }}
   imagePullPolicy: {{ include "fullstack.images.pullPolicy" (dict "image" $recordStream.image "defaults" $defaults) }}
@@ -14,12 +16,15 @@
     - /usr/local/bin/mirror.py
     - --linux
     - --watch-directory
-    - /opt/hgcapp/recordstream
+    - /opt/hgcapp/recordStreams
     - --csv-stats-directory
-    - /opt/hgcapp/recordstream/uploader-stats
+    - /opt/hgcapp/recordStreams/uploader-stats
+    - --s3-endpoint
+    - http://{{ $minioserver.tenant.name }}-hl:9000
   volumeMounts:
     - name: hgcapp-storage
-      mountPath: /opt/hgcapp/
+      mountPath: /opt/hgcapp/recordStreams
+      subPath: recordStreams/record{{ $nodeId }}
   env:
     - name: DEBUG
       value: {{ default $defaults.config.debug ($recordStream.config).debug | quote }}
@@ -38,7 +43,7 @@
     - name: STREAM_EXTENSION
       value: {{ default $defaults.config.compression ($recordStream.config).compression | eq "true" | ternary "rcd.gz" "rcd" | quote }}
     - name: SIG_EXTENSION
-      value: {{ default $defaults.config.compression ($recordStream.config).compression | eq "true" | ternary "rcd_sig.gz" "rcd_sig" | quote }}
+      value: "rcd_sig"
     - name: RECORD_STREAM_COMPRESSION
       value: {{ default $defaults.config.compression $recordStream.config.compression | quote }}
     - name: RECORD_STREAM_SIDECAR
@@ -48,13 +53,13 @@
     - name: SIG_PRIORITIZE
       value: {{ default $defaults.config.signature.prioritize (($recordStream.config).signature).prioritize | quote }}
     - name: BUCKET_PATH
-      value: "/recordstream"
+      value: "recordstreams"
     - name: BUCKET_NAME
       value: {{ $cloud.buckets.streamBucket | quote }}
     - name: S3_ENABLE
-      value: {{ $cloud.s3.enable | quote }}
+      value: "true"
     - name: GCS_ENABLE
-      value: {{ $cloud.gcs.enable | quote }}
+      value: "false"
   envFrom:
     - secretRef:
         name: uploader-mirror-secrets

charts/hedera-network/templates/sidecars/_sidecars.tpl

@@ -7,19 +7,21 @@
 {{- $otelCollector := .otelCollector | required "context must include 'otelCollector'!" -}}
 {{- $cloud := .cloud | required "context must include 'cloud'!" -}}
 {{- $chart := .chart | required "context must include 'chart'!" -}}
+{{- $nodeId := .node.accountId -}}
+{{- $minioserver := .minioserver -}}
   {{- if default $defaults.sidecars.recordStreamUploader.enable  $recordStream.enable | eq "true" }}
   # Sidecar: Record Stream Uploader
-  {{- $data := dict "recordStream" $recordStream "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.recordStreamUploader -}}
+  {{- $data := dict "recordStream" $recordStream "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.recordStreamUploader "nodeId" $nodeId "minioserver" $minioserver -}}
   {{ include "fullstack.sidecars.recordStreamUploader" $data | nindent 0 }}
   {{- end }}
   {{- if default $defaults.sidecars.eventStreamUploader.enable  $eventStream.enable | eq "true" }}
   # Sidecar: Event Stream Uploader
-  {{- $data := dict "eventStream" $eventStream "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.eventStreamUploader -}}
+  {{- $data := dict "eventStream" $eventStream "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.eventStreamUploader  "nodeId" $nodeId "minioserver" $minioserver -}}
   {{ include "fullstack.sidecars.eventStreamUploader" $data | nindent 0 }}
   {{- end }}
   {{- if default $defaults.sidecars.accountBalanceUploader.enable $balanceUploader.enable | eq "true" }}
   # Sidecar: Account Balance Uploader
-  {{- $data := dict "balanceUploader" $balanceUploader "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.accountBalanceUploader -}}
+  {{- $data := dict "balanceUploader" $balanceUploader "cloud" $cloud "chart" $chart "defaults" $defaults.sidecars.accountBalanceUploader  "nodeId" $nodeId "minioserver" $minioserver -}}
   {{ include "fullstack.sidecars.accountBalanceUploader" $data | nindent 0 }}
   {{- end }}
   {{- if default $defaults.sidecars.backupUploader.enable  $backupUploader.enable | eq "true" }}

charts/hedera-network/values.yaml

@@ -2,13 +2,11 @@
 
 # cloud configuration
 cloud:
+  minio:
+    enable: true
   buckets:
     streamBucket: "fst-streams"
     backupBucket: "fst-backups"
-  s3:
-    enable: "true"
-  gcs:
-    enable: "true"
 
 # reduce default termination grace period
 terminationGracePeriodSeconds: 10
@@ -162,10 +160,34 @@ defaults:
           port: healthcheck
       resources: {}
 
+# This configures the minio tenant subchart
+# Reference for configuration: https://github.com/minio/operator/blob/master/helm/tenant/values.yaml
+minio-server:
+  secrets:
+    # This secret has [accessKey, secretKey] and will be randomly generated by helm
+    existingSecret: minio-secrets
+  tenant:
+    buckets:
+      - name: fst-streams
+      - name: fst-backups
+    name: minio
+    pools:
+      - servers: 1
+        name: pool-1
+        volumesPerServer: 1
+        size: 10Gi
+    configuration:
+      name: minio-secrets
+    certificate:
+      requestAutoCert: false
+
 # hedera node configuration
 # Only the name of the node is required. The rest of the configuration will be inherited from `defaults` section
 hedera:
   nodes:
     - name: node0
+      accountId: 0.0.3
     - name: node1
+      accountId: 0.0.4
     - name: node2
+      accountId: 0.0.5

dev/Makefile

@@ -28,6 +28,10 @@ setup:
 setup-cluster:
 	kind create cluster -n fst
 
+.PHONY: update-helm-dependencies
+update-helm-dependencies:
+	helm dependency update ../charts/hedera-network
+
 .PHONY: deploy-chart
 deploy-chart:
 	echo ">> Deploying helm chart..." && \
@@ -78,7 +82,7 @@ test:
 		$(MAKE) destroy-network
 	}
 	trap cleanup_test EXIT # always destroy-network on exit
-	$(MAKE) setup deploy-network helm-test setup-nodes start-nodes
+	$(MAKE) setup deploy-minio-operator-if-required update-helm-dependencies deploy-network helm-test setup-nodes start-nodes
 
 .PHONY: setup-nodes
 setup-nodes: setup
@@ -116,11 +120,34 @@ run-func:
 	source "${SCRIPTS_DIR}/${SCRIPT_NAME}" && ${FUNC}
 
 .PHONY: start
-start: deploy-network setup-nodes start-nodes
+start: deploy-minio-operator-if-required update-helm-dependencies deploy-network setup-nodes start-nodes
 
 .PHONY: restart
 restart: stop-nodes start-nodes
 
 .PHONY: reset
 reset: destroy-network start
 
+.PHONY: deploy-minio-operator
+deploy-minio-operator:
+	@echo ">> Deploying minio operator..."; \
+	helm install --repo https://operator.min.io/ --namespace=minio-operator --create-namespace --version 5.0.7 minio-operator operator && \
+	echo ">> Waiting for minio operator to be ready..." && \
+	kubectl --namespace=minio-operator  wait --for=condition=available --timeout=600s deployment.apps/minio-operator && \
+	sleep 5
+
+.PHONY: is-minio-operator-installed
+is-minio-operator-installed:
+	@echo ">> Checking for minio operator..."; \
+	helm list --namespace=minio-operator | grep minio-operator &> /dev/null && { echo "Found minio operator."; exit 0; }; \
+	echo "Minio operator not found."; exit 1
+
+.PHONY: deploy-minio-operator-if-required
+deploy-minio-operator-if-required:
+	@$(MAKE) is-minio-operator-installed || $(MAKE) deploy-minio-operator
+
+
+.PHONY: undeploy-minio-operator
+undeploy-minio-operator:
+	echo ">> Deploying minio operator..." && \
+	helm delete  --namespace=minio-operator minio-operator