Merge branch 'dev' into new-changes

* dev: Use helmfile layering for more flexibility Minor fixes and improvments Added Confluent helm charts as submodule
RADAR-base · Sep 2, 2020 · a3568f3 · a3568f3
2 parents 855e582 + 2978641
commit a3568f3
Show file tree

Hide file tree

Showing 18 changed files with 73 additions and 113 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,4 @@
 /.env
-cp-helm-charts/
 kubernetes-HDFS/
 keystore.p12
 radar-is.yml
@@ -8,3 +7,4 @@ production.yaml
 .idea/
 RADAR-Kubernetes.iml
 postgreskeystore.jks
+environments.yaml
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "cp-helm-charts"]
+	path = cp-helm-charts
+	url = https://github.com/RADAR-base/cp-helm-charts
diff --git a/README.md b/README.md
@@ -1,9 +1,6 @@
 # RADAR-Kubernetes [![Build Status](https://travis-ci.org/RADAR-base/RADAR-Kubernetes.svg?branch=master)](https://travis-ci.org/RADAR-base/RADAR-Kubernetes)
 Kubernetes deployment of RADAR-base.
 
-**Note:**
-This repository is still in **alpha** stage and it's not ready for production use.
-
 
 ## Installation
 You need to have a working Kubernetes installation and there are 3 ways to have that:
@@ -22,7 +19,7 @@ You need to have a working Kubernetes installation and there are 3 ways to have
   * [Minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/)
   * [K3S](https://k3s.io/)
 
-**Note 1:** This setup is currently only tested on [AWS EKS](https://aws.amazon.com/eks/) however because of cloud agnostic approach of Kubernetes you should be able install this stack on any Kubernetes installation. Also the idea behind using `helm` and `helmfile` has been allowing more complex and customized setups without too much change in the original repository, so if current approach isn't working in your environment you can easily change components to your needs.
+**Note 1:** This setup is currently only tested on AWS EKS, OpenStack Magnum and Azure ASK however because of cloud agnostic approach of Kubernetes you should be able install this stack on any Kubernetes installation. Also the idea behind using `helm` and `helmfile` has been allowing more complex and customized setups without too much change in the original repository, so if current approach isn't working in your environment you can easily change components to your needs.
 
 **Note 2:** If you're not using a cloud provider you need to make sure that you can [load balance](https://kubernetes.github.io/ingress-nginx/deploy/baremetal/) and [expose  applications](https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#exposing-the-service) and provide [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/).
 
@@ -36,9 +33,9 @@ You need to have following tools installed in your machine to install the stack:
 
 After installing them run following commands:
 ```shell
-git clone https://github.com/RADAR-base/RADAR-Kubernetes.git
+git clone --recurse-submodules https://github.com/RADAR-base/RADAR-Kubernetes.git
 cd RADAR-Kubernetes
-git clone https://github.com/RADAR-base/cp-helm-charts.git
+cp environments.yaml.tmpl environments.yaml
 cp base.yaml production.yaml
 vim production.yaml  # Change setup parameters and configurations
 ./bin/keystore-init
@@ -202,29 +199,3 @@ Alternatively you can forward SSH port to your local machine and connect locally
 kubectl port-forward svc/radar-output 2222:22
 ```
 Now you can use "127.0.0.1" as `host` and "2222" as the `port` to connect to SFTP server.
-
-
-
-
-# - name: rook
-#   chart: rook-release/rook-ceph
-#   version: v1.2.3
-#   namespace: rook-ceph
-#   wait: true
-#   installed: {{ .Values.rook._install }}
-#
-# - name: ceph
-#   chart: ../charts/ceph
-#   namespace: rook-ceph
-#   wait: true
-#   installed: {{ .Values.ceph._install }}
-#   values:
-#     - {{ .Values.ceph | toYaml | indent 8 | trim }}
-
-#
-# - name: sftp
-#   chart: ../charts/sftp
-#   wait: true
-#   installed: {{ .Values.sftp._install }}
-#   values:
-#     - {{ .Values.sftp | toYaml | indent 8 | trim }}
diff --git a/charts/cert-manager/requirements.lock b/charts/cert-manager/requirements.lock
@@ -2,5 +2,5 @@ dependencies:
 - name: cert-manager
   repository: https://charts.jetstack.io
   version: v0.13.0
-digest: sha256:286bd0a59060907dfadcadc5b8d6445635086132184b43f93a0eb9333fc1fc8b
-generated: "2020-07-09T16:52:09.217042516+02:00"
+digest: sha256:c7b4858b27dbfb9ac616c51a173f78bb13794d0c438fa1c62469caa814c842f2
+generated: "2020-09-02T02:53:56.322995185+02:00"
diff --git a/cp-helm-charts b/cp-helm-charts
diff --git a/environments.yaml.tmpl b/environments.yaml.tmpl
@@ -0,0 +1,5 @@
+environments:
+ default:
+   values:
+   - ../base.yaml
+   - ../production.yaml
diff --git a/helmfile.d/00-init.yaml b/helmfile.d/00-init.yaml
@@ -1,15 +1,14 @@
+bases:
+- ../environments.yaml
+
+---
+
 repositories:
 - name: stable
   url: https://kubernetes-charts.storage.googleapis.com
 - name: elastic
   url: https://helm.elastic.co
 
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
-
 helmDefaults:
   atomic: true
   # force: true

diff --git a/helmfile.d/10-base.yaml b/helmfile.d/10-base.yaml
@@ -1,10 +1,7 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../../etc/production.yaml
-    secrets:
-    - ../../secrets/production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true

diff --git a/helmfile.d/20-dashboard.yaml b/helmfile.d/20-dashboard.yaml
@@ -1,14 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: radar-dashboard
     chart: ../charts/radar-dashboard

diff --git a/helmfile.d/20-fitbit.yaml b/helmfile.d/20-fitbit.yaml
@@ -1,14 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: radar-fitbit-connector
     chart: ../charts/radar-fitbit-connector

diff --git a/helmfile.d/20-grafana.yaml b/helmfile.d/20-grafana.yaml
@@ -1,13 +1,12 @@
+bases:
+- ../environments.yaml
+
+---
+
 repositories:
   - name: stable
     url: https://kubernetes-charts.storage.googleapis.com
 
-environments:
-  default:
-    values:
-      - ../base.yaml
-      - ../production.yaml
-
 helmDefaults:
   atomic: true
   timeout: 600

diff --git a/helmfile.d/20-hdfs.yaml b/helmfile.d/20-hdfs.yaml
@@ -1,10 +1,7 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../../etc/production.yaml
-    secrets:
-    - ../../secrets/production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true

diff --git a/helmfile.d/20-ingestion.yaml b/helmfile.d/20-ingestion.yaml
@@ -1,16 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../../etc/production.yaml
-    secrets:
-    - ../../secrets/production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: cp-kafka-rest
     chart: ../cp-helm-charts/charts/cp-kafka-rest

diff --git a/helmfile.d/20-kafka-analysis.yaml b/helmfile.d/20-kafka-analysis.yaml
@@ -1,14 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: radar-backend-monitor
     chart: ../charts/radar-backend

diff --git a/helmfile.d/20-redcap.yaml b/helmfile.d/20-redcap.yaml
@@ -1,14 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: radar-integration
     chart: ../charts/radar-integration

diff --git a/helmfile.d/20-s3.yaml b/helmfile.d/20-s3.yaml
@@ -1,17 +1,14 @@
+bases:
+- ../environments.yaml
+
+---
+
 repositories:
 - name: stable
   url: https://kubernetes-charts.storage.googleapis.com
 - name: bitnami
   url: https://charts.bitnami.com/bitnami
 
-environments:
-  default:
-    values:
-      - ../base.yaml
-      - ../../etc/production.yaml
-    secrets:
-      - ../../secrets/production.yaml
-
 helmDefaults:
   atomic: false
   force: true

diff --git a/helmfile.d/20-upload.yaml b/helmfile.d/20-upload.yaml
@@ -1,14 +1,13 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: radar-upload-postgresql
     chart: stable/postgresql

diff --git a/helmfile.d/99-velero.yaml b/helmfile.d/99-velero.yaml
@@ -1,15 +1,14 @@
-environments:
-  default:
-    values:
-    - ../base.yaml
-    - ../production.yaml
+bases:
+- ../environments.yaml
+
+---
 
 helmDefaults:
   atomic: true
   force: true
   timeout: 600
   kubeContext: {{ .Values.kubeContext }}
-  
+
 releases:
   - name: velero
     chart: ../charts/velero