Merge pull request #170 from replicatedhq/laverya/integration-testing

integration testing

.github/workflows/build-test.yaml

@@ -8,7 +8,7 @@ on:
 
 jobs:
 
-  build:
+  unit-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -21,5 +21,27 @@ jobs:
       - name: Test
         run: make test
 
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.19
+
       - name: Build
         run: make build
+
+      - name: Create k8s Kind Cluster
+        uses: helm/[email protected]
+
+      - name: Create Testdata in Cluster
+        run: ./testing/init.sh
+
+      - name: Run PVMigrate
+        run: ./bin/pvmigrate --source-sc int-source --dest-sc int-dest
+
+      - name: Validate Cluster End State
+        run: ./testing/validate.sh

.github/workflows/release.yaml

@@ -7,7 +7,7 @@ on:
 
 jobs:
 
-  test:
+  unit-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -20,11 +20,36 @@ jobs:
       - name: Test
         run: make test
 
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.19
+
+      - name: Build
+        run: make build
+
+      - name: Create k8s Kind Cluster
+        uses: helm/[email protected]
+
+      - name: Create Testdata in Cluster
+        run: ./testing/init.sh
+
+      - name: Run PVMigrate
+        run: ./bin/pvmigrate --source-sc int-source --dest-sc int-dest
+
+      - name: Validate Cluster End State
+        run: ./testing/validate.sh
 
   goreleaser:
     runs-on: ubuntu-latest
     needs:
-      - test
+      - unit-test
+      - integration-test
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout

testing/init.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+# this waits for a deployment to have all replicas up-to-date and available
+function deployment_fully_updated() {
+    x_fully_updated "$1" deployment "$2"
+}
+
+# this waits for a statefulset to have all replicas up-to-date and available
+function statefulset_fully_updated() {
+    x_fully_updated "$1" statefulset "$2"
+}
+
+# this waits for a resource type (deployment or statefulset) to have all replicas up-to-date and available
+function x_fully_updated() {
+    local namespace=$1
+    local resourcetype=$2
+    local name=$3
+
+    local desiredReplicas
+    desiredReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.replicas}')
+
+    local availableReplicas
+    availableReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.availableReplicas}')
+
+    local readyReplicas
+    readyReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.readyReplicas}')
+
+    local updatedReplicas
+    updatedReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.updatedReplicas}')
+
+    if [ "$desiredReplicas" != "$availableReplicas" ] ; then
+        return 1
+    fi
+
+    if [ "$desiredReplicas" != "$readyReplicas" ] ; then
+        return 1
+    fi
+
+    if [ "$desiredReplicas" != "$updatedReplicas" ] ; then
+        return 1
+    fi
+
+    return 0
+}
+
+# Run a test every second with a spinner until it succeeds
+function spinner_until() {
+    local timeoutSeconds="$1"
+    local cmd="$2"
+    local args=${@:3}
+
+    if [ -z "$timeoutSeconds" ]; then
+        timeoutSeconds=-1
+    fi
+
+    local delay=1
+    local elapsed=0
+    local spinstr='|/-\'
+
+    while ! $cmd $args; do
+        elapsed=$((elapsed + delay))
+        if [ "$timeoutSeconds" -ge 0 ] && [ "$elapsed" -gt "$timeoutSeconds" ]; then
+            return 1
+        fi
+        local temp=${spinstr#?}
+        printf " [%c]  " "$spinstr"
+        local spinstr=$temp${spinstr%"$temp"}
+        sleep $delay
+        printf "\b\b\b\b\b\b"
+    done
+}
+
+# create the storageclasses that the PVCs will use
+kubectl apply -f ./testing/yaml/storageclasses.yaml
+
+sleep 5
+
+# create the PVCs to be migrated
+kubectl apply -f ./testing/yaml/pvcs.yaml
+
+# populate the PVCs with data
+kubectl apply -f ./testing/yaml/datacreation.yaml
+
+# wait for jobs to complete
+kubectl wait --for=condition=complete --timeout=60s job/www-web-0
+kubectl wait --for=condition=complete --timeout=10s job/www-web-1
+kubectl wait --for=condition=complete --timeout=10s job/deployment-pvc
+
+# delete data creation jobs
+kubectl delete -f ./testing/yaml/datacreation.yaml
+
+# run deployments/statefulsets
+kubectl apply -f ./testing/yaml/datavalidation.yaml
+
+echo ""
+echo "waiting for the 'web' statefulset"
+spinner_until 120 statefulset_fully_updated default web
+echo ""
+echo "'web' statefulset healthy"
+echo "waiting for the 'short-pvc-name' deployment"
+echo ""
+spinner_until 120 deployment_fully_updated default short-pvc-name
+echo ""
+echo "'short-pvc-name' deployment healthy"
+

testing/validate.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# this waits for a deployment to have all replicas up-to-date and available
+function deployment_fully_updated() {
+    x_fully_updated "$1" deployment "$2"
+}
+
+# this waits for a statefulset to have all replicas up-to-date and available
+function statefulset_fully_updated() {
+    x_fully_updated "$1" statefulset "$2"
+}
+
+# this waits for a resource type (deployment or statefulset) to have all replicas up-to-date and available
+function x_fully_updated() {
+    local namespace=$1
+    local resourcetype=$2
+    local name=$3
+
+    local desiredReplicas
+    desiredReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.replicas}')
+
+    local availableReplicas
+    availableReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.availableReplicas}')
+
+    local readyReplicas
+    readyReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.readyReplicas}')
+
+    local updatedReplicas
+    updatedReplicas=$(kubectl get $resourcetype -n "$namespace" "$name" -o jsonpath='{.status.updatedReplicas}')
+
+    if [ "$desiredReplicas" != "$availableReplicas" ] ; then
+        return 1
+    fi
+
+    if [ "$desiredReplicas" != "$readyReplicas" ] ; then
+        return 1
+    fi
+
+    if [ "$desiredReplicas" != "$updatedReplicas" ] ; then
+        return 1
+    fi
+
+    return 0
+}
+
+# Run a test every second with a spinner until it succeeds
+function spinner_until() {
+    local timeoutSeconds="$1"
+    local cmd="$2"
+    local args=${@:3}
+
+    if [ -z "$timeoutSeconds" ]; then
+        timeoutSeconds=-1
+    fi
+
+    local delay=1
+    local elapsed=0
+    local spinstr='|/-\'
+
+    while ! $cmd $args; do
+        elapsed=$((elapsed + delay))
+        if [ "$timeoutSeconds" -ge 0 ] && [ "$elapsed" -gt "$timeoutSeconds" ]; then
+            return 1
+        fi
+        local temp=${spinstr#?}
+        printf " [%c]  " "$spinstr"
+        local spinstr=$temp${spinstr%"$temp"}
+        sleep $delay
+        printf "\b\b\b\b\b\b"
+    done
+}
+
+kubectl get statefulsets
+kubectl get deployments
+
+echo ""
+echo "waiting for the 'web' statefulset"
+spinner_until 120 statefulset_fully_updated default web
+echo ""
+echo "'web' statefulset healthy"
+echo "waiting for the 'short-pvc-name' deployment"
+echo ""
+spinner_until 120 deployment_fully_updated default short-pvc-name
+echo ""
+echo "'short-pvc-name' deployment healthy"
+
+kubectl get statefulsets
+kubectl get deployments
+kubectl get pvc
+
+if kubectl get pvc | grep -q int-source; then
+  echo "found PVCs in the int-source namespace"
+  exit 1
+fi

testing/yaml/datacreation.yaml

@@ -0,0 +1,92 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: createfile
+data:
+  create.sh: |
+    echo 'pvmigrate test string' > /pvmigrate/test
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: www-web-0
+spec:
+  template:
+    spec:
+      containers:
+        - name: training-container
+          image: registry.k8s.io/nginx-slim:0.8
+          command:
+            - bash
+            - /app/create.sh
+          volumeMounts:
+            - mountPath: /pvmigrate/
+              name: data-volume
+            - name: config-volume
+              mountPath: /app/create.sh
+              subPath: create.sh
+      restartPolicy: Never
+      volumes:
+        - name: data-volume
+          persistentVolumeClaim:
+            claimName: www-web-0
+        - name: config-volume
+          configMap:
+            name: createfile
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: www-web-1
+spec:
+  template:
+    spec:
+      containers:
+        - name: training-container
+          image: registry.k8s.io/nginx-slim:0.8
+          command:
+            - bash
+            - /app/create.sh
+          volumeMounts:
+            - mountPath: /pvmigrate/
+              name: data-volume
+            - name: config-volume
+              mountPath: /app/create.sh
+              subPath: create.sh
+      restartPolicy: Never
+      volumes:
+        - name: data-volume
+          persistentVolumeClaim:
+            claimName: www-web-1
+        - name: config-volume
+          configMap:
+            name: createfile
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: deployment-pvc
+spec:
+  template:
+    spec:
+      containers:
+        - name: training-container
+          image: registry.k8s.io/nginx-slim:0.8
+          command:
+            - bash
+            - /app/create.sh
+          volumeMounts:
+            - mountPath: /pvmigrate/
+              name: data-volume
+            - name: config-volume
+              mountPath: /app/create.sh
+              subPath: create.sh
+      restartPolicy: Never
+      volumes:
+        - name: data-volume
+          persistentVolumeClaim:
+            claimName: deployment-pvc
+        - name: config-volume
+          configMap:
+            name: createfile
+---