E2E workflow for test app in OpenShift and Enterprise in Jenkins

- Minor changes to get a new E2E flow working (authenticator sidecars in OpenShift and Conjur Enterprise and follower in Jenkins) - Minor cleanups and deflaking
cyberark · Sep 6, 2021 · 21a34cf · 21a34cf
1 parent b5979a2
commit 21a34cf
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 57 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -95,19 +95,23 @@ pipeline {
             sh './bin/helm-dependency-update-in-docker'
           }
         }
-        stage('Openshift E2E Workflow Tests') {
-          steps {
-            sh 'cd bin/test-workflow && summon --environment openshift -D ENV=ci -D VER=current ./start --platform oc'
-          }
-        }
-        stage('Run E2E Tests') {
+        stage('Test app with') {
           parallel {
-            stage('Enterprise and test app deployed to GKE') {
+            stage('Enterprise in GKE') {
               steps {
                 sh 'cd bin/test-workflow && summon --environment gke ./start --enterprise --platform gke'
               }
             }
-            stage('Enterprise deployed locally, test app deployed to GKE') {
+            stage('OSS in OpenShift') {
+              steps {
+                sh 'cd bin/test-workflow && summon --environment openshift -D ENV=ci -D VER=current ./start --platform openshift'
+              }
+            }
+          }
+        }
+        stage('Enterprise in Jenkins') {
+          stages {
+            stage('Test app in GKE') {
               steps {
                 sh '''
                   HOST_IP="$(curl http://169.254.169.254/latest/meta-data/public-ipv4)";
@@ -116,6 +120,15 @@ pipeline {
                 '''
               }
             }
+            stage('Test app in OpenShift') {
+              steps {
+                sh '''
+                  HOST_IP="$(curl http://169.254.169.254/latest/meta-data/public-ipv4)";
+                  echo "HOST_IP=${HOST_IP}"
+                  cd bin/test-workflow && summon --environment openshift -D ENV=ci -D VER=current ./start --enterprise --platform jenkins
+                '''
+              }
+            }
           }
         }
       }
@@ -125,9 +138,10 @@ pipeline {
       parallel {
         stage('On a master build') {
           when { branch 'master' }
-            steps {
-              sh 'summon ./bin/publish --edge'
-            }
+
+          steps {
+            sh 'summon ./bin/publish --edge'
+          }
         }
         stage('On a new tag') {
           when { tag "v*" }

diff --git a/bin/test-workflow/0_prep_env.sh b/bin/test-workflow/0_prep_env.sh
@@ -6,30 +6,37 @@ set -o pipefail
 export CONJUR_OSS_HELM_INSTALLED="${CONJUR_OSS_HELM_INSTALLED:-true}"
 export UNIQUE_TEST_ID="$(uuidgen | tr "[:upper:]" "[:lower:]" | head -c 10)"
 
-# PLATFORM is used to differentiate between general Kubernetes platforms (K8s vs. oc), while
-# CONJUR_PLATFORM is used to differentiate between sub-platforms (for vanilla K8s, KinD vs. GKE) for the Conjur deployment
-# APP_PLATFORM serves the same purpose as CONJUR_PLATFORM, but for the test app deployment
+# PLATFORM is used to differentiate between general Kubernetes platforms (kubernetes, openshift), while
+# CONJUR_PLATFORM is used to differentiate between sub-platforms (kind, gke, jenkins, openshift) for the Conjur deployment
+# APP_PLATFORM serves the same purpose as CONJUR_PLATFORM, but for the test app deployment (kind, gke, openshift)
 if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
   CONJUR_PLATFORM="${CONJUR_PLATFORM:-kind}"
 else
   CONJUR_PLATFORM="${CONJUR_PLATFORM:-gke}"
 fi
 export CONJUR_PLATFORM
 
-if [[ "$CONJUR_PLATFORM" == "oc" ]]; then
-  PLATFORM="openshift"
+if [[ "$CONJUR_PLATFORM" == "openshift" ]]; then
+  PLATFORM="${PLATFORM:-openshift}"
 else
-  PLATFORM="kubernetes"
+  PLATFORM="${PLATFORM:-kubernetes}"
 fi
 export PLATFORM
+
 if [[ "$CONJUR_PLATFORM" == "kind" ]]; then
-    RUN_CLIENT_CONTAINER="false"
+  RUN_CLIENT_CONTAINER="false"
+else
+  RUN_CLIENT_CONTAINER="true"
 fi
 
-if [[ "$CONJUR_PLATFORM" != "jenkins" ]]; then
-  APP_PLATFORM="$CONJUR_PLATFORM"
-else
-  APP_PLATFORM="gke"
+if [[ "$CONJUR_PLATFORM" != "kind" ]]; then
+  if [[ "$CONJUR_PLATFORM" != "jenkins" ]]; then
+    APP_PLATFORM="$CONJUR_PLATFORM"
+  elif [[ "$PLATFORM" == "kubernetes" ]]; then
+    APP_PLATFORM="gke"
+  elif [[ "$PLATFORM" == "openshift" ]]; then
+    APP_PLATFORM="openshift"
+  fi
 fi
 export APP_PLATFORM
 
@@ -39,6 +46,7 @@ export DOCKER_REGISTRY_URL="${DOCKER_REGISTRY_URL:-localhost:5000}"
 export DOCKER_REGISTRY_PATH="${DOCKER_REGISTRY_PATH:-localhost:5000}"
 export PULL_DOCKER_REGISTRY_URL="${PULL_DOCKER_REGISTRY_URL:-${DOCKER_REGISTRY_URL}}"
 export PULL_DOCKER_REGISTRY_PATH="${PULL_DOCKER_REGISTRY_PATH:-${DOCKER_REGISTRY_PATH}}"
+export PLATFORM_CONTAINER="platform-container"
 
 ### CONJUR AND TEST APP CONFIG
 export CONJUR_ACCOUNT="${CONJUR_ACCOUNT:-myConjurAccount}"
@@ -99,8 +107,6 @@ elif [[ "$CONJUR_PLATFORM" == "jenkins" ]]; then
 fi
 
 if [[ "$RUN_CLIENT_CONTAINER" == "true" ]]; then
-  export PLATFORM_CONTAINER="platform-container"
-
   docker build --tag "$PLATFORM_CONTAINER:$CONJUR_NAMESPACE_NAME" \
       --file Dockerfile \
       --build-arg KUBECTL_VERSION="$KUBECTL_VERSION" \

diff --git a/bin/test-workflow/1_deploy_conjur.sh b/bin/test-workflow/1_deploy_conjur.sh
@@ -59,14 +59,9 @@ CONJUR_AUTHENTICATORS=authn-k8s/\"${AUTHENTICATOR_ID}\",authn
 
 function setup_conjur_open_source {
   pushd temp > /dev/null
+    git clone --single-branch --branch main https://github.com/cyberark/conjur-oss-helm-chart.git "conjur-oss-helm-chart-$UNIQUE_TEST_ID"
 
-    if [ -d "conjur-oss-helm-chart" ]; then
-      rm -rf conjur-oss-helm-chart
-    fi
-
-    git clone https://github.com/cyberark/conjur-oss-helm-chart.git
-
-    pushd conjur-oss-helm-chart/examples/common > /dev/null
+    pushd "conjur-oss-helm-chart-$UNIQUE_TEST_ID/examples/common" > /dev/null
       source ./utils.sh
 
       announce "Setting demo environment variable defaults"
@@ -91,8 +86,6 @@ function setup_conjur_open_source {
       announce "Enabling the Conjur Kubernetes authenticator if necessary"
       ./4_ensure_authn_k8s_enabled.sh
     popd > /dev/null
-
-    rm -rf conjur-oss-helm-chart
   popd > /dev/null
 }
 

diff --git a/bin/test-workflow/2_admin_load_conjur_policies.sh b/bin/test-workflow/2_admin_load_conjur_policies.sh
@@ -68,7 +68,9 @@ pushd policy > /dev/null
   if [[ "$PLATFORM" == "openshift" ]]; then
     is_openshift=true
     is_kubernetes=false
-    oc adm policy add-scc-to-user anyuid system:serviceaccount:$CONJUR_NAMESPACE_NAME:conjur-oss
+    if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
+      oc adm policy add-scc-to-user anyuid system:serviceaccount:$CONJUR_NAMESPACE_NAME:conjur-oss
+    fi
   else
     is_openshift=false
     is_kubernetes=true

diff --git a/bin/test-workflow/dev-start.sh b/bin/test-workflow/dev-start.sh
@@ -71,8 +71,8 @@ function main() {
     echo "Openshift"
     echo "Running" "${cmd}"
     # some scripts need these set
-    export CONJUR_PLATFORM="oc"
-    export APP_PLATFORM="oc"
+    export CONJUR_PLATFORM="openshift"
+    export APP_PLATFORM="openshift"
     export RUN_CLIENT_CONTAINER="$local_container"
     summon -e openshift -D ENV=$env -D VER=$openshift_version \
         sh -c "${cmd}"

diff --git a/bin/test-workflow/platform_login.sh b/bin/test-workflow/platform_login.sh
@@ -12,7 +12,7 @@ if [[ "$CONJUR_PLATFORM" == "gke" || "$APP_PLATFORM" == "gke" ]]; then
   check_env_var GCLOUD_PROJECT_NAME
 fi
 
-if [[ "$CONJUR_PLATFORM" == "oc" || "$APP_PLATFORM" == "oc" ]]; then
+if [[ "$CONJUR_PLATFORM" == "openshift" || "$APP_PLATFORM" == "openshift" ]]; then
   check_env_var CONJUR_PLATFORM
   check_env_var APP_PLATFORM
   check_env_var OPENSHIFT_URL
@@ -31,7 +31,7 @@ function main {
     docker login "$DOCKER_REGISTRY_URL" \
       -u oauth2accesstoken \
       -p "$(gcloud auth print-access-token)"
-  elif [[ "$CONJUR_PLATFORM" == "oc" || "$APP_PLATFORM" == "oc" ]]; then
+  elif [[ "$CONJUR_PLATFORM" == "openshift" || "$APP_PLATFORM" == "openshift" ]]; then
     oc login "$OPENSHIFT_URL" \
       --username="$OPENSHIFT_USERNAME" \
       --password="$OPENSHIFT_PASSWORD" \

diff --git a/bin/test-workflow/start b/bin/test-workflow/start
@@ -18,7 +18,7 @@ Usage: ./start [options]:
     -p, --platform <pform>    Platform on which to deploy Conjur
                               For Open Source workflow:
                                 - Defaults to 'kind'
-                                - Supports 'oc'
+                                - Supports 'openshift'
                               For Enterprise workflow:
                                 - Defaults to 'gke'
                                 - Supports 'jenkins'
@@ -105,9 +105,9 @@ export RUN_CLIENT_CONTAINER="${RUN_CLIENT_CONTAINER:-true}"
 if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
   if [[ -z "$CONJUR_PLATFORM" ]]; then
     CONJUR_PLATFORM="kind"
-  elif [[ "$CONJUR_PLATFORM" != "kind" && "$CONJUR_PLATFORM" != "oc" ]]; then
+  elif [[ "$CONJUR_PLATFORM" != "kind" && "$CONJUR_PLATFORM" != "openshift" ]]; then
     echo "Conjur Open Source workflow not compatible with platform \"$CONJUR_PLATFORM\""
-    echo "Workflow currently only compatible with \"kind\" or \"oc\""
+    echo "Workflow currently only compatible with \"kind\" or \"openshift\""
     exit
   fi
 else
@@ -123,7 +123,7 @@ fi
 if [[ -z "$APP_PLATFORM" ]]; then
   APP_PLATFORM="$CONJUR_PLATFORM"
   if [[ "$CONJUR_PLATFORM" == "jenkins" ]]; then
-    APP_PLATFORM="gke"
+    APP_PLATFORM="openshift"
   fi
 fi
 
@@ -149,14 +149,14 @@ test_app_workflow="
 ./8_app_verify_authentication.sh"
 
 if [[ "$CONJUR_OSS_HELM_INSTALLED" == "true" ]]; then
-  if [[ "$CONJUR_PLATFORM" == "oc" && "$RUN_CLIENT_CONTAINER" == "true" ]]; then
+  if [[ "$CONJUR_PLATFORM" == "openshift" && "$RUN_CLIENT_CONTAINER" == "true" ]]; then
     source "./0_prep_env.sh"
     run_command_with_platform "./1_deploy_conjur.sh"
     run_command_with_platform "$conjur_prep"
     run_command_with_platform "$cluster_prep"
     run_command_with_platform "$test_app_workflow"
   else
-    if [[ "$CONJUR_PLATFORM" == "oc" ]]; then
+    if [[ "$CONJUR_PLATFORM" == "openshift" ]]; then
       ./platform_login.sh
     fi
     eval "$conjur_init"
@@ -169,7 +169,7 @@ elif [[ "$CONJUR_PLATFORM" == "gke" ]]; then
   run_command_with_platform "$conjur_prep"
   run_command_with_platform "$cluster_prep"
   run_command_with_platform "$test_app_workflow"
-elif [[ "$CONJUR_PLATFORM" == "jenkins" && "$APP_PLATFORM" == "gke" ]]; then
+elif [[ "$CONJUR_PLATFORM" == "jenkins" ]]; then
   eval "$conjur_init"
   eval "$conjur_prep"
   run_command_with_platform "$cluster_prep"

diff --git a/bin/test-workflow/stop b/bin/test-workflow/stop
@@ -8,31 +8,36 @@ check_env_var RUN_CLIENT_CONTAINER
 if [[ "${CONJUR_OSS_HELM_INSTALLED}" == "true" ]]; then
   if [[ "$RUN_CLIENT_CONTAINER" == "true" ]]; then
     run_command_with_platform "
-    ./cleanup_helm.sh
-    ./cleanup_namespaces.sh
-    rm -rf temp
+      ./cleanup_helm.sh
+      ./cleanup_namespaces.sh
+      rm -rf "temp/conjur-oss-helm-chart-$UNIQUE_TEST_ID"
     "
   else 
     ./cleanup_helm.sh
     ./cleanup_namespaces.sh
+    rm -rf "temp/conjur-oss-helm-chart-$UNIQUE_TEST_ID"
   fi
+
 elif [[ "$CONJUR_PLATFORM" == "gke" ]]; then
   run_command_with_platform "
     ./cleanup_helm.sh
-    pushd temp/kubernetes-conjur-deploy-$UNIQUE_TEST_ID && ./stop && popd
+    pushd 'temp/kubernetes-conjur-deploy-$UNIQUE_TEST_ID' && ./stop && popd
     ./cleanup_namespaces.sh
   "
-  rm -rf temp/kubernetes-conjur-deploy-"$UNIQUE_TEST_ID"
+
+  rm -rf "temp/kubernetes-conjur-deploy-$UNIQUE_TEST_ID"
 
 elif [[ "$CONJUR_PLATFORM" == "jenkins" ]]; then
-  pushd temp/conjur-intro-"$UNIQUE_TEST_ID" > /dev/null
+  pushd "temp/conjur-intro-$UNIQUE_TEST_ID" > /dev/null
     ./bin/dap --stop
+    docker run --rm -w /dap-intro -v "$(pwd):/dap-intro" alpine ash -c "rm -rf system"
   popd > /dev/null
-  rm -rf temp/conjur-intro-"$UNIQUE_TEST_ID"
 
   run_command_with_platform "
     ./cleanup_helm.sh
     ./cleanup_namespaces.sh
   "
 
+  rm -rf "temp/conjur-intro-$UNIQUE_TEST_ID"
+
 fi
diff --git a/bin/test-workflow/utils.sh b/bin/test-workflow/utils.sh
@@ -266,14 +266,14 @@ function uninstall_helm_release {
 function run_command_with_platform {
 
   GCLOUD_INCLUDES="-i"
-  if [[ "$PLATFORM" == "openshift" ]]; then
-    GCLOUD_CLUSTER_NAME="gke"
-    GCLOUD_ZONE="gke"
-    GCLOUD_PROJECT_NAME="gke"
-  else
+  if [[ "$CONJUR_PLATFORM" == "gke" || "$APP_PLATFORM" == "gke" ]]; then
     if [[ ! -z "${GCLOUD_SERVICE_KEY}" ]]; then
       GCLOUD_INCLUDES="-v$GCLOUD_SERVICE_KEY:/tmp$GCLOUD_SERVICE_KEY"
     fi
+  else
+    GCLOUD_CLUSTER_NAME="gke"
+    GCLOUD_ZONE="gke"
+    GCLOUD_PROJECT_NAME="gke"
   fi
 
   docker run --rm \