Automated test script improvements and fixes

deploy/ci/automation/cfallinonetest.sh

@@ -29,20 +29,22 @@ fi
 
 echo "Running Stratos All-in-one"
 
+CF=https://api.${CF_DOMAIN}
+UAA=$(curl -k -s $CF | jq -r .links.uaa.href)
+
+echo "Using UAA Endpoint: ${UAA}"
+
 # Run the all-in-one Stratos
 # Configure env to use the UAA provided by PCF dev
 CONTAINER_ID=$(docker run \
 -d \
 -p 5443:443 \
 -e CONSOLE_CLIENT='cf' \
--e UAA_ENDPOINT='https://login.local.pcfdev.io' \
+-e UAA_ENDPOINT="${UAA}" \
 -e SKIP_SSL_VALIDATION='true' \
 -e CONSOLE_ADMIN_SCOPE='cloud_controller.admin' \
 $IMAGE)
 
-# Get the E2E config
-curl -k ${TEST_CONFIG_URL} --output secrets.yaml
-
 # Need node modules to run the tests
 rm -rf node_modules
 npm install

deploy/ci/automation/cfdockercomposetest.sh

@@ -31,8 +31,16 @@ mkdir -p "${E2E_REPORT_FOLDER}/logs"
 
 echo "Running Stratos in Docker Compose"
 pushd deploy
-#docker-compose -f docker-compose.development.yml up -d | tee "${E2E_REPORT_FOLDER}/logs/build.log"
-docker-compose -f docker-compose.development.yml up -d
+
+# Patch the docker compose file to run Stratos on a different port
+rm -f docker-compose.testing.yml
+rm -f common.testing.yml
+rm -f *.bak
+sed -e 's/80:80/2080:80/g' common.yml > common.testing.yml
+sed -i.bak -e 's/443:443/2443:443/g' common.testing.yml
+sed -e 's/file: common.yml/file: common.testing.yml/g' docker-compose.development.yml > docker-compose.testing.yml
+
+docker-compose -f docker-compose.testing.yml up -d
 popd
 
 echo "Docker Containers"
@@ -42,16 +50,12 @@ docker ps
 echo "Waiting 30 seconds for UAA to start up ..."
 sleep 30
 
-# Get the E2E config
-rm -f secrets.yaml
-curl -k ${TEST_CONFIG_URL} --output secrets.yaml
-
 # Need node modules to run the tests
 rm -rf node_modules
 npm install
 
 # Run the E2E tests
-"$DIRPATH/runandrecord.sh" https://127.0.0.1
+"$DIRPATH/runandrecord.sh" https://127.0.0.1:2443
 RET=$?
 
 set +e
@@ -60,13 +64,13 @@ pushd deploy
 
 # Store logs if there was a test failure
 if [ $RET -ne 0 ]; then
-  docker-compose -f docker-compose.development.yml logs proxy > "${E2E_REPORT_FOLDER}/logs/jetstream.log"
-  docker-compose -f docker-compose.development.yml logs db-migrator > "${E2E_REPORT_FOLDER}/logs/db-migrator.log"
-  docker-compose -f docker-compose.development.yml logs nginx > "${E2E_REPORT_FOLDER}/logs/nginx.log"
-  docker-compose -f docker-compose.development.yml logs mariadb > "${E2E_REPORT_FOLDER}/logs/mariadb.log"
+  docker-compose -f docker-compose.testing.yml logs proxy > "${E2E_REPORT_FOLDER}/logs/jetstream.log"
+  docker-compose -f docker-compose.testing.yml logs db-migrator > "${E2E_REPORT_FOLDER}/logs/db-migrator.log"
+  docker-compose -f docker-compose.testing.yml logs nginx > "${E2E_REPORT_FOLDER}/logs/nginx.log"
+  docker-compose -f docker-compose.testing.yml logs mariadb > "${E2E_REPORT_FOLDER}/logs/mariadb.log"
 fi
 
-docker-compose -f docker-compose.development.yml down
+docker-compose -f docker-compose.testing.yml down
 popd
 
 echo "All done"

deploy/ci/automation/cfpushtest.sh

@@ -22,7 +22,7 @@ if [ "$1" == "mysql" ]; then
   PORT=3306
   echo "Using MYSQL database"
   echo "Starting MySQL Database..."
-  docker kill $(docker ps -q --filter "ancestor=mysql:latest")
+  killDockerContainer "mysql:latest"
   DB_DOCKER_PID=$(docker run -d -p $PORT:3306 --env MYSQL_ROOT_PASSWORD=stratos mysql:latest --default-authentication-plugin=mysql_native_password)
   echo "Waiting for mysql"
   until mysql -h $HOST -uroot -pstratos -e "SHOW DATABASES;" &> /dev/null
@@ -31,19 +31,24 @@ if [ "$1" == "mysql" ]; then
     sleep 2
   done
 
-  mysql -u root -pstratos -h $HOST -e "CREATE DATABASE $DB_NAME;"
-  mysql -u root -pstratos -h $HOST -e "CREATE USER '$USERNAME'@'$HOST' IDENTIFIED BY '$PASSWORD';"
-  mysql -u root -pstratos -h $HOST -e "GRANT ALL PRIVILEGES ON * . * TO '$USERNAME'@'$HOST';"
-  mysql -u root -pstratos -h $HOST -e "FLUSH PRIVILEGES;"
+  echo ""
+
+  echo "Creating database and setting permissions..."
+  mysql -uroot -pstratos -h $HOST -e "CREATE DATABASE $DB_NAME;"
+  mysql -uroot -pstratos -h $HOST -e "CREATE USER '$USERNAME'@'%' IDENTIFIED BY '$PASSWORD';"
+  mysql -uroot -pstratos -h $HOST -e "GRANT ALL PRIVILEGES ON * . * TO '$USERNAME'@'%';"
+  mysql -uroot -pstratos -h $HOST -e "FLUSH PRIVILEGES;"
 fi
 
 if [ "$1" == "pgsql" ]; then
   DB_TYPE=pgsql
   PORT=5432
   echo "Using Postgres database"
   echo "Starting Postgres Database..."
-  docker kill $(docker ps -q --filter "ancestor=postgres:latest")
-  DB_DOCKER_PID=$(docker run -d -p $PORT:5432 --env POSTGRES_PASSWORD=stratos --env POSTGRES_DB=$DB_NAME --env POSTGRES_USER=$USERNAME postgres:latest)
+  killDockerContainer "postgres:latest"
+  USERNAME=stratos_pgsql
+  PASSWORD=stratos_pgsql_passw0rd
+  DB_DOCKER_PID=$(docker run -d -p $PORT:5432 --env POSTGRES_PASSWORD=${PASSWORD} --env POSTGRES_DB=$DB_NAME --env POSTGRES_USER=$USERNAME postgres:latest)
 fi
 
 DB=stratos-${DB_TYPE}
@@ -53,9 +58,13 @@ DB=stratos-${DB_TYPE}
 cf delete -f -r console
 cf ds -f $DB
 
-CONFIG='{"dbname":"'$DB_NAME'","name":"'$DB_NAME'","username":"'$USERNAME'","password":"'$PASSWORD'","uri":"'${DB_TYPE}'://database","port":"'$PORT'","hostname":"'$HOST'"}'
-echo "Creating user provided service for the database..."
-cf cups ${DB} -p "'${CONFIG}'"
+if [ -n "${DB_TYPE}" ]; then
+  CONFIG='{"dbname":"'$DB_NAME'","name":"'$DB_NAME'","username":"'$USERNAME'","password":"'$PASSWORD'","uri":"'${DB_TYPE}'://database","port":"'$PORT'","hostname":"'$HOST'"}'
+  echo "Creating user provided service for the database..."
+  echo "Database Server: ${HOST}:${PORT}"
+  cf cups ${DB} -p "'${CONFIG}'"
+  echo ${CONFIG}
+fi
 
 set -e
 
@@ -96,6 +105,9 @@ if [ "$2" == "prebuild" ]; then
   npm run prebuild-ui
 fi
 
+# If the push fails, we want to continue and show the logs
+set +e
+
 # Push Stratos to the Cloud Foundry
 cf push -f $MANIFEST
 RET=$?
@@ -107,32 +119,27 @@ if [ $RET -ne 0 ]; then
   echo "Push failed... showing recent log of the Stratos app"
   cf logs console --recent
   set -e
-fi
-
-# Get the E2E config
-rm -f secrets.yaml
-curl -k ${TEST_CONFIG_URL} --output secrets.yaml
-
-rm -rf node_modules
-npm install
-
-# Clean the E2E reports folder
-rm -rf ./e2e-reports
-mkdir -p ./e2e-reports
-export E2E_REPORT_FOLDER=./e2e-reports
+else
+  # Push was okay, so we can prepare and run E2E tests
+  rm -rf node_modules
+  npm install
 
-# Run the E2E tests
-"$DIRPATH/runandrecord.sh" https://console.local.pcfdev.io ${SUITE}
-RET=$?
+  # Clean the E2E reports folder
+  rm -rf ./e2e-reports
+  mkdir -p ./e2e-reports
+  export E2E_REPORT_FOLDER=./e2e-reports
 
-# If we had test failures then copy console log to reports folder
-if [ $RET -ne 0 ]; then
-  cf logs --recent console > "${E2E_REPORT_FOLDER}/console-app.log"
-fi 
+  # Run the E2E tests
+  "$DIRPATH/runandrecord.sh" https://console.${CF_DOMAIN} ${SUITE}
+  RET=$?
 
-# Delete the app
-cf delete -f -r console
+  # If we had test failures then copy console log to reports folder
+  if [ $RET -ne 0 ]; then
+    cf logs --recent console > "${E2E_REPORT_FOLDER}/console-app.log"
+  fi 
+fi
 
+# Clean up
 rm $MANIFEST
 
 # Stop the database server
@@ -143,6 +150,13 @@ fi
 
 set +e
 
+# Delete the app - add one retry if it fails first time
+cf delete -f -r console
+if [ $? -ne 0 ]; then
+  sleep 60
+  cf delete -f -r console
+fi
+
 echo "All done"
 
 # Return exit code form the e2e tests

deploy/ci/automation/cfutils.sh

@@ -13,12 +13,32 @@ done <<< "$ORGS"
 
 }
 
+function killDockerContainer {
+  local ANCESTOR=$1
+  local ID=$(docker ps -q --filter "ancestor=${ANCESTOR}")
+
+  if [ "${ID}" != "" ]; then
+    echo "Killing docker container: ${ID}"
+    docker kill ${ID}
+  else
+    echo "No existing running container: ${ANCESTOR}"
+  fi
+}
+
 # Any test run could have been aborted, leaving remnants - clean them up
 function clean_deployments {
   echo "Stopping any MySQL Database"
   MYSQL_DOCKER=$(docker ps -q --filter "ancestor=mysql:latest")
   if [ -n "$MYSQL_DOCKER" ]; then
     docker kill $MYSQL_DOCKER
+    docker rm $MYSQL_DOCKER
+  fi
+
+  echo "Stopping any Postgres Database"
+  PGSQL_DOCKER=$(docker ps -q --filter "ancestor=postgres:latest")
+  if [ -n "$PGSQL_DOCKER" ]; then
+    docker kill $PGSQL_DOCKER
+    docker rm $PGSQL_DOCKER
   fi
 
   echo "Stopping previous Docker Compose (if any)"
@@ -31,27 +51,29 @@ function clean_deployments {
   RUNNING=$(docker ps -q --filter "ancestor=stratos-aio:latest")
   if [ -n "$RUNNING" ]; then
     docker kill $RUNNING
+    docker rm $RUNNING
   fi
 
   echo "Stopping previous Docker All-in-one Nightly (if any)"
   # Kill any existing docker all-in-one nightly docker containers
   RUNNING=$(docker ps -q --filter "ancestor=splatform/stratos")
   if [ -n "$RUNNING" ]; then
     docker kill $RUNNING
+    docker rm $RUNNING
   fi
 
   # Delete existing Stratos instance if there is one
   echo "Deleting previous Stratos app from CF (if any)"
   cf delete -f -r console > /dev/null 2>&1
 
   # Kill all containers
-  echo "Killing all Docker containers"
-  docker kill $(docker ps -q) > /dev/null 2>&1
-  docker rm $(docker ps -a -q) > /dev/null 2>&1
+  # echo "Killing all Docker containers"
+  # docker kill $(docker ps -q) > /dev/null 2>&1
+  # docker rm $(docker ps -a -q) > /dev/null 2>&1
 
   # Remove all images
-  echo "Removing docker images"
-  docker rmi $(docker images -q) -f > /dev/null 2>&1
+  # echo "Removing docker images"
+  # docker rmi $(docker images -q) -f > /dev/null 2>&1
 }
 
 # Need TEST_CONFIG_URL to be set
@@ -60,21 +82,30 @@ if [ -z "${TEST_CONFIG_URL}" ]; then
   exit 1
 fi
 
-echo "Checking PCF Dev status"
-FULL_STATUS=$(cf pcfdev status)
-echo "$FULL_STATUS"
+# Is the test config file a local file? (must be absolute path)
+if [[ "${TEST_CONFIG_URL}" == /* ]]; then
+  rm -f secrets.yaml
+  cp "${TEST_CONFIG_URL}" secrets.yaml
+else
+  # Get the E2E config
+  rm -f secrets.yaml
+  curl -k ${TEST_CONFIG_URL} --output secrets.yaml
+fi
 
-STATUS=$(echo "$FULL_STATUS" | head -n 1)
-if [ "$STATUS" != "Running" ]; then
-  echo "PCF Dev is not running... aborting"
-  exit 1
+CF_DOMAIN=${CF_DOMAIN-local.pcfdev.io}
+if [ -z "${CF_URL}" ]; then
+  CF_URL=https://api.${CF_DOMAIN}
 fi
+CF_USER=${CF_USER}
+CF_PASSWORD=${CF_PASSWORD}
+
+echo ${CF_URL}
 
-cf login -a https://api.local.pcfdev.io --skip-ssl-validation -u admin -p admin -o e2e -s e2e
-cf apps
+cf login -a ${CF_URL} --skip-ssl-validation -u ${CF_USER} -p ${CF_PASSWORD} -o e2e -s e2e
+cf buildpacks
 
 if [ $? -ne 0 ]; then
-  echo "Unable to list apps - looks like a problem with PCFDev"
+  echo "Unable to list buildpacks - looks like a problem with the test Cloud Foundry"
   exit 1
 fi
 

deploy/ci/automation/runandrecord.sh

@@ -20,7 +20,7 @@ export E2E_REPORT_FOLDER=./e2e-reports
 export DISPLAY=:99.0
 mkdir -p "${E2E_REPORT_FOLDER}"
 echo "Starting ffmpeg to capture screen as video"
-ffmpeg -video_size 1366x768 -framerate 25 -f x11grab -draw_mouse 0 -i :99.0 "${E2E_REPORT_FOLDER}/ScreenCapture.mp4" > "${E2E_REPORT_FOLDER}>ffmpeg.log" 2>&1 &
+ffmpeg -video_size 1366x768 -framerate 25 -f x11grab -draw_mouse 0 -i :99.0 "${E2E_REPORT_FOLDER}/ScreenCapture.mp4" > "${E2E_REPORT_FOLDER}/ffmpeg.log" 2>&1 &
 FFMPEG=$!
 
 export STRATOS_E2E_LOG_TIME=true