Support staging ES security tests

.ci/jobs/elastic+helm-charts+staging+integration-elasticsearch.yml

@@ -35,11 +35,20 @@
         VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
         export VAULT_TOKEN
         unset VAULT_ROLE_ID VAULT_SECRET_ID
+
+        DOCKER_PASSWORD=$(retry 5 vault read -field password secret/devops-ci/docker.elastic.co/devops-ci)
+        retry 5 docker login -u devops-ci -p $DOCKER_PASSWORD docker.elastic.co
+        unset DOCKER_PASSWORD
         set -x
 
         env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py
 
         cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}"
 
         cd helpers/terraform/
+
+        # pull private images while we have the hosts docker daemon authenticated
+        make pull-private-images
+
+        # the private images will be used in here
         ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${ES_SUITE} CHART=elasticsearch

elasticsearch/examples/security/Makefile

@@ -3,6 +3,7 @@ default: test
 include ../../../helpers/examples.mk
 
 RELEASE := helm-es-security
+ELASTICSEARCH_IMAGE := docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION)
 
 install:
 	helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../
@@ -13,12 +14,15 @@ purge:
 
 test: secrets install goss
 
+pull-elasticsearch-image:
+	docker pull $(ELASTICSEARCH_IMAGE)
+
 secrets:
 	docker rm -f elastic-helm-charts-certs || true
 	rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true
-	password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \
+	password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm $(ELASTICSEARCH_IMAGE) /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \
 	docker run --name elastic-helm-charts-certs -i -w /app \
-		docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \
+		$(ELASTICSEARCH_IMAGE) \
 		/bin/sh -c " \
 			elasticsearch-certutil ca --out /app/elastic-stack-ca.p12 --pass '' && \
 			elasticsearch-certutil cert --name security-master --dns security-master --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \

helpers/bumper.py

@@ -74,7 +74,8 @@
     image_file_patterns = file_patterns + [
         "*/tests/*.py",
         "**/templates/*.tpl",
-        "**/Makefile",
+        # some tests use docker images in their makefile
+        "*/examples/*/Makefile",
     ]
 
     print("\nUpdating namespaces...")
@@ -85,7 +86,7 @@
             for line in fileinput.input([f], inplace=True):
                 print(
                     re.sub(
-                        r"docker.elastic.co/.+/",
+                        r"docker.elastic.co/.+?/",
                         "docker.elastic.co/staging/",
                         line.rstrip(),
                     )

helpers/terraform/Makefile

@@ -93,3 +93,9 @@ integration: creds ## Deploy helm chart and run integration tests
 .PHONY: build
 build: ## Build helm-charts docker image
 	for i in 1 2 3 4 5; do docker build -t helm-charts . && break || sleep 5; done
+
+.PHONY: pull-private-images
+pull-private-images: ## Pull private images used in testing
+	cd ../../elasticsearch/examples/security/ && \
+		make pull-elasticsearch-image
+