Baked required deps in docker image for CircleCI. (thanos-io#1484)

This should fix thanos-io#1483 Also it should massively improve our CI time. Signed-off-by: Bartek Plotka <[email protected]>
fatsheep9146 · Sep 2, 2019 · c4fff05 · c4fff05
1 parent ba509f9
commit c4fff05
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 18 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -3,12 +3,11 @@ version: 2
 jobs:
   test:
     docker:
-      # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12.5
+     # Build by Thanos make docker-ci
+      - image: quay.io/thanos/thanos-ci:v0.1.0
     working_directory: /go/src/github.com/thanos-io/thanos
     environment:
       GO111MODULE: 'on'
-      GOBIN: "/go/bin"
       # Run garbage collection more aggresively to avoid getting OOMed during the lint phase (4GB limit).
       GOGC: "20"
       # By default Go uses GOMAXPROCS but a Circle CI executor has many
@@ -60,11 +59,9 @@ jobs:
 
   publish_master:
     docker:
-      # Available from https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.12.5
+      # Build by Thanos make docker-ci
+      - image: quay.io/thanos/thanos-ci:v0.1.0
     working_directory: /go/src/github.com/thanos-io/thanos
-    environment:
-      GOBIN: "/go/bin"
     steps:
       - checkout
       - setup_remote_docker:

diff --git a/.gitignore b/.gitignore
@@ -23,3 +23,5 @@ data/
 website/public/
 website/docs-pre-processed/
 !website/data
+
+tmp/bin
diff --git a/Dockerfile.thanos-ci b/Dockerfile.thanos-ci
@@ -0,0 +1,6 @@
+# Available from https://hub.docker.com/r/circleci/golang/
+FROM circleci/golang:1.12.5
+
+ENV GOBIN=/go/bin
+
+COPY tmp/bin ${GOBIN}
diff --git a/Makefile b/Makefile
@@ -46,9 +46,13 @@ ME                ?= $(shell whoami)
 
 # Limited prom version, because testing was not possible. This should fix it: https://github.com/thanos-io/thanos/issues/758
 PROM_VERSIONS           ?= v2.4.3 v2.5.0 v2.8.1 v2.9.2
+PROMS ?= $(GOBIN)/prometheus-v2.4.3 $(GOBIN)/prometheus-v2.5.0 $(GOBIN)/prometheus-v2.8.1 $(GOBIN)/prometheus-v2.9.2
 
 ALERTMANAGER_VERSION    ?= v0.15.2
+ALERTMANAGER            ?= $(GOBIN)/alertmanager-$(ALERTMANAGER_VERSION)
+
 MINIO_SERVER_VERSION    ?= RELEASE.2018-10-06T00-15-16Z
+MINIO_SERVER            ?=$(GOBIN)/minio-$(MINIO_SERVER_VERSION)
 
 # fetch_go_bin_version downloads (go gets) the binary from specific version and installs it in $(GOBIN)/<bin>-<version>
 # arguments:
@@ -178,11 +182,17 @@ tarballs-release: $(PROMU)
 # test runs all Thanos golang tests against each supported version of Prometheus.
 .PHONY: test
 test: export GOCACHE= $(TMP_GOPATH)/gocache
-test: check-git test-deps
+test: export THANOS_TEST_MINIO_PATH= $(MINIO_SERVER)
+test: export THANOS_TEST_PROMETHEUS_VERSIONS= $(PROM_VERSIONS)
+test: export THANOS_TEST_ALERTMANAGER_PATH= $(ALERTMANAGER)
+test: check-git install-deps
+	@echo ">> install thanos GOOPTS=${GOOPTS}"
+	# Thanos binary is required by e2e tests.
+	@go install github.com/thanos-io/thanos/cmd/thanos
 	# Be careful on GOCACHE. Those tests are sometimes using built Thanos/Prometheus binaries directly. Don't cache those.
 	@rm -rf ${GOCACHE}
 	@echo ">> running all tests. Do export THANOS_SKIP_GCS_TESTS='true' or/and THANOS_SKIP_S3_AWS_TESTS='true' or/and THANOS_SKIP_AZURE_TESTS='true' and/or THANOS_SKIP_SWIFT_TESTS='true' and/or THANOS_SKIP_TENCENT_COS_TESTS='true' if you want to skip e2e tests against real store buckets"
-	THANOS_TEST_PROMETHEUS_VERSIONS="$(PROM_VERSIONS)" THANOS_TEST_ALERTMANAGER_PATH="alertmanager-$(ALERTMANAGER_VERSION)" go test $(shell go list ./... | grep -v /vendor/);
+	@go test $(shell go list ./... | grep -v /vendor/);
 
 .PHONY: test-only-gcs
 test-only-gcs: export THANOS_SKIP_S3_AWS_TESTS = true
@@ -202,15 +212,23 @@ test-local:
 	@echo ">> Skipping GCE tests"
 	$(MAKE) test-only-gcs
 
-# test-deps installs dependency for e2e tets.
-# It installs current Thanos, supported versions of Prometheus and alertmanager to test against in e2e.
-.PHONY: test-deps
-test-deps:
-	@echo ">> install thanos GOOPTS=${GOOPTS}"
-	@go install github.com/thanos-io/thanos/cmd/thanos
-	$(foreach ver,$(PROM_VERSIONS),$(call fetch_go_bin_version,github.com/prometheus/prometheus/cmd/prometheus,$(ver)))
-	$(call fetch_go_bin_version,github.com/prometheus/alertmanager/cmd/alertmanager,$(ALERTMANAGER_VERSION))
-	$(call fetch_go_bin_version,github.com/minio/minio,$(MINIO_SERVER_VERSION))
+# install-deps installs dependencies for e2e tetss.
+# It installs supported versions of Prometheus and alertmanager to test against in e2e.
+.PHONY: install-deps
+install-deps: $(ALERTMANAGER) $(MINIO_SERVER) $(PROMS)
+	@echo ">>GOBIN=$(GOBIN)"
+
+.PHONY: docker-ci
+# To be run by Thanos maintainer.
+docker-ci: install-deps
+	# Copy all to tmp local dir as this is required by docker.
+	@rm -rf ./tmp/bin
+	@mkdir -p ./tmp/bin
+	@cp -r $(GOBIN)/* ./tmp/bin
+	@docker build -t thanos-ci -f Dockerfile.thanos-ci .
+	@echo ">> pushing thanos-ci image"
+	@docker tag "thanos-ci" "quay.io/thanos/thanos-ci:v0.1.0"
+	@docker push "quay.io/thanos/thanos-ci:v0.1.0"
 
 # tooling deps. TODO(bwplotka): Pin them all to certain version!
 .PHONY: check-git
@@ -277,6 +295,15 @@ $(GOLANGCILINT):
 $(MISSPELL):
 	$(call fetch_go_bin_version,github.com/client9/misspell/cmd/misspell,$(MISSPELL_VERSION))
 
+$(ALERTMANAGER):
+	$(call fetch_go_bin_version,github.com/prometheus/alertmanager/cmd/alertmanager,$(ALERTMANAGER_VERSION))
+
+$(MINIO_SERVER):
+	$(call fetch_go_bin_version,github.com/minio/minio,$(MINIO_SERVER_VERSION))
+
+$(PROMS):
+	$(foreach ver,$(PROM_VERSIONS),$(call fetch_go_bin_version,github.com/prometheus/prometheus/cmd/prometheus,$(ver)))
+
 $(PROTOC):
 	@mkdir -p $(TMP_GOPATH)
 	@echo ">> fetching protoc@${PROTOC_VERSION}"