Merge pull request #456 from hasheddan/bundles-of-fun

Use bundled packages and update CI to dual publish to DockerHub and Upbound Marketplace

.github/workflows/backport.yml

@@ -18,7 +18,7 @@ jobs:
   # The main gotchas with this action are that it _only_ supports merge commits,
   # and that PRs _must_ be labelled before they're merged to trigger a backport.
   open-pr:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     if: github.event.pull_request.merged
     steps:
       - name: Checkout

.github/workflows/ci.yml

@@ -10,19 +10,20 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.17'
-  GOLANGCI_VERSION: 'v1.31'
-  DOCKER_BUILDX_VERSION: 'v0.4.2'
+  GO_VERSION: '1.18'
+  GOLANGCI_VERSION: 'v1.47.1'
+  DOCKER_BUILDX_VERSION: 'v0.8.2'
 
   # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run
   # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether
   # credentials have been provided before trying to run steps that need them.
   DOCKER_USR: ${{ secrets.DOCKER_USR }}
+  XPKG_ACCESS_ID: ${{ secrets.XPKG_ACCESS_ID }}
   AWS_USR: ${{ secrets.AWS_USR }}
 
 jobs:
   detect-noop:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     outputs:
       noop: ${{ steps.noop.outputs.should_skip }}
     steps:
@@ -36,7 +37,7 @@ jobs:
 
 
   lint:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     needs: detect-noop
     if: needs.detect-noop.outputs.noop != 'true'
 
@@ -80,7 +81,7 @@ jobs:
           version: ${{ env.GOLANGCI_VERSION }}
 
   check-diff:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     needs: detect-noop
     if: needs.detect-noop.outputs.noop != 'true'
 
@@ -120,7 +121,7 @@ jobs:
         run: make check-diff
 
   unit-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     needs: detect-noop
     if: needs.detect-noop.outputs.noop != 'true'
 
@@ -169,7 +170,7 @@ jobs:
           file: _output/tests/linux_amd64/coverage.txt
 
   e2e-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     needs: detect-noop
     if: needs.detect-noop.outputs.noop != 'true'
 
@@ -231,7 +232,7 @@ jobs:
         run: make e2e USE_HELM3=true
 
   publish-artifacts:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     needs: detect-noop
     if: needs.detect-noop.outputs.noop != 'true'
 
@@ -301,6 +302,14 @@ jobs:
           username: ${{ secrets.DOCKER_USR }}
           password: ${{ secrets.DOCKER_PSW }}
 
+      - name: Login to Upbound
+        uses: docker/login-action@v1
+        if: env.XPKG_ACCESS_ID != ''
+        with:
+          registry: xpkg.upbound.io
+          username: ${{ secrets.XPKG_ACCESS_ID }}
+          password: ${{ secrets.XPKG_TOKEN }}
+
       - name: Publish Artifacts to S3 and Docker Hub
         run: make -j2 publish BRANCH_NAME=${GITHUB_REF##*/}
         if: env.AWS_USR != '' && env.DOCKER_USR != ''

.github/workflows/commands.yml

@@ -4,7 +4,7 @@ on: issue_comment
 
 jobs:
   points:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     if: startsWith(github.event.comment.body, '/points')
 
     steps:
@@ -65,7 +65,7 @@ jobs:
   # NOTE(negz): See also backport.yml, which is the variant that triggers on PR
   # merge rather than on comment.
   backport:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     if: github.event.issue.pull_request && startsWith(github.event.comment.body, '/backport')
     steps:
     - name: Extract Command

.github/workflows/promote.yml

@@ -20,7 +20,7 @@ env:
 
 jobs:
   promote-artifacts:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
       - name: Checkout

.github/workflows/tag.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   create-tag:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
       - name: Checkout

Makefile

@@ -37,14 +37,29 @@ GO111MODULE = on
 # ====================================================================================
 # Setup Kubernetes tools
 
+UP_VERSION = v0.13.0
+UP_CHANNEL = stable
 -include build/makelib/k8s_tools.mk
 
 # ====================================================================================
 # Setup Images
 
-DOCKER_REGISTRY ?= crossplane
-IMAGES = provider-gcp provider-gcp-controller
--include build/makelib/image.mk
+IMAGES = provider-gcp
+-include build/makelib/imagelight.mk
+
+# ====================================================================================
+# Setup XPKG
+
+XPKG_REG_ORGS ?= xpkg.upbound.io/crossplane index.docker.io/crossplane
+# NOTE(hasheddan): skip promoting on xpkg.upbound.io as channel tags are
+# inferred.
+XPKG_REG_ORGS_NO_PROMOTE ?= xpkg.upbound.io/crossplane
+XPKGS = provider-gcp
+-include build/makelib/xpkg.mk
+
+# NOTE(hasheddan): we force image building to happen prior to xpkg build so that
+# we ensure image is present in daemon.
+xpkg.build.provider-gcp: do.build.images
 
 # ====================================================================================
 # Targets
@@ -69,6 +84,11 @@ fallthrough: submodules
 go.cachedir:
 	@go env GOCACHE
 
+# NOTE(hasheddan): we must ensure up is installed in tool cache prior to build
+# as including the k8s_tools machinery prior to the xpkg machinery sets UP to
+# point to tool cache.
+build.init: $(UP)
+
 # Generate a coverage report for cobertura applying exclusions on
 # - generated file
 cobertura:

apis/storage/v1alpha3/types.go

@@ -321,16 +321,14 @@ func NewRetentionPolicy(rp *storage.RetentionPolicy) *RetentionPolicy {
 
 // CopyToRetentionPolicy create a copy in storage format
 func CopyToRetentionPolicy(rp *RetentionPolicy) *storage.RetentionPolicy {
-	var d time.Duration
+	d := time.Duration(0)
 
-	if rp == nil {
-		d = time.Duration(0)
-	} else {
+	if rp != nil {
 		d = time.Duration(rp.RetentionPeriodSeconds)
 	}
 
 	return &storage.RetentionPolicy{
-		RetentionPeriod: d * time.Second,
+		RetentionPeriod: d * time.Second, //nolint:durationcheck
 	}
 }
 

cluster/images/provider-gcp/Dockerfile

@@ -1,3 +1,9 @@
-FROM BASEIMAGE
+FROM gcr.io/distroless/static@sha256:1f580b0a1922c3e54ae15b0758b5747b260bd99d39d40c2edb3e7f6e2452298b
 
-COPY package.yaml .
+ARG TARGETOS
+ARG TARGETARCH
+
+ADD bin/$TARGETOS\_$TARGETARCH/provider /usr/local/bin/crossplane-gcp-provider
+
+USER 65532
+ENTRYPOINT ["crossplane-gcp-provider"]

cluster/images/provider-gcp/Makefile

@@ -1,30 +1,35 @@
 # ====================================================================================
 # Setup Project
 
-PLATFORMS := linux_amd64 linux_arm64
 include ../../../build/makelib/common.mk
 
 # ====================================================================================
 #  Options
-DOCKER_REGISTRY ?= crossplane
-IMAGE = $(BUILD_REGISTRY)/provider-gcp-$(ARCH)
-OSBASEIMAGE = scratch
-include ../../../build/makelib/image.mk
+
+include ../../../build/makelib/imagelight.mk
 
 # ====================================================================================
 # Targets
 
 img.build:
 	@$(INFO) docker build $(IMAGE)
+	@$(MAKE) BUILD_ARGS="--load" img.build.shared
+	@$(OK) docker build $(IMAGE)
+
+img.publish:
+	@$(INFO) Skipping image publish for $(IMAGE)
+	@echo Publish is deferred to xpkg machinery
+	@$(OK) Image publish skipped for $(IMAGE)
+
+img.build.shared:
 	@cp Dockerfile $(IMAGE_TEMP_DIR) || $(FAIL)
-	@cp -R ../../../package $(IMAGE_TEMP_DIR) || $(FAIL)
-	@cd $(IMAGE_TEMP_DIR) && $(SED_CMD) 's|BASEIMAGE|$(OSBASEIMAGE)|g' Dockerfile || $(FAIL)
-	@cd $(IMAGE_TEMP_DIR) && $(SED_CMD) 's|VERSION|$(VERSION)|g' package/crossplane.yaml || $(FAIL)
-	@cd $(IMAGE_TEMP_DIR) && $(SED_CMD) 's|DOCKER_REGISTRY|$(DOCKER_REGISTRY)|g' package/crossplane.yaml || $(FAIL)
-	@cd $(IMAGE_TEMP_DIR) && find package -type f -name '*.yaml' -exec cat {} >> 'package.yaml' \; -exec printf '\n---\n' \; || $(FAIL)
-	@docker build $(BUILD_ARGS) \
-		--build-arg ARCH=$(ARCH) \
-		--build-arg TINI_VERSION=$(TINI_VERSION) \
+	@cp -r $(OUTPUT_DIR)/bin/ $(IMAGE_TEMP_DIR)/bin || $(FAIL)
+	@docker buildx build $(BUILD_ARGS) \
+		--platform $(IMAGE_PLATFORMS) \
 		-t $(IMAGE) \
 		$(IMAGE_TEMP_DIR) || $(FAIL)
-	@$(OK) docker build $(IMAGE)
+
+img.promote:
+	@$(INFO) Skipping image promotion from $(FROM_IMAGE) to $(TO_IMAGE)
+	@echo Promote is deferred to xpkg machinery
+	@$(OK) Image promotion skipped for $(FROM_IMAGE) to $(TO_IMAGE)

cluster/local/integration_tests.sh

@@ -41,12 +41,10 @@ eval $(make --no-print-directory -C ${projectdir} build.vars)
 # ------------------------------
 
 SAFEHOSTARCH="${SAFEHOSTARCH:-amd64}"
-BUILD_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-${SAFEHOSTARCH}"
-CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-controller-${SAFEHOSTARCH}"
+CONTROLLER_IMAGE="${BUILD_REGISTRY}/${PROJECT_NAME}-${SAFEHOSTARCH}"
 
 version_tag="$(cat ${projectdir}/_output/version)"
 # tag as latest version to load into kind cluster
-PACKAGE_CONTROLLER_IMAGE="${DOCKER_REGISTRY}/${PROJECT_NAME}-controller:${VERSION}"
 K8S_CLUSTER="${K8S_CLUSTER:-${BUILD_REGISTRY}-inttests}"
 
 CROSSPLANE_NAMESPACE="crossplane-system"
@@ -68,7 +66,7 @@ echo_step "setting up local package cache"
 CACHE_PATH="${projectdir}/.work/inttest-package-cache"
 mkdir -p "${CACHE_PATH}"
 echo "created cache dir at ${CACHE_PATH}"
-docker save "${BUILD_IMAGE}" -o "${CACHE_PATH}/${PACKAGE_NAME}.xpkg" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.xpkg"
+"${UP}" alpha xpkg xp-extract --from-xpkg "${OUTPUT_DIR}"/xpkg/"${HOSTOS}"_"${SAFEHOSTARCH}"/"${PACKAGE_NAME}"-"${VERSION}".xpkg -o "${CACHE_PATH}/${PACKAGE_NAME}.gz" && chmod 644 "${CACHE_PATH}/${PACKAGE_NAME}.gz"
 
 # create kind cluster with extra mounts
 echo_step "creating k8s cluster using kind"
@@ -85,13 +83,8 @@ EOF
 echo "${KIND_CONFIG}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --config=-
 
 # tag controller image and load it into kind cluster
-docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_CONTROLLER_IMAGE}"
-"${KIND}" load docker-image "${PACKAGE_CONTROLLER_IMAGE}" --name="${K8S_CLUSTER}"
-
-# files are not synced properly from host to kind node container on Jenkins, so
-# we must manually copy image from host to node
-echo_step "pre-cache package by copying to kind node"
-docker cp "${CACHE_PATH}/${PACKAGE_NAME}.xpkg" "${K8S_CLUSTER}-control-plane":"/cache/${PACKAGE_NAME}.xpkg"
+docker tag "${CONTROLLER_IMAGE}" "${PACKAGE_NAME}"
+"${KIND}" load docker-image "${PACKAGE_NAME}" --name="${K8S_CLUSTER}"
 
 echo_step "create crossplane-system namespace"
 "${KUBECTL}" create ns crossplane-system
@@ -142,7 +135,7 @@ echo_info "using crossplane version ${chart_version}"
 echo
 # we replace empty dir with our PVC so that the /cache dir in the kind node
 # container is exposed to the crossplane pod
-"${HELM3}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version 1.6.4 --wait --set packageCache.pvc=package-cache
+"${HELM3}" install crossplane --namespace crossplane-system crossplane-stable/crossplane --version ${chart_version} --wait --set packageCache.pvc=package-cache
 
 # ----------- integration tests
 echo_step "--- INTEGRATION TESTS ---"

go.mod

@@ -1,6 +1,6 @@
 module github.com/crossplane-contrib/provider-gcp
 
-go 1.17
+go 1.18
 
 require (
 	cloud.google.com/go/storage v1.15.0