From fec9f4e96600a6dfe596b083f548697fec910228 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Wed, 29 May 2024 16:54:11 -0700 Subject: [PATCH] Test Chart installation for Plus and remote images Problem: We're not testing the Helm Chart installation with NGINX Plus and with pre-built images Solution: Add installation steps for both --- .github/workflows/ci.yml | 121 +++++----------------- .github/workflows/conformance.yml | 11 +- .github/workflows/functional.yml | 13 +-- .github/workflows/helm.yml | 166 ++++++++++++++++++++++++++++++ Makefile | 2 - tests/Makefile | 3 +- 6 files changed, 203 insertions(+), 113 deletions(-) create mode 100644 .github/workflows/helm.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 660b69c215..297a2667d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,20 +31,33 @@ jobs: outputs: go_path: ${{ steps.vars.outputs.go_path }} min_k8s_version: ${{ steps.vars.outputs.min_k8s_version }} + k8s_latest: ${{ steps.vars.outputs.k8s_latest }} + helm_changes: ${{ steps.filter.outputs.charts }} steps: - name: Checkout Repository uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 - name: Setup Golang Environment uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: go-version: stable + - name: Check for changes + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + charts: + - charts/nginx-gateway-fabric/**/* + - name: Output Variables id: vars run: | echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT - echo "min_k8s_version=1.25.16" >> $GITHUB_OUTPUT + echo "min_k8s_version=v1.25.16" >> $GITHUB_OUTPUT + echo "k8s_latest=$(grep -m1 'FROM kindest/node' > $GITHUB_OUTPUT - name: Check if go.mod and go.sum are up to date run: go mod tidy && git diff --exit-code -- go.mod go.sum @@ -211,7 +224,7 @@ jobs: fail-fast: false matrix: image: [nginx, plus] - k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"] + k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"] uses: ./.github/workflows/functional.yml with: image: ${{ matrix.image }} @@ -226,7 +239,7 @@ jobs: fail-fast: false matrix: image: [nginx, plus] - k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"] + k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"] enable-experimental: [true, false] uses: ./.github/workflows/conformance.yml with: @@ -238,96 +251,18 @@ jobs: helm-tests: name: Helm Tests - runs-on: ubuntu-22.04 - needs: [vars, build-oss] - steps: - - name: Checkout Repository - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - - - name: Fetch Cached Artifacts - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 - with: - path: ${{ github.workspace }}/dist - key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }} - - - name: Docker Buildx - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 - - - name: NGF Docker meta - id: ngf-meta - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 - with: - images: | - name=ghcr.io/nginxinc/nginx-gateway-fabric - tags: | - type=semver,pattern={{version}} - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: NGINX Docker meta - id: nginx-meta - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 - with: - images: | - name=ghcr.io/nginxinc/nginx-gateway-fabric/nginx - tags: | - type=semver,pattern={{version}} - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} - - - name: Build NGF Docker Image - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 - with: - file: build/Dockerfile - tags: ${{ steps.ngf-meta.outputs.tags }} - context: "." - target: goreleaser - load: true - cache-from: type=gha,scope=ngf - pull: true - - - name: Build NGINX Docker Image - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 - with: - file: build/Dockerfile.nginx - tags: ${{ steps.nginx-meta.outputs.tags }} - context: "." - load: true - cache-from: type=gha,scope=nginx - pull: true - build-args: | - NJS_DIR=internal/mode/static/nginx/modules/src - NGINX_CONF_DIR=internal/mode/static/nginx/conf - BUILD_AGENT=gha - - - name: Deploy Kubernetes - id: k8s - run: | - kube_config=${{ github.workspace }}/kube-${{ github.run_id }}-helm - make create-kind-cluster KIND_KUBE_CONFIG=${kube_config} - echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV" - kind load docker-image ghcr.io/nginxinc/nginx-gateway-fabric:${{ steps.ngf-meta.outputs.version }} ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${{ steps.nginx-meta.outputs.version }} - kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - - - - name: Install Chart - run: > - helm install - helm-$(echo ${{ steps.ngf-meta.outputs.version }} | tr '.' '-') - . - --wait - --create-namespace - --set nginxGateway.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric - --set nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} - --set nginxGateway.image.pullPolicy=Never - --set nginxGateway.productTelemetry.enable=false - --set nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx - --set nginx.image.tag=${{ steps.nginx-meta.outputs.version }} - --set nginx.image.pullPolicy=Never - --set service.type=NodePort - -n nginx-gateway - working-directory: ${{ github.workspace }}/charts/nginx-gateway-fabric + needs: [vars, build-oss, build-plus] + strategy: + fail-fast: false + matrix: + image: [nginx, plus] + k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"] + uses: ./.github/workflows/helm.yml + with: + image: ${{ matrix.image }} + k8s-version: ${{ matrix.k8s-version }} + secrets: inherit + if: ${{ needs.vars.outputs.helm_changes == 'true' || github.event_name == 'schedule' }} publish-helm: name: Package and Publish Helm Chart diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 72b403a70e..d6d3c1a3a3 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -129,10 +129,8 @@ jobs: - name: Deploy Kubernetes id: k8s run: | - k8s_version=${{ inputs.k8s-version }} - make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }} - echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV" - working-directory: ./tests + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} - name: Setup conformance tests run: | @@ -140,12 +138,13 @@ jobs: ngf_tag=${{ steps.ngf-meta.outputs.version }} if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi - make install-ngf-local-no-build${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} + make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag} working-directory: ./tests - name: Run conformance tests run: | - make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} + make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }} core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result') extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result') if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 4cec276d40..dadd05e0f6 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -101,17 +101,8 @@ jobs: - name: Deploy Kubernetes id: k8s run: | - k8s_version=${{ inputs.k8s-version }} - make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }} - echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV" - - - name: Setup functional tests - id: setup - run: | - ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric - ngf_tag=${{ steps.ngf-meta.outputs.version }} - make load-images${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag} - working-directory: ./tests + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} - name: Run functional telemetry tests run: | diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 0000000000..e849321da3 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,166 @@ +name: Helm Testing + +on: + workflow_call: + inputs: + image: + required: true + type: string + k8s-version: + required: true + type: string + +permissions: + contents: read + +jobs: + helm-tests-local: + name: Helm Tests Local + runs-on: ubuntu-22.04 + if: ${{ github.event_name != 'schedule' }} + steps: + - name: Checkout Repository + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: Fetch Cached Artifacts + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ github.workspace }}/dist + key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }} + + - name: Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + + - name: NGF Docker meta + id: ngf-meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: | + name=ghcr.io/nginxinc/nginx-gateway-fabric + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: NGINX Docker meta + id: nginx-meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: | + name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} + tags: | + type=semver,pattern={{version}} + type=edge + type=schedule + type=ref,event=pr + type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + + - name: Build NGF Docker Image + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + file: build/Dockerfile + tags: ${{ steps.ngf-meta.outputs.tags }} + context: "." + target: goreleaser + load: true + cache-from: type=gha,scope=ngf + pull: true + + - name: Build NGINX Docker Image + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + tags: ${{ steps.nginx-meta.outputs.tags }} + context: "." + load: true + cache-from: type=gha,scope=${{ inputs.image }} + pull: true + build-args: | + NJS_DIR=internal/mode/static/nginx/modules/src + NGINX_CONF_DIR=internal/mode/static/nginx/conf + BUILD_AGENT=gha + + - name: Install cloud-provider-kind + run: | + curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz + tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz + + - name: Run cloud-provider-kind + run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1 + + - name: Deploy Kubernetes + id: k8s + run: | + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} + kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }} + kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - + + - name: Set up Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.x" + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + + - name: Install Chart + run: | + ct install --config .ct.yaml --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \ + --set=nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx${{ inputs.image == 'plus' && '-plus' || ''}} \ + --set=nginx.plus=${{ inputs.image == 'plus' }} \ + --set=nginx.image.tag=${{ steps.nginx-meta.outputs.version }} \ + --set=nginxGateway.image.pullPolicy=Never \ + --set=nginx.image.pullPolicy=Never \ + --set=nginxGateway.productTelemetry.enable=false" + + helm-test-remote: + name: Helm Tests Remote + runs-on: ubuntu-22.04 + if: ${{ github.event_name == 'schedule' }} + steps: + - name: Checkout Repository + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: Install cloud-provider-kind + run: | + curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz + tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz + + - name: Run cloud-provider-kind + run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1 + + - name: Deploy Kubernetes + id: k8s + run: | + kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }} + kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f - + kubectl create namespace nginx-gateway + + - name: Create k8s secret + if: ${{ inputs.image == 'plus' }} + run: | + kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway + + - name: Set up Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: "3.x" + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + + - name: Install Chart + run: | + ct install --config .ct.yaml --all --namespace nginx-gateway --helm-extra-set-args="--set=nginxGateway.image.tag=nightly \ + --set=nginx.plus=${{ inputs.image == 'plus' }} \ + --set=nginx.image.tag=nightly \ + --set=nginxGateway.productTelemetry.enable=false \ + ${{ inputs.image == 'plus' && '--set=serviceAccount.imagePullSecret=nginx-plus-registry-secret --set=nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus' || '' }}" diff --git a/Makefile b/Makefile index 288b4eb56f..e652e4c1a7 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,6 @@ NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: ngin NGINX_PLUS_PREFIX ?= $(PREFIX)/nginx-plus## The name of the nginx plus image. For example: nginx-gateway-fabric/nginx-plus TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 1.1.0 TARGET ?= local## The target of the build. Possible values: local and container -KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig OUT_DIR ?= build/out## The folder where the binary will be stored GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin @@ -149,7 +148,6 @@ deps: ## Add missing and remove unused modules, verify deps and download them to create-kind-cluster: ## Create a kind cluster $(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <$(SELF_DIR)tests/Dockerfile | awk -F'[ ]' '{print $$2}')) kind create cluster --image $(KIND_IMAGE) - kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG) .PHONY: delete-kind-cluster delete-kind-cluster: ## Delete kind cluster diff --git a/tests/Makefile b/tests/Makefile index 7aba9fafb2..eacc490596 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,5 @@ CI=false +CLUSTER_NAME ?= kind CONFORMANCE_PREFIX = conformance-test-runner## Prefix for the conformance test runner image CONFORMANCE_TAG = latest## Tag for the conformance test runner image GATEWAY_CLASS = nginx## Gateway class to use @@ -29,7 +30,7 @@ build-test-runner-image: ## Build conformance test runner image .PHONY: run-conformance-tests run-conformance-tests: ## Run conformance tests - kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) + kind load docker-image $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --name $(CLUSTER_NAME) kubectl apply -f conformance/conformance-rbac.yaml kubectl run -i conformance \ --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Never \