Add Gateway API CRDs to the repo with kustomize (#2011)

Problem: We don't have a way to tie a version of the Gateway API to our release (or edge). There's also not a single source of truth for the API version.

Solution: Use kustomize to add a specific version of the CRDs to our codebase. This way we only have to manage the version in one place and we can tie it to our releases.

.github/workflows/ci.yml

@@ -309,7 +309,7 @@ jobs:
           make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
           echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
           kind load docker-image ghcr.io/nginxinc/nginx-gateway-fabric:${{ steps.ngf-meta.outputs.version }} ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${{ steps.nginx-meta.outputs.version }}
-          kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+          kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
 
       - name: Install Chart
         run: >

Makefile

@@ -17,7 +17,6 @@ TELEMETRY_REPORT_PERIOD = 24h
 TELEMETRY_ENDPOINT=# if empty, NGF will report telemetry in its logs at debug level.
 TELEMETRY_ENDPOINT_INSECURE = false
 
-GW_API_VERSION = 1.1.0
 ENABLE_EXPERIMENTAL ?= false
 NODE_VERSION = $(shell cat .nvmrc)
 
@@ -112,15 +111,15 @@ generate-crds: ## Generate CRDs and Go types using kubebuilder
 
 .PHONY: install-crds
 install-crds: ## Install CRDs
-	kubectl kustomize config/crd | kubectl apply -f -
+	kubectl kustomize $(SELF_DIR)config/crd | kubectl apply -f -
 
 .PHONY: install-gateway-crds
 install-gateway-crds: ## Install Gateway API CRDs
-	$(SELF_DIR)tests/scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL)
+	kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply -f -
 
 .PHONY: uninstall-gateway-crds
 uninstall-gateway-crds: ## Uninstall Gateway API CRDs
-	$(SELF_DIR)tests/scripts/uninstall-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL)
+	kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl delete -f -
 
 .PHONY: generate-manifests
 generate-manifests: ## Generate manifests using Helm.

charts/nginx-gateway-fabric/README.md

@@ -40,7 +40,7 @@ This chart deploys the NGINX Gateway Fabric in your Kubernetes cluster.
 > [see the Technical Specifications](https://github.com/nginxinc/nginx-gateway-fabric/blob/main/README.md#technical-specifications).
 
 ```shell
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl apply -f -
 ```
 
 ## Installing the Chart
@@ -128,7 +128,7 @@ Gateway Fabric - [see the Technical Specifications](../../README.md#technical-sp
 To upgrade the Gateway CRDs from [the Gateway API repo](https://github.com/kubernetes-sigs/gateway-api), run:
 
 ```shell
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl apply -f -
 ```
 
 ### Upgrading the CRDs
@@ -238,7 +238,7 @@ These commands remove all the Kubernetes components associated with the release
 To delete the Gateway API CRDs from [the Gateway API repo](https://github.com/kubernetes-sigs/gateway-api), run:
 
 ```shell
-kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+kubectl kustomize https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard | kubectl delete -f -
 ```
 
 ## Configuration

config/crd/gateway-api/experimental/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - https://github.com/kubernetes-sigs/gateway-api/config/crd/experimental?timeout=120&ref=v1.1.0

config/crd/gateway-api/standard/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- https://github.com/kubernetes-sigs/gateway-api/config/crd?timeout=120&ref=v1.1.0

config/crd/kustomization.yaml

@@ -1,3 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
 resources:
   - bases/gateway.nginx.org_clientsettingspolicies.yaml
   - bases/gateway.nginx.org_nginxgateways.yaml

docs/developer/quickstart.md

@@ -133,13 +133,13 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
 3. Install Gateway API CRDs:
 
    ```shell
-   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+   kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
    ```
 
    If you're implementing experimental Gateway API features, install Gateway API CRDs from the experimental channel:
 
    ```shell
-   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml
+   kubectl kustomize config/crd/gateway-api/experimental | kubectl apply -f -
    ```
 
 4. Install NGF using your custom image and expose NGF with a NodePort Service:

docs/developer/release-process.md

@@ -37,10 +37,8 @@ To create a new release, follow these steps:
 7. Create a release branch following the `release-X.Y` naming convention.
 8. Run the [Release PR](./../../.github/workflows/release-pr.yml) workflow to update the repo files for the release. Then there are a few manual steps to complete:
    1. Update the tag of NGF container images used in the [provisioner manifest](/tests/conformance/provisioner/provisioner.yaml).
-   2. Update any installation instructions to ensure that the supported Gateway API is correct.
-      Specifically, helm README and `site/content/includes/installation/install-gateway-api-resources.md`.
-   3. Update the [README](/README.md) to include information about the release.
-   4. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly.
+   2. Update the [README](/README.md) to include information about the release.
+   3. Update the [changelog](/CHANGELOG.md). There is going to be a new blank section generated by the automation that needs to be adjusted accordingly.
       The changelog includes only important (from the user perspective)
       changes to NGF. This is in contrast with the autogenerated full changelog, which is created in the next
       step. As a starting point, copy the important features, bug fixes, and dependencies from the autogenerated
@@ -51,14 +49,13 @@ To create a new release, follow these steps:
    - Build NGF, NGINX and NGINX Plus container images with the release tag `X.Y.Z` and push them to the registries.
    - Package and publish the Helm chart to the registry.
    - Create a GitHub release with an autogenerated changelog and attached release artifacts.
-10. Prepare and merge a PR into the main branch to update with similar information that you did in the release
-branch docs PR. Specifically:
-      1. [README](/README.md) to include the information about the latest release.
-      2. [changelog](/CHANGELOG.md).
-      3. Helm chart `version` field.
-      4. `VERSION` in top-level Makefile.
-      5. Any references in the docs to the previous release.
-      6. Any installation instructions to ensure that the supported Gateway API and NGF versions are correct.Specifically, helm README and `site/content/includes/installation/install-gateway-api-resources.md`.
+10. Prepare and merge a PR into the main branch to update with similar information that you did in the release branch docs PR. Specifically:
+    1. [README](/README.md) to include the information about the latest release.
+    2. [changelog](/CHANGELOG.md).
+    3. Helm chart `version` field.
+    4. `VERSION` in top-level Makefile.
+    5. Any references in the docs to the previous release.
+    6. Any installation instructions to ensure that the supported Gateway API and NGF versions are correct. Specifically, helm README.
 11. Ask the docs team to update the production branch for NGF in Netlify to our latest release branch and run the deployment pipeline.
 12. Close the issue created in Step 1.
 13. Ensure that the [associated milestone](https://github.com/nginxinc/nginx-gateway-fabric/milestones) is closed.

site/content/includes/installation/install-gateway-api-resources.md

@@ -7,12 +7,14 @@ docs: "DOCS-1438"
 To install the Gateway API resources, run the following:
 
 ```shell
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f -
 ```
 
+{{<note>}}If you plan to use the `edge` version of NGINX Gateway Fabric, you can replace the vesion in `ref` with `main`, for example `ref=main`.{{</note>}}
+
 Alternatively, you can install the Gateway API resources from the experimental channel. We support a subset of the
 additional features provided by the experimental channel. To install from the experimental channel, run the following:
 
 ```shell
-kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml
+kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f -
 ```

site/content/includes/installation/uninstall-gateway-api-resources.md

@@ -2,16 +2,16 @@
 docs: "DOCS-1436"
 ---
 
-   {{<warning>}}This will remove all corresponding custom resources in your entire cluster, across all namespaces. Double-check to make sure you don't have any custom resources you need to keep, and confirm that there are no other Gateway API implementations active in your cluster.{{</warning>}}
+{{<warning>}}This will remove all corresponding custom resources in your entire cluster, across all namespaces. Double-check to make sure you don't have any custom resources you need to keep, and confirm that there are no other Gateway API implementations active in your cluster.{{</warning>}}
 
-   To uninstall the Gateway API resources, run the following:
+To uninstall the Gateway API resources, run the following:
 
-   ```shell
-   kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
-   ```
+```shell
+kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl delete -f -
+```
 
-   Alternatively, if you installed the Gateway APIs from the experimental channel, run the following:
+Alternatively, if you installed the Gateway APIs from the experimental channel, run the following:
 
-   ```shell
-   kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml
-   ```
+```shell
+kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl delete -f -
+```

site/content/installation/installing-ngf/helm.md

@@ -126,17 +126,17 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta
 To upgrade your Gateway API resources, take the following steps:
 
 - Verify the Gateway API resources are compatible with your NGINX Gateway Fabric version. Refer to the [Technical Specifications]({{< relref "reference/technical-specifications.md" >}}) for details.
-- Review the [release notes](https://github.com/kubernetes-sigs/gateway-api/releases/tag/v1.1.0) for any important upgrade-specific information.
+- Review the [release notes](https://github.com/kubernetes-sigs/gateway-api/releases) for any important upgrade-specific information.
 - To upgrade the Gateway API resources, run:
 
   ```shell
-  kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+  kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f -
   ```
 
   or, if you installed the from the experimental channel:
 
   ```shell
-  kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml
+  kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f -
   ```
 
 ### Upgrade NGINX Gateway Fabric CRDs

site/content/installation/installing-ngf/manifests.md

@@ -148,13 +148,13 @@ To upgrade NGINX Gateway Fabric and get the latest features and improvements, ta
    - To upgrade the Gateway API resources, run:
 
      ```shell
-     kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+     kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.2.0" | kubectl apply -f -
      ```
 
      or, if you installed the from the experimental channel:
 
      ```shell
-     kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/experimental-install.yaml
+     kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/experimental?ref=v1.2.0" | kubectl apply -f -
      ```
 
 1. **Upgrade NGINX Gateway Fabric CRDs:**

tests/Makefile

@@ -4,6 +4,7 @@ CONFORMANCE_TAG = latest## Tag for the conformance test runner image
 GATEWAY_CLASS = nginx## Gateway class to use
 GINKGO_FLAGS =
 GINKGO_LABEL =
+GW_API_VERSION ?= $(shell yq eval '.resources[0] | split("ref=v") | .[1]' ../config/crd/gateway-api/standard/kustomization.yaml)## Supported Gateway API version from current NGF
 GW_API_PREV_VERSION ?= 1.0.0## Supported Gateway API version from previous NGF release
 GW_SERVICE_TYPE=NodePort## Service type to use for the gateway
 GW_SVC_GKE_INTERNAL=false
@@ -159,6 +160,13 @@ deploy-updated-provisioner: ## Update provisioner manifest and deploy to the con
 generate-static-deployment:
 	helm template nginx-gateway $(CHART_DIR) $(HELM_TEMPLATE_COMMON_ARGS) --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) --set nginx.plus=$(PLUS_ENABLED) > $(SELF_DIR)config/tests/static-deployment.yaml
 
+# this target is used to install the gateway-api CRDs from the main branch (only used in the nightly CI job)
+# it overrides the target in the main Makefile when the GW_API_VERSION is set to main
+ifeq ($(GW_API_VERSION),main)
+install-gateway-crds:
+	kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f -
+endif
+
 .PHONY: install-ngf-local-build
 install-ngf-local-build: deploy-updated-provisioner
 

tests/reconfig/setup.md

@@ -24,18 +24,18 @@
 
 ## Test Environment
 
- The following cluster will be sufficient:
+The following cluster will be sufficient:
 
 - A Kubernetes cluster with 4 nodes on GKE
   - Node: e2-medium (2 vCPU, 4GB memory)
 
 ## Setup
 
 1. Create cloud cluster
-2. Deploy CRDs:
+2. Install Gateway API Resources:
 
    ```bash
-   kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml
+   kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
    ```
 
 3. Deploy NGF from edge using Helm install and wait for LoadBalancer Service to be ready
@@ -59,16 +59,17 @@
         fixtures `scripts/delete-multiple.sh` which takes a number (needs to be the same number as what was used in the
         create script.)
 5. After each individual test:
-    - Describe the Gateway resource and make sure the status is correct.
-    - Check the logs of both NGF containers for errors.
-    - Parse the logs for TimeToReady numbers (see steps 6-7 below).
-    - Grab metrics.
-      Note: You can expose metrics by running the below snippet and then navigating to `127.0.0.1:9113/metrics`:
-
-       ```console
-       GW_POD=$(k get pods -n nginx-gateway | sed -n '2s/^\([^[:space:]]*\).*$/\1/p')
-       kubectl port-forward $GW_POD -n nginx-gateway 9113:9113 &
-       ```
+
+   - Describe the Gateway resource and make sure the status is correct.
+   - Check the logs of both NGF containers for errors.
+   - Parse the logs for TimeToReady numbers (see steps 6-7 below).
+   - Grab metrics.
+     Note: You can expose metrics by running the below snippet and then navigating to `127.0.0.1:9113/metrics`:
+
+     ```console
+     GW_POD=$(k get pods -n nginx-gateway | sed -n '2s/^\([^[:space:]]*\).*$/\1/p')
+     kubectl port-forward $GW_POD -n nginx-gateway 9113:9113 &
+     ```
 
 6. Measure NGINX Reloads and Time to Ready Results
    1. TimeToReadyTotal as described in each test - NGF logs.