Use OpenTofu to create GCP infrastructure

.editorconfig

@@ -8,6 +8,6 @@ trim_trailing_whitespace = true
 indent_size = 4
 indent_style = tab
 
-[*.{md,yml,yaml}]
+[*.{md,yml,yaml,tf}]
 indent_size = 2
 indent_style = space

.github/workflows/nfr.yml

@@ -88,7 +88,6 @@ jobs:
         id: auth
         uses: google-github-actions/auth@71f986410dfbc7added4569d411d040a91dc6935 # v2.1.8
         with:
-          token_format: access_token
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
           service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
 
@@ -98,44 +97,48 @@ jobs:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           install_components: kubectl
 
+      - name: Setup OpenTofu
+        uses: opentofu/setup-opentofu@592200bd4b9bbf4772ace78f887668b1aee8f716 # v1.0.5
+        with:
+          tofu_version: 1.9.0 # renovate: datasource=github-tags depName=opentofu/opentofu
+          tofu_wrapper: false
+
+      - name: Initialize OpenTofu
+        working-directory: ./tests/tofu
+        run: |
+          tofu version
+          tofu init
+
+      - name: Setup tfvars file
+        working-directory: ./tests/tofu
+        run: |
+          cat <<EOF > terraform.tfvars
+          gke_project = "${{ secrets.GCP_PROJECT_ID }}"
+          gke_cluster_name = "nfr-tests-${{ github.run_id }}-${{ matrix.type }}"
+          gke_num_nodes = 12
+          gke_machine_type = "n2d-standard-16"
+          gke_nodes_service_account = "${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}"
+          vm_service_account = "${{ secrets.GCP_SERVICE_ACCOUNT }}"
+          ngf_branch = "${{ github.ref_name }}"
+          EOF
+
       - name: Setup dotenv file
         working-directory: ./tests/scripts
         run: |
-          echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
           echo "TAG=${{ needs.vars.outputs.image_tag }}" >> vars.env
           echo "PREFIX=ghcr.io/nginx/nginx-gateway-fabric" >> vars.env
           echo "NGINX_PREFIX=ghcr.io/nginx/nginx-gateway-fabric/nginx" >> vars.env
           echo "NGINX_PLUS_PREFIX=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus" >> vars.env
-          echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "GKE_CLUSTER_ZONE=us-west1-b" >> vars.env
-          echo "GKE_CLUSTER_REGION=us-west1" >> vars.env
           echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
-          echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
-          echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
-          echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}-${{ matrix.type }}" >> vars.env
-          echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
-          echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
-          echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
           echo "PLUS_ENABLED=${{ matrix.type == 'plus' }}" >> vars.env
           echo "GINKGO_LABEL=" >> vars.env
           echo "NGF_VERSION=${{ needs.vars.outputs.version }}" >> vars.env
-          echo "GKE_NUM_NODES=12" >> vars.env
-          echo "GKE_MACHINE_TYPE=n2d-standard-16" >> vars.env
           echo "PLUS_USAGE_ENDPOINT=${{ secrets.JWT_PLUS_REPORTING_ENDPOINT }}" >> vars.env
 
-      - name: Setup license file for plus
-        if: matrix.type == 'plus'
-        env:
-          PLUS_LICENSE: ${{ secrets.JWT_PLUS_REPORTING }}
-        run: echo "${PLUS_LICENSE}" > license.jwt
-
-      - name: Create GKE cluster
-        working-directory: ./tests
-        run: make create-gke-cluster CI=true
-
-      - name: Create and setup VM
-        working-directory: ./tests
-        run: make create-and-setup-vm
+      - name: Create GKE cluster and VM
+        working-directory: ./tests/tofu
+        run: |
+          tofu apply --auto-approve
 
       - name: Run Tests
         working-directory: ./tests
@@ -153,12 +156,10 @@ jobs:
           path: tests/results/**/*-${{ matrix.type }}.*
 
       - name: Cleanup
-        working-directory: ./tests
+        working-directory: ./tests/tofu
         if: always()
         run: |
-          bash scripts/cleanup-vm.sh true
-          make delete-gke-cluster
-          rm -rf scripts/vars.env
+          tofu destroy --auto-approve
 
   pr-results:
     name: Open PR with results

.gitignore

@@ -61,3 +61,32 @@ internal/mode/static/nginx/modules/coverage
 # temporary files used for manifest generation
 config/base/deploy.yaml
 config/base/deploy.yaml.bak
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc

.pre-commit-config.yaml

@@ -95,6 +95,12 @@ repos:
           - "--skip-auto-generation=required,additionalProperties"
           - --append-newline
 
+# - repo: https://github.com/tofuutils/pre-commit-opentofu
+#   rev: v2.1.0
+#   hooks:
+#     - id: tofu_fmt
+#     - id: tofu_docs
+
 ci:
   skip: [golangci-lint-full, prettier, markdownlint-cli2, yamllint]
   autofix_prs: false

tests/Makefile

@@ -90,19 +90,11 @@ ifeq ($(PLUS_ENABLED),true)
 endif
 
 .PHONY: setup-gcp-and-run-nfr-tests
-setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
+setup-gcp-and-run-nfr-tests: create-gcp-resources nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests
 
 .PHONY: create-gke-cluster
-create-gke-cluster: ## Create a GKE cluster
-	./scripts/create-gke-cluster.sh $(CI)
-
-.PHONY: create-and-setup-vm
-create-and-setup-vm: ## Create and setup a GCP VM for tests
-	./scripts/create-and-setup-gcp-vm.sh
-
-.PHONY: create-gke-router
-create-gke-router: ## Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
-	./scripts/create-gke-router.sh
+create-gcp-resources: ## Create a GKE cluster and a GCP VM for tests
+	tofu -chdir=tofu apply
 
 .PHONY: sync-files-to-vm
 sync-files-to-vm: ## Syncs your local NGF files with the NGF repo on the VM
@@ -124,7 +116,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
 	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending \
-		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) \
+		--trace -r -vv --buildvcs --force-newlines $(GITHUB_OUTPUT) \
 		--label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
@@ -151,23 +143,8 @@ test-with-plus: PLUS_ENABLED=true
 test-with-plus: check-for-plus-usage-endpoint test ## Runs the functional tests for NGF with NGINX Plus on your default k8s cluster
 
 .PHONY: cleanup-gcp
-cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
-
-.PHONY: cleanup-router
-cleanup-router: ## Delete the GKE router
-	./scripts/cleanup-router.sh
-
-.PHONY: cleanup-vm
-cleanup-vm: ## Delete the test GCP VM and delete the firewall rule
-	./scripts/cleanup-vm.sh
-
-.PHONY: delete-gke-cluster
-delete-gke-cluster: ## Delete the GKE cluster
-	./scripts/delete-gke-cluster.sh
-
-.PHONY: add-local-ip-to-cluster
-add-local-ip-to-cluster: ## Add local IP to the GKE cluster master-authorized-networks
-	./scripts/add-local-ip-auth-networks.sh
+cleanup-gcp: ## Cleanup all GCP resources
+	tofu -chdir=tofu destroy
 
 HELM_PARAMETERS += --set nameOverride=nginx-gateway --set nginxGateway.kind=skip --set service.create=false --skip-schema-validation
 

tests/framework/timeout.go

@@ -44,7 +44,7 @@ func DefaultTimeoutConfig() TimeoutConfig {
 		UpdateTimeout:           60 * time.Second,
 		DeleteTimeout:           10 * time.Second,
 		DeleteNamespaceTimeout:  60 * time.Second,
-		GetTimeout:              10 * time.Second,
+		GetTimeout:              60 * time.Second,
 		ManifestFetchTimeout:    10 * time.Second,
 		RequestTimeout:          10 * time.Second,
 		ContainerRestartTimeout: 10 * time.Second,