Test automation: zero-downtime upgrades

.gitignore

@@ -29,6 +29,9 @@ build/.out
 build/out
 dist/
 
+# Test artifacts
+tests/**/*.csv
+
 # Node modules
 node_modules/
 

.markdownlint-cli2.yaml

@@ -18,3 +18,4 @@ config:
 # Define glob expressions to ignore
 ignores:
   - ".github/"
+  - "tests/results/"

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        exclude: (^tests/results/)
       - id: end-of-file-fixer
       - id: check-yaml
         args: [--allow-multiple-documents]

tests/Makefile

@@ -2,7 +2,8 @@ TAG = edge
 PREFIX = nginx-gateway-fabric
 NGINX_PREFIX = $(PREFIX)/nginx
 PULL_POLICY=Never
-GW_API_VERSION ?= 1.0.0
+GW_API_PREV_VERSION ?= 1.0.0 ## Supported Gateway API version from previous NGF release
+GW_API_VERSION ?= 1.0.0 ## Supported Gateway API version for NGF under test
 K8S_VERSION ?= latest ## Expected format: 1.24 (major.minor) or latest
 GW_SERVICE_TYPE=NodePort
 GW_SVC_GKE_INTERNAL=false
@@ -30,7 +31,8 @@ load-images: ## Load NGF and NGINX images on configured kind cluster
 	kind load docker-image $(PREFIX):$(TAG) $(NGINX_PREFIX):$(TAG)
 
 test: ## Run the system tests against your default k8s cluster
-	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) --image-tag=$(TAG) \
+	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) \
+		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
 		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
@@ -46,9 +48,20 @@ run-tests-on-vm: ## Run the tests on a GCP VM
 create-and-setup-vm: ## Create and setup a GCP VM for tests
 	bash scripts/create-and-setup-gcp-vm.sh
 
-.PHONY: create-vm-and-run-tests
-create-vm-and-run-tests: create-and-setup-vm run-tests-on-vm ## Create and setup a GCP VM for tests and run the tests
-
 .PHONY: cleanup-vm
 cleanup-vm: ## Delete the test GCP VM and delete the firewall rule
 	bash scripts/cleanup-vm.sh
+
+.PHONY: create-gke-router
+create-gke-router: ## Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
+	bash scripts/create-gke-router.sh
+
+.PHONY: cleanup-router
+cleanup-router: ## Delete the GKE router
+	bash scripts/cleanup-router.sh
+
+.PHONY: setup-gcp-and-run-tests
+setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the tests
+
+.PHONY: cleanup-gcp
+cleanup-gcp: cleanup-router cleanup-vm ## Cleanup all GCP resources

tests/README.md

@@ -38,14 +38,17 @@ make
 
 ```text
 build-images                   Build NGF and NGINX images
-cleanup-vm                     Delete the test GCP VM and the firewall rule
+cleanup-gcp                    Cleanup all GCP resources
+cleanup-router                 Delete the GKE router
+cleanup-vm                     Delete the test GCP VM and delete the firewall rule
 create-and-setup-vm            Create and setup a GCP VM for tests
+create-gke-router              Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
 create-kind-cluster            Create a kind cluster
-create-vm-and-run-tests        Create and setup a GCP VM for tests and run the tests
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
 load-images                    Load NGF and NGINX images on configured kind cluster
 run-tests-on-vm                Run the tests on a GCP VM
+setup-gcp-and-run-tests        Create and setup a GKE router and GCP VM for tests and run the tests
 test                           Run the system tests against your default k8s cluster
 ```
 
@@ -101,15 +104,24 @@ make test TAG=$(whoami)
 This step only applies if you would like to run the tests from a GCP based VM.
 
 Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
-required env vars. The `GKE_CLUSTER_ZONE` needs to be the zone of your GKE cluster, and `GKE_SVC_ACCOUNT` needs to be
-the name of a service account that has Kubernetes admin permissions.
+required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
 
-To create and setup the VM (including creating a firewall rule allowing SSH access from your local machine, and
-optionally adding the VM IP to the `master-authorized-networks` list of your GKE cluster if
-`ADD_VM_IP_AUTH_NETWORKS` is set to `true`) and run the tests, run the following
+In order to run the tests in GCP, you need a few things:
+
+- GKE router to allow egress traffic (used by upgrade tests for pulling images from Github)
+  - this assumes that your GKE cluster is using private nodes. If using public nodes, you don't need this.
+- GCP VM and firewall rule to send ingress traffic to GKE
+
+To set up the GCP environment with the router and VM and then run the tests, run the following command:
 
 ```makefile
-make create-vm-and-run-tests
+make setup-gcp-and-run-tests
+```
+
+If you just need a VM and no router (this will not run the tests):
+
+```makefile
+make create-and-setup-vm
 ```
 
 To use an existing VM to run the tests, run the following
@@ -179,7 +191,17 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     make delete-kind-cluster
     ```
 
-2. Delete the cloud VM and cleanup the firewall rule, if required
+2. Delete the GCP components (GKE router, VM, and firewall rule), if required
+
+    ```makefile
+    make cleanup-gcp
+    ```
+
+    or
+
+    ```makefile
+    make cleanup-router
+    ```
 
     ```makefile
     make cleanup-vm

tests/framework/load.go

@@ -1,10 +1,10 @@
 package framework
 
 import (
-	"fmt"
+	"context"
+	"crypto/tls"
+	"net"
 	"net/http"
-	"net/url"
-	"os"
 	"time"
 
 	vegeta "github.com/tsenart/vegeta/v12/lib"
@@ -31,38 +31,57 @@ func convertTargetToVegetaTarget(targets []Target) []vegeta.Target {
 	return vegTargets
 }
 
+// LoadTestConfig is the configuration to run a load test.
+type LoadTestConfig struct {
+	Description string
+	Proxy       string
+	ServerName  string
+	Targets     []Target
+	Rate        int
+	Duration    time.Duration
+}
+
+// Metrics is a wrapper around the vegeta Metrics.
+type Metrics struct {
+	vegeta.Metrics
+}
+
 // RunLoadTest uses Vegeta to send traffic to the provided Targets at the given rate for the given duration and writes
 // the results to the provided file
-func RunLoadTest(
-	targets []Target,
-	rate int,
-	duration time.Duration,
-	desc string,
-	outFile *os.File,
-	proxy string,
-) error {
-	vegTargets := convertTargetToVegetaTarget(targets)
+func RunLoadTest(cfg LoadTestConfig) (vegeta.Results, Metrics) {
+	vegTargets := convertTargetToVegetaTarget(cfg.Targets)
 	targeter := vegeta.NewStaticTargeter(vegTargets...)
-	proxyURL, err := url.Parse(proxy)
-	if err != nil {
-		return fmt.Errorf("error getting proxy URL: %w", err)
+
+	dialer := &net.Dialer{
+		LocalAddr: &net.TCPAddr{IP: vegeta.DefaultLocalAddr.IP, Zone: vegeta.DefaultLocalAddr.Zone},
+		KeepAlive: 30 * time.Second,
 	}
 
-	attacker := vegeta.NewAttacker(
-		vegeta.Proxy(http.ProxyURL(proxyURL)),
-	)
+	httpClient := http.Client{
+		Timeout: vegeta.DefaultTimeout,
+		Transport: &http.Transport{
+			DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				return dialer.DialContext(ctx, network, cfg.Proxy)
+			},
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true, //nolint:gosec // self-signed cert for testing
+				ServerName:         cfg.ServerName,
+			},
+			MaxIdleConnsPerHost: vegeta.DefaultConnections,
+			MaxConnsPerHost:     vegeta.DefaultMaxConnections,
+		},
+	}
+
+	attacker := vegeta.NewAttacker(vegeta.Client(&httpClient))
 
-	r := vegeta.Rate{Freq: rate, Per: time.Second}
+	r := vegeta.Rate{Freq: cfg.Rate, Per: time.Second}
+	var results vegeta.Results
 	var metrics vegeta.Metrics
-	for res := range attacker.Attack(targeter, r, duration, desc) {
+	for res := range attacker.Attack(targeter, r, cfg.Duration, cfg.Description) {
+		results = append(results, *res)
 		metrics.Add(res)
 	}
 	metrics.Close()
 
-	reporter := vegeta.NewTextReporter(&metrics)
-
-	if err = reporter.Report(outFile); err != nil {
-		return fmt.Errorf("error reporting results: %w", err)
-	}
-	return nil
+	return results, Metrics{metrics}
 }

tests/framework/ngf.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -83,39 +84,50 @@ func InstallGatewayAPI(
 	return nil, nil
 }
 
+// UninstallGatewayAPI uninstalls the specified version of the Gateway API resources.
+func UninstallGatewayAPI(apiVersion, k8sVersion string) ([]byte, error) {
+	apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion)
+
+	if webhookRequired(k8sVersion) {
+		webhookPath := fmt.Sprintf("%s/v%s/webhook-install.yaml", gwInstallBasePath, apiVersion)
+
+		if output, err := exec.Command("kubectl", "delete", "-f", webhookPath).CombinedOutput(); err != nil {
+			return output, err
+		}
+	}
+
+	output, err := exec.Command("kubectl", "delete", "-f", apiPath).CombinedOutput()
+	if err != nil && !strings.Contains(string(output), "not found") {
+		return output, err
+	}
+
+	return nil, nil
+}
+
 // InstallNGF installs NGF.
 func InstallNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
 	args := []string{
 		"install", cfg.ReleaseName, cfg.ChartPath, "--create-namespace", "--namespace", cfg.Namespace, "--wait",
 	}
 
-	if cfg.NgfImageRepository != "" {
-		args = append(args, formatValueSet("nginxGateway.image.repository", cfg.NgfImageRepository)...)
-		if cfg.ImageTag != "" {
-			args = append(args, formatValueSet("nginxGateway.image.tag", cfg.ImageTag)...)
-		}
-		if cfg.ImagePullPolicy != "" {
-			args = append(args, formatValueSet("nginxGateway.image.pullPolicy", cfg.ImagePullPolicy)...)
-		}
-	}
+	args = append(args, setImageArgs(cfg)...)
+	fullArgs := append(args, extraArgs...)
 
-	if cfg.NginxImageRepository != "" {
-		args = append(args, formatValueSet("nginx.image.repository", cfg.NginxImageRepository)...)
-		if cfg.ImageTag != "" {
-			args = append(args, formatValueSet("nginx.image.tag", cfg.ImageTag)...)
-		}
-		if cfg.ImagePullPolicy != "" {
-			args = append(args, formatValueSet("nginx.image.pullPolicy", cfg.ImagePullPolicy)...)
-		}
+	return exec.Command("helm", fullArgs...).CombinedOutput()
+}
+
+// UpgradeNGF upgrades NGF. CRD upgrades assume the chart is local.
+func UpgradeNGF(cfg InstallationConfig, extraArgs ...string) ([]byte, error) {
+	crdPath := filepath.Join(cfg.ChartPath, "crds")
+	if output, err := exec.Command("kubectl", "apply", "-f", crdPath).CombinedOutput(); err != nil {
+		return output, err
 	}
 
-	if cfg.ServiceType != "" {
-		args = append(args, formatValueSet("service.type", cfg.ServiceType)...)
-		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
-			args = append(args, formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
-		}
+	args := []string{
+		"upgrade", cfg.ReleaseName, cfg.ChartPath, "--namespace", cfg.Namespace, "--wait",
 	}
 
+	args = append(args, setImageArgs(cfg)...)
 	fullArgs := append(args, extraArgs...)
 
 	return exec.Command("helm", fullArgs...).CombinedOutput()
@@ -128,7 +140,7 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro
 	}
 
 	output, err := exec.Command("helm", args...).CombinedOutput()
-	if err != nil {
+	if err != nil && !strings.Contains(string(output), "release: not found") {
 		return output, err
 	}
 
@@ -157,19 +169,37 @@ func UninstallNGF(cfg InstallationConfig, k8sClient client.Client) ([]byte, erro
 	return nil, nil
 }
 
-// UninstallGatewayAPI uninstalls the specified version of the Gateway API resources.
-func UninstallGatewayAPI(apiVersion, k8sVersion string) ([]byte, error) {
-	apiPath := fmt.Sprintf("%s/v%s/standard-install.yaml", gwInstallBasePath, apiVersion)
+func setImageArgs(cfg InstallationConfig) []string {
+	var args []string
 
-	if webhookRequired(k8sVersion) {
-		webhookPath := fmt.Sprintf("%s/v%s/webhook-install.yaml", gwInstallBasePath, apiVersion)
+	if cfg.NgfImageRepository != "" {
+		args = append(args, formatValueSet("nginxGateway.image.repository", cfg.NgfImageRepository)...)
+		if cfg.ImageTag != "" {
+			args = append(args, formatValueSet("nginxGateway.image.tag", cfg.ImageTag)...)
+		}
+		if cfg.ImagePullPolicy != "" {
+			args = append(args, formatValueSet("nginxGateway.image.pullPolicy", cfg.ImagePullPolicy)...)
+		}
+	}
 
-		if output, err := exec.Command("kubectl", "delete", "-f", webhookPath).CombinedOutput(); err != nil {
-			return output, err
+	if cfg.NginxImageRepository != "" {
+		args = append(args, formatValueSet("nginx.image.repository", cfg.NginxImageRepository)...)
+		if cfg.ImageTag != "" {
+			args = append(args, formatValueSet("nginx.image.tag", cfg.ImageTag)...)
+		}
+		if cfg.ImagePullPolicy != "" {
+			args = append(args, formatValueSet("nginx.image.pullPolicy", cfg.ImagePullPolicy)...)
+		}
+	}
+
+	if cfg.ServiceType != "" {
+		args = append(args, formatValueSet("service.type", cfg.ServiceType)...)
+		if cfg.ServiceType == "LoadBalancer" && cfg.IsGKEInternalLB {
+			args = append(args, formatValueSet(`service.annotations.networking\.gke\.io\/load-balancer-type`, "Internal")...)
 		}
 	}
 
-	return exec.Command("kubectl", "delete", "-f", apiPath).CombinedOutput()
+	return args
 }
 
 func formatValueSet(key, value string) []string {