[Release-1.27] June Testing Backports (#6157)

* Fix loadManifests function (#6058) Signed-off-by: Derek Nola <[email protected]> * Slim down E2E artifacts (#6097) * Don't use rke2-runtime.tar in e2e Signed-off-by: Derek Nola <[email protected]> * Don't upload rke2-runtime.tar Signed-off-by: Derek Nola <[email protected]> --------- Signed-off-by: Derek Nola <[email protected]> * Add custom golang setup action for better caching (#6144) Signed-off-by: Derek Nola <[email protected]> * Support MixedOS E2E local testing (#6137) * Consolidate CreateWindowsCluster test function Signed-off-by: Derek Nola <[email protected]> * Support local windows builds in mixedOS e2e test Signed-off-by: Derek Nola <[email protected]> * Add mixedos to E2E CI Signed-off-by: Derek Nola <[email protected]> * Simplify local cluster provisioning with libvirt native parallel bringup Signed-off-by: Derek Nola <[email protected]> * Don't run mixedOS on E2E Signed-off-by: Derek Nola <[email protected]> --------- Signed-off-by: Derek Nola <[email protected]> --------- Signed-off-by: Derek Nola <[email protected]>
rancher · Jun 10, 2024 · 15afec7 · 15afec7
1 parent 916f750
commit 15afec7
Show file tree

Hide file tree

Showing 17 changed files with 216 additions and 142 deletions.
diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml
@@ -0,0 +1,39 @@
+name: 'Setup golang with master only caching'
+description: 'A composite action that installs golang, but with a caching strategy that only updates the cache on master branch.'
+inputs:
+  go-version:
+    description: 'Override the version of Go to install.'
+    required: false
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-go@v5
+      if: inputs.go-version == ''
+      with:
+        go-version-file: 'go.mod'  # Just use whatever version is in the go.mod file
+        cache: ${{ github.ref == 'refs/heads/master' }}
+
+    - uses: actions/setup-go@v5
+      if: inputs.go-version != ''
+      with:
+        go-version: ${{ inputs.go-version }}
+        cache: ${{ github.ref == 'refs/heads/master' }}
+
+    - name: Prepare for go cache
+      if: ${{ github.ref != 'refs/heads/master' }}
+      shell: bash
+      run: |
+        echo "GO_CACHE=$(go env GOCACHE)" | tee -a "$GITHUB_ENV"
+        echo "GO_MODCACHE=$(go env GOMODCACHE)" | tee -a "$GITHUB_ENV"
+        echo "GO_VERSION=$(go env GOVERSION | tr -d 'go')" | tee -a "$GITHUB_ENV"
+
+    - name: Setup read-only cache
+      if: ${{ github.ref != 'refs/heads/master' }}
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          ${{ env.GO_MODCACHE }}
+          ${{ env.GO_CACHE }}
+        # Match the cache key to the setup-go action https://github.com/actions/setup-go/blob/main/src/cache-restore.ts#L34
+        key: setup-go-${{ runner.os }}-${{ env.ImageOS }}-go-${{ env.GO_VERSION }}-${{ hashFiles('go.sum') }}
diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml
@@ -35,23 +35,24 @@ jobs:
         set +x
         VERSION_GOLANG=$(echo $VERSION_GOLANG | sed 's/go//')
         echo "VERSION_GOLANG=${VERSION_GOLANG}" >> "$GITHUB_OUTPUT"
-    - uses: actions/setup-go@v5
+    - name: Install Go
+      uses: ./.github/actions/setup-go
       with:
         go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }}
-        cache-dependency-path: |
-          **/go.sum
-          **/go.mod
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
     - name: Install OS Packages
-      run: sudo apt-get install -y libarchive-tools
-    - name: Build RKE2 Binary and Runtime Image
+      run: sudo apt-get install -y libarchive-tools g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
+    # Can only upload from a single path, so we need to copy the binary to the image directory
+    # Additionally, we have a rke2-runtime.tar and a rke2-images.linux-amd64.tar.zst which are the same thing
+    # just compressed. We remove the rke2-runtime.tar as its not used by the install script.
+    - name: Build RKE2 Binary and Compressed Runtime Image
       run: |
         GOCOVER=true make package-bundle
         make package-image-runtime
         cp ./bin/rke2 ./build/images/rke2
-        cp ./dist/artifacts/rke2.linux-amd64.tar.gz ./build/images/rke2.linux-amd64.tar.gz
-    # Can only upload from a single path, so we need to copy the binary to the image directory
+        cp ./dist/artifacts/rke2.*-amd64.tar.gz ./build/images/
+        rm ./build/images/rke2-runtime.tar
     - name: Upload RKE2 Binary and Runtime Image
       uses: actions/upload-artifact@v4
       with:
@@ -160,8 +161,9 @@ jobs:
       run: |
         mkdir -p ./dist/artifacts
         mkdir -p ./build/images
+        mkdir -p ./bin
         mv ./artifacts/rke2-* ./build/images/
-        mv ./artifacts/rke2.linux-amd64.tar.gz ./dist/artifacts/rke2.linux-amd64.tar.gz
+        mv ./artifacts/rke2.*-amd64.tar.gz ./dist/artifacts
         sha256sum ./dist/artifacts/rke2.linux-amd64.tar.gz > ./dist/artifacts/sha256sum-amd64.txt
     - name: Run ${{ matrix.etest }} Test
       run: | 

diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml
@@ -31,9 +31,7 @@ jobs:
       with:
         fetch-depth: 1
     - name: Install Go
-      uses: actions/setup-go@v5
-      with: 
-        go-version-file: 'go.mod'
+      uses: ./.github/actions/setup-go
     - name: Run Unit Tests
       run: | 
         go test -coverpkg=./... -coverprofile=coverage.out ./pkg/... -run Unit

diff --git a/scripts/build-image-runtime b/scripts/build-image-runtime
@@ -29,6 +29,11 @@ if [ "${GOARCH}" != "s390x" ] && [ "${GOARCH}" != "arm64" ]; then
       --target windows-runtime \
       --file Dockerfile.windows \
       .
+  # Only ever used in its compressed form for e2e tests
+  mkdir -p build/images
+  docker image save \
+    ${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-windows-${GOARCH} | \
+    zstd -T0 -16 -f --long=25 --no-progress - -o build/images/${PROG}-images.windows-${GOARCH}.tar.zst    
 fi
 mkdir -p build/images
 docker image save \

diff --git a/tests/e2e/ciliumnokp/Vagrantfile b/tests/e2e/ciliumnokp/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2310', 'generic/ubuntu2310'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/dnscache/Vagrantfile b/tests/e2e/dnscache/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/dualstack/Vagrantfile b/tests/e2e/dualstack/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||

diff --git a/tests/e2e/mixedos/Vagrantfile b/tests/e2e/mixedos/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
@@ -22,24 +22,19 @@ def provision(vm, role, role_num, node_num)
   load vagrant_defaults if File.exist?(vagrant_defaults)
 
   defaultOSConfigure(vm)
-
+ 
   if role.include?("windows")
     vm.provision "shell", path: "../scripts/install-bgp.ps1"
-    if !RELEASE_VERSION.empty?
-      install_type = "Version=#{RELEASE_VERSION}"
+    if RELEASE_VERSION == "skip"
+      install_type = "-ArtifactPath 'C:\tmp'" 
+    elsif !RELEASE_VERSION.empty?
+      install_type = "-Version #{RELEASE_VERSION}"
     else 
       vm.provision "shell", path: "../scripts/latest_commit.ps1", args: [GITHUB_BRANCH, "./rke2_commits.txt"]
-      install_type = "Commit=(Get-Content -TotalCount 1 ./rke2_commits.txt)"
+      install_type = "-Commit (Get-Content -TotalCount 1 ./rke2_commits.txt)"
     end
   else
-    if !RELEASE_VERSION.empty?
-      install_type = "INSTALL_RKE2_VERSION=#{RELEASE_VERSION}"
-    else 
-      # Grabs the last 5 commit SHA's from the given branch, then purges any commits that do not have a passing CI build
-      vm.provision "shell", path: "../scripts/latest_commit.sh", args: [GITHUB_BRANCH, "/tmp/rke2_commits"]
-      install_type = "INSTALL_RKE2_COMMIT=$(head\ -n\ 1\ /tmp/rke2_commits)"
-    end
-    vm.provision "shell", inline: "ping -c 2 rke2.io"
+    install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
     vm.provision "Create Calico Manifest", type: "shell", path: "../scripts/calico_manifest.sh", args: [ "#{NETWORK_PREFIX}.1" ]
   end
 

diff --git a/tests/e2e/mixedos/mixedos_test.go b/tests/e2e/mixedos/mixedos_test.go
@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 
@@ -16,11 +15,10 @@ import (
 // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "operating system for linux nodes")
 var serverCount = flag.Int("serverCount", 1, "number of server nodes")
-var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent nodes")
+var linuxAgentCount = flag.Int("linuxAgentCount", 0, "number of linux agent nodes")
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
-
-const defaultWindowsOS = "jborean93/WindowsServer2022"
+var local = flag.Bool("local", false, "deploy a locally built RKE2")
 
 func Test_E2EMixedOSValidation(t *testing.T) {
 	flag.Parse()
@@ -29,41 +27,6 @@ func Test_E2EMixedOSValidation(t *testing.T) {
 	RunSpecs(t, "Validate MixedOS Test Suite", suiteConfig, reporterConfig)
 }
 
-func createMixedCluster(nodeOS string, serverCount, linuxAgentCount, windowsAgentCount int) ([]string, []string, []string, error) {
-	serverNodeNames := []string{}
-	for i := 0; i < serverCount; i++ {
-		serverNodeNames = append(serverNodeNames, "server-"+strconv.Itoa(i))
-	}
-	linuxAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		linuxAgentNames = append(linuxAgentNames, "linux-agent-"+strconv.Itoa(i))
-	}
-	windowsAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		windowsAgentNames = append(windowsAgentNames, "windows-agent-"+strconv.Itoa(i))
-	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(linuxAgentNames, " ") + " " + strings.Join(windowsAgentNames, " ")
-	nodeRoles = strings.TrimSpace(nodeRoles)
-	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+linuxAgentCount)
-	nodeBoxes += strings.Repeat(defaultWindowsOS+" ", windowsAgentCount)
-	nodeBoxes = strings.TrimSpace(nodeBoxes)
-
-	var testOptions string
-	for _, env := range os.Environ() {
-		if strings.HasPrefix(env, "E2E_") {
-			testOptions += " " + env
-		}
-	}
-
-	cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, testOptions)
-	fmt.Println(cmd)
-	if _, err := e2e.RunCommand(cmd); err != nil {
-		fmt.Println("Error Creating Cluster", err)
-		return nil, nil, nil, err
-	}
-	return serverNodeNames, linuxAgentNames, windowsAgentNames, nil
-}
-
 var (
 	kubeConfigFile    string
 	serverNodeNames   []string
@@ -76,7 +39,11 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 
 	It("Starts up with no issues", func() {
 		var err error
-		serverNodeNames, linuxAgentNames, windowsAgentNames, err = createMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		if *local {
+			serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateLocalMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		} else {
+			serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		}
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)

diff --git a/tests/e2e/mixedosbgp/Vagrantfile b/tests/e2e/mixedosbgp/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
 ["server-0", "linux-agent-0", "windows-agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||

diff --git a/tests/e2e/mixedosbgp/mixedosbgp_test.go b/tests/e2e/mixedosbgp/mixedosbgp_test.go
@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"testing"
 
@@ -20,50 +19,13 @@ var linuxAgentCount = flag.Int("linuxAgentCount", 1, "number of linux agent node
 var windowsAgentCount = flag.Int("windowsAgentCount", 1, "number of windows agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
 
-const defaultWindowsOS = "jborean93/WindowsServer2022"
-
 func Test_E2EMixedOSBGPValidation(t *testing.T) {
 	flag.Parse()
 	RegisterFailHandler(Fail)
 	suiteConfig, reporterConfig := GinkgoConfiguration()
 	RunSpecs(t, "Validate MixedOS BGP Test Suite", suiteConfig, reporterConfig)
 }
 
-func createMixedCluster(nodeOS string, serverCount, linuxAgentCount, windowsAgentCount int) ([]string, []string, []string, error) {
-	serverNodeNames := []string{}
-	for i := 0; i < serverCount; i++ {
-		serverNodeNames = append(serverNodeNames, "server-"+strconv.Itoa(i))
-	}
-	linuxAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		linuxAgentNames = append(linuxAgentNames, "linux-agent-"+strconv.Itoa(i))
-	}
-	windowsAgentNames := []string{}
-	for i := 0; i < linuxAgentCount; i++ {
-		windowsAgentNames = append(windowsAgentNames, "windows-agent-"+strconv.Itoa(i))
-	}
-	nodeRoles := strings.Join(serverNodeNames, " ") + " " + strings.Join(linuxAgentNames, " ") + " " + strings.Join(windowsAgentNames, " ")
-	nodeRoles = strings.TrimSpace(nodeRoles)
-	nodeBoxes := strings.Repeat(nodeOS+" ", serverCount+linuxAgentCount)
-	nodeBoxes += strings.Repeat(defaultWindowsOS+" ", windowsAgentCount)
-	nodeBoxes = strings.TrimSpace(nodeBoxes)
-
-	var testOptions string
-	for _, env := range os.Environ() {
-		if strings.HasPrefix(env, "E2E_") {
-			testOptions += " " + env
-		}
-	}
-
-	cmd := fmt.Sprintf("NODE_ROLES=\"%s\" NODE_BOXES=\"%s\" %s vagrant up &> vagrant.log", nodeRoles, nodeBoxes, testOptions)
-	fmt.Println(cmd)
-	if _, err := e2e.RunCommand(cmd); err != nil {
-		fmt.Println("Error Creating Cluster", err)
-		return nil, nil, nil, err
-	}
-	return serverNodeNames, linuxAgentNames, windowsAgentNames, nil
-}
-
 var (
 	kubeConfigFile    string
 	serverNodeNames   []string
@@ -76,7 +38,7 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 
 	It("Starts up with no issues", func() {
 		var err error
-		serverNodeNames, linuxAgentNames, windowsAgentNames, err = createMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
+		serverNodeNames, linuxAgentNames, windowsAgentNames, err = e2e.CreateMixedCluster(*nodeOS, *serverCount, *linuxAgentCount, *windowsAgentCount)
 		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)
@@ -91,7 +53,7 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 		Eventually(func(g Gomega) {
 			nodes, err := e2e.ParseNodes(kubeConfigFile, false)
 			g.Expect(err).NotTo(HaveOccurred())
-			g.Expect(len(serverNodeNames)+len(linuxAgentNames)+len(windowsAgentNames)).Should(Equal(len(nodes)))
+			g.Expect(len(serverNodeNames) + len(linuxAgentNames) + len(windowsAgentNames)).Should(Equal(len(nodes)))
 			for _, node := range nodes {
 				g.Expect(node.Status).Should(Equal("Ready"))
 			}
@@ -160,7 +122,6 @@ var _ = Describe("Verify Basic Cluster Creation", Ordered, func() {
 		Eventually(func() (string, error) {
 			return e2e.RunCommand(cmd)
 		}, "20s", "3s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
-
 	})
 	It("Runs the mixed os sonobuoy plugin", func() {
 		cmd := "sonobuoy run --kubeconfig=/etc/rancher/rke2/rke2.yaml --plugin my-sonobuoy-plugins/mixed-workload-e2e/mixed-workload-e2e.yaml --aggregator-node-selector kubernetes.io/os:linux --wait"

diff --git a/tests/e2e/multus/Vagrantfile b/tests/e2e/multus/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] || ["server-0", "agent-0" ])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] || ['generic/ubuntu2004', 'generic/ubuntu2004'])
 GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")

diff --git a/tests/e2e/splitserver/Vagrantfile b/tests/e2e/splitserver/Vagrantfile
@@ -1,4 +1,4 @@
-ENV['VAGRANT_NO_PARALLEL'] = 'no'
+ENV['VAGRANT_NO_PARALLEL'] = ENV['E2E_STANDUP_PARALLEL'] ? nil : 'no'
 NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-etcd-0", "server-cp-0", "server-cp-1", "agent-0"])
 NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||