chore: add integration tests for image-cache

Provide separate `integration/image-cache` tag. Closes siderolabs#9860 Signed-off-by: Dmitriy Matrenichev <[email protected]>
DmitriyMV · Dec 5, 2024 · 42ecae3 · 42ecae3
1 parent dd61ad8
commit 42ecae3
Show file tree

Hide file tree

Showing 9 changed files with 229 additions and 21 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-12-04T15:25:22Z by kres 232fe63.
+# Generated on 2024-12-05T11:38:21Z by kres 232fe63.
 
 name: default
 concurrency:
@@ -1606,6 +1606,105 @@ jobs:
           TF_SCRIPT_DIR: _out/contrib
         run: |
           make e2e-cloud-tf
+  integration-image-cache:
+    permissions:
+      actions: read
+      contents: write
+      issues: read
+      packages: write
+      pull-requests: read
+    runs-on:
+      - self-hosted
+      - talos
+    if: contains(fromJSON(needs.default.outputs.labels), 'integration/image-cache')
+    needs:
+      - default
+    steps:
+      - name: gather-system-info
+        id: system-info
+        uses: kenchan0130/[email protected]
+        continue-on-error: true
+      - name: print-system-info
+        run: |
+          MEMORY_GB=$((${{ steps.system-info.outputs.totalmem }}/1024/1024/1024))
+
+          OUTPUTS=(
+            "CPU Core: ${{ steps.system-info.outputs.cpu-core }}"
+            "CPU Model: ${{ steps.system-info.outputs.cpu-model }}"
+            "Hostname: ${{ steps.system-info.outputs.hostname }}"
+            "NodeName: ${NODE_NAME}"
+            "Kernel release: ${{ steps.system-info.outputs.kernel-release }}"
+            "Kernel version: ${{ steps.system-info.outputs.kernel-version }}"
+            "Name: ${{ steps.system-info.outputs.name }}"
+            "Platform: ${{ steps.system-info.outputs.platform }}"
+            "Release: ${{ steps.system-info.outputs.release }}"
+            "Total memory: ${MEMORY_GB} GB"
+          )
+
+          for OUTPUT in "${OUTPUTS[@]}";do
+            echo "${OUTPUT}"
+          done
+        continue-on-error: true
+      - name: checkout
+        uses: actions/checkout@v4
+      - name: Unshallow
+        run: |
+          git fetch --prune --unshallow
+      - name: Set up Docker Buildx
+        id: setup-buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: remote
+          endpoint: tcp://buildkit-amd64.ci.svc.cluster.local:1234
+        timeout-minutes: 10
+      - name: Download artifacts
+        if: github.event_name != 'schedule'
+        uses: actions/download-artifact@v4
+        with:
+          name: talos-artifacts
+          path: _out
+      - name: Fix artifact permissions
+        if: github.event_name != 'schedule'
+        run: |
+          xargs -a _out/executable-artifacts -I {} chmod +x {}
+      - name: ci-temp-release-tag
+        if: github.event_name != 'schedule'
+        run: |
+          make ci-temp-release-tag
+      - name: uki-certs
+        if: github.event_name == 'schedule'
+        env:
+          PLATFORM: linux/amd64
+        run: |
+          make uki-certs
+      - name: image-cache
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          MORE_IMAGES: alpine
+          PLATFORM: linux/amd64,linux/arm64
+          PUSH: "true"
+        run: |
+          make cache-create
+      - name: e2e-image-cache
+        env:
+          GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          REGISTRY_MIRROR_FLAGS: "no"
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml'
+          WITH_ISO: "true"
+        run: |
+          sudo -E make e2e-qemu
+      - name: save artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: talos-logs-integration-image-cache
+          path: |-
+            /tmp/logs-*.tar.gz
+            /tmp/support-*.zip
+          retention-days: "5"
   integration-image-factory:
     permissions:
       actions: read

diff --git a/.kres.yaml b/.kres.yaml
@@ -72,6 +72,7 @@ spec:
     - integration-images
     - integration-reproducibility-test
     - integration-cloud-images
+    - integration-image-cache
     - integration-image-factory
     - integration-aws
     - integration-aws-nvidia-oss
@@ -1514,6 +1515,60 @@ spec:
             PLATFORM: linux/amd64,linux/arm64
             IMAGE_REGISTRY: registry.dev.siderolabs.io
         - name: cloud-images
+    - name: integration-image-cache
+      buildxOptions:
+        enabled: true
+      depends:
+        - default
+      runners:
+        - self-hosted
+        - talos
+      triggerLabels:
+        - integration/image-cache
+      steps:
+        - name: download-artifacts
+          conditions:
+            - not-on-schedule
+          artifactStep:
+            type: download
+            artifactName: talos-artifacts
+            artifactPath: _out
+        - name: ci-temp-release-tag
+          conditions:
+            - not-on-schedule
+        - name: uki-certs
+          conditions:
+            - only-on-schedule
+          environment:
+            PLATFORM: linux/amd64
+        - name: image-cache
+          command: cache-create
+          environment:
+            PLATFORM: linux/amd64,linux/arm64
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
+            PUSH: true
+            MORE_IMAGES: "alpine"
+        - name: e2e-image-cache
+          command: e2e-qemu
+          withSudo: true
+          environment:
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
+            GITHUB_STEP_NAME: ${{ github.job}}-e2e-image-cache
+            SHORT_INTEGRATION_TEST: "yes"
+            WITH_CONFIG_PATCH: '@hack/test/patches/image-cache.yaml'
+            REGISTRY_MIRROR_FLAGS: "no"
+            WITH_ISO: "true"
+            VIA_MAINTENANCE_MODE: "true"
+        - name: save-talos-logs
+          conditions:
+            - always
+          artifactStep:
+            type: upload
+            artifactName: talos-logs-integration-image-cache
+            disableExecutableListGeneration: true
+            artifactPath: /tmp/logs-*.tar.gz
+            additionalArtifacts:
+              - "/tmp/support-*.zip"
     - name: integration-image-factory
       buildxOptions:
         enabled: true

diff --git a/Makefile b/Makefile
@@ -125,6 +125,7 @@ SHORT_INTEGRATION_TEST ?=
 CUSTOM_CNI_URL ?=
 INSTALLER_ARCH ?= all
 IMAGER_ARGS ?=
+MORE_IMAGES ?=
 
 CGO_ENABLED ?= 0
 GO_BUILDFLAGS ?=
@@ -459,6 +460,12 @@ uki-certs: talosctl ## Generate test certificates for SecureBoot/PCR Signing
 	@$(TALOSCTL_EXECUTABLE) gen secureboot pcr
 	@$(TALOSCTL_EXECUTABLE) gen secureboot database
 
+.PHONY: cache-create
+cache-create: installer imager ## Generate image cache.
+	@( $(TALOSCTL_EXECUTABLE) images default | grep -v 'siderolabs/installer'; echo "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)"; echo $(MORE_IMAGES) | tr ';' '\n' ) | $(TALOSCTL_EXECUTABLE) images cache-create --image-cache-path=/tmp/cache.tar --images=- --force
+	@crane push /tmp/cache.tar $(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG)
+	@$(MAKE) image-iso IMAGER_ARGS="--image-cache=$(REGISTRY_AND_USERNAME)/image-cache:$(IMAGE_TAG) --extra-kernel-arg='console=ttyS0'"
+
 # Code Quality
 
 api-descriptors: ## Generates API descriptors used to detect breaking API changes.

diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh
@@ -131,6 +131,14 @@ case "${WITH_CONFIG_PATCH:-false}" in
     ;;
 esac
 
+case "${WITH_ISO:-false}" in
+  false)
+    ;;
+  *)
+    QEMU_FLAGS+=("--iso-path=${ARTIFACTS}/metal-amd64.iso")
+    ;;
+esac
+
 case "${WITH_CONFIG_PATCH_WORKER:-false}" in
   false)
     ;;

diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh
@@ -142,6 +142,12 @@ function dump_cluster_state {
 }
 
 function build_registry_mirrors {
+  if [[ "${REGISTRY_MIRROR_FLAGS:-yes}" == "no" ]]; then
+    REGISTRY_MIRROR_FLAGS=()
+
+    return
+  fi
+
   if [[ "${CI:-false}" == "true" ]]; then
     REGISTRY_MIRROR_FLAGS=()
 
@@ -151,9 +157,6 @@ function build_registry_mirrors {
 
       REGISTRY_MIRROR_FLAGS+=("--registry-mirror=${registry}=http://${addr}:5000")
     done
-  else
-    # use the value from the environment, if present
-    REGISTRY_MIRROR_FLAGS=("${REGISTRY_MIRROR_FLAGS:-}")
   fi
 }
 

diff --git a/hack/test/patches/image-cache.yaml b/hack/test/patches/image-cache.yaml
@@ -0,0 +1,25 @@
+machine:
+  features:
+    imageCache:
+      localEnabled: true
+  registries:
+    mirrors:
+      "*":
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+      k8s.gcr.io:
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+      registry.k8s.io:
+        skipFallback: true
+        endpoints:
+        - http://172.20.0.251:65000
+---
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: IMAGECACHE
+provisioning:
+  diskSelector:
+    match: 'system_disk'
diff --git a/internal/integration/cli/image.go b/internal/integration/cli/image.go
@@ -52,6 +52,13 @@ func (suite *ImageSuite) TestPull() {
 	node := suite.RandomDiscoveredNodeInternalIP()
 	image := "registry.k8s.io/kube-apiserver:v1.27.0"
 
+	stdout, _ := suite.RunCLI([]string{"get", "imagecacheconfig", "--output", "jsonpath='{.spec.copyStatus}'"})
+	if strings.Contains(stdout, "ready") {
+		suite.T().Logf("skipping as the image cache is present")
+
+		return
+	}
+
 	suite.RunCLI([]string{"image", "pull", "--nodes", node, image},
 		base.StdoutEmpty(),
 	)

diff --git a/internal/integration/cli/list.go b/internal/integration/cli/list.go
@@ -40,8 +40,6 @@ func (suite *ListSuite) TestSuccess() {
 
 // TestDepth tests various combinations of --recurse and --depth flags.
 func (suite *ListSuite) TestDepth() {
-	suite.T().Parallel()
-
 	node := suite.RandomDiscoveredNodeInternalIP(machine.TypeControlPlane)
 
 	// checks that enough separators are encountered in the output
@@ -96,7 +94,6 @@ func (suite *ListSuite) TestDepth() {
 		{separators: 5, flags: []string{"--recurse=true"}},
 	} {
 		suite.Run(strings.Join(test.flags, ","), func() {
-			suite.T().Parallel()
 			runAndCheck(suite.T(), test.separators, test.flags...)
 		})
 	}

diff --git a/pkg/imager/cache/cache.go b/pkg/imager/cache/cache.go
@@ -37,6 +37,15 @@ const (
 	manifestsDir = "manifests"
 )
 
+// rewriteRegistry name back to workaround https://github.com/google/go-containerregistry/pull/69.
+func rewriteRegistry(registryName, origRef string) string {
+	if registryName == name.DefaultRegistry && !strings.HasPrefix(origRef, name.DefaultRegistry+"/") {
+		return "docker.io"
+	}
+
+	return registryName
+}
+
 // Generate generates a cache tarball from the given images.
 //
 //nolint:gocyclo,cyclop
@@ -65,9 +74,7 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa
 		return err
 	}
 
-	nameOptions := []name.Option{
-		name.StrictValidation,
-	}
+	var nameOptions []name.Option
 
 	craneOpts := []crane.Option{
 		crane.WithAuthFromKeychain(
@@ -99,17 +106,17 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa
 			return fmt.Errorf("parsing reference %q: %w", src, err)
 		}
 
-		referenceDir := filepath.Join(tmpDir, manifestsDir, ref.Context().RegistryStr(), ref.Context().RepositoryStr(), "reference")
-		digestDir := filepath.Join(tmpDir, manifestsDir, ref.Context().RegistryStr(), ref.Context().RepositoryStr(), "digest")
+		referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), ref.Context().RepositoryStr(), "reference")
+		digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), ref.Context().RepositoryStr(), "digest")
 
-		// get the tag from the reference (if it's there)
-		var tag name.Tag
+		// if the reference was parsed as a tag, use it
+		tag, ok := ref.(name.Tag)
 
-		base, _, ok := strings.Cut(src, "@")
 		if !ok {
-			tag, _ = name.NewTag(src, nameOptions...) //nolint:errcheck
-		} else {
-			tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck
+			if base, _, ok := strings.Cut(src, "@"); ok {
+				// if the reference was a digest, but contained a tag, re-parse it
+				tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck
+			}
 		}
 
 		if err = os.MkdirAll(referenceDir, 0o755); err != nil {
@@ -121,19 +128,19 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa
 		}
 
 		manifest, err := crane.Manifest(
-			src,
+			ref.String(),
 			craneOpts...,
 		)
 		if err != nil {
-			return fmt.Errorf("fetching manifest %q: %w", src, err)
+			return fmt.Errorf("fetching manifest %q: %w", ref.String(), err)
 		}
 
 		rmt, err := remote.Get(
 			ref,
 			remoteOpts...,
 		)
 		if err != nil {
-			return fmt.Errorf("fetching image %q: %w", src, err)
+			return fmt.Errorf("fetching image %q: %w", ref.String(), err)
 		}
 
 		if tag.TagStr() != "" {
-Original file line number
+Diff line change
@@ Expand Up / @@ -131,6 +131,14 @@ case "${WITH_CONFIG_PATCH:-false}" in @@
         ;;
     esac
+    case "${WITH_ISO:-false}" in
+      false)
+        ;;
+      *)
+        QEMU_FLAGS+=("--iso-path=${ARTIFACTS}/metal-amd64.iso")
+        ;;
+    esac
     case "${WITH_CONFIG_PATCH_WORKER:-false}" in
       false)
         ;;
@@ Expand Down @@