Add support for building aarch64 ISO

.gitignore

@@ -26,14 +26,19 @@ _testmain.go
 *.test
 *.prof
 
-/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause
+/deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/usr/bin/auto-pause
+/deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/usr/bin/auto-pause
+/deploy/iso/minikube-iso/Config.in
 /deploy/kicbase/auto-pause
 /deploy/addons/auto-pause/auto-pause-hook
 /out
 /_gopath
 
 #iso version file
-deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/VERSION
+deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/VERSION
+deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/etc/VERSION
+deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/CHANGELOG
+deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/etc/CHANGELOG
 
 /minikube
 

Makefile

@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.25.2-1650391217-13814
+ISO_VERSION ?= v1.25.2-1651892774-13762
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 DEB_REVISION ?= 0
@@ -93,7 +93,7 @@ GOOS ?= $(shell go env GOOS)
 GOARCH ?= $(shell go env GOARCH)
 GOARM ?= 7 # the default is 5
 GOPATH ?= $(shell go env GOPATH)
-BUILD_DIR ?= ./out
+BUILD_DIR ?= $(PWD)/out
 $(shell mkdir -p $(BUILD_DIR))
 CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
@@ -282,17 +282,27 @@ out/e2e-%: out/minikube-%
 out/e2e-windows-amd64.exe: out/e2e-windows-amd64
 	cp $< $@
 
-minikube_iso: deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause # build minikube iso
-	echo $(ISO_VERSION) > deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/VERSION
+minikube-iso-amd64: minikube-iso-x86_64
+minikube-iso-arm64: minikube-iso-aarch64
+
+minikube-iso-%: deploy/iso/minikube-iso/board/minikube/%/rootfs-overlay/usr/bin/auto-pause # build minikube iso
+	echo $(ISO_VERSION) > deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/etc/VERSION
+	cp deploy/iso/minikube-iso/CHANGELOG deploy/iso/minikube-iso/board/minikube/$*/rootfs-overlay/etc/CHANGELOG
+	cp deploy/iso/minikube-iso/arch/$*/Config.in.tmpl deploy/iso/minikube-iso/Config.in
 	if [ ! -d $(BUILD_DIR)/buildroot ]; then \
 		mkdir -p $(BUILD_DIR); \
 		git clone --depth=1 --branch=$(BUILDROOT_BRANCH) https://github.com/buildroot/buildroot $(BUILD_DIR)/buildroot; \
 		cp deploy/iso/minikube-iso/go.hash $(BUILD_DIR)/buildroot/package/go/go.hash; \
 	fi;
-	$(MAKE) BR2_EXTERNAL=../../deploy/iso/minikube-iso minikube_defconfig -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS)
-	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) host-python
-	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS)
-	mv $(BUILD_DIR)/buildroot/output/images/rootfs.iso9660 $(BUILD_DIR)/minikube.iso
+	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$* minikube_$*_defconfig
+	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$* host-python
+	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) O=$(BUILD_DIR)/buildroot/output-$*
+	# x86_64 ISO is still BIOS rather than EFI because of AppArmor issues for KVM, and Gen 2 issues for Hyper-V
+	if [ "$*" = "aarch64" ]; then \
+                mv $(BUILD_DIR)/buildroot/output-aarch64/images/boot.iso $(BUILD_DIR)/minikube-arm64.iso; \
+        else \
+                mv $(BUILD_DIR)/buildroot/output-x86_64/images/rootfs.iso9660 $(BUILD_DIR)/minikube-amd64.iso; \
+        fi;
 
 # Change buildroot configuration for the minikube ISO
 .PHONY: iso-menuconfig
@@ -301,15 +311,14 @@ iso-menuconfig: ## Configure buildroot configuration
 	$(MAKE) -C $(BUILD_DIR)/buildroot $(BUILDROOT_OPTIONS) savedefconfig
 
 # Change the kernel configuration for the minikube ISO
-.PHONY: linux-menuconfig
-linux-menuconfig:  ## Configure Linux kernel configuration
+linux-menuconfig-%:  ## Configure Linux kernel configuration
 	$(MAKE) -C $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/ menuconfig
 	$(MAKE) -C $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/ savedefconfig
-	cp $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/defconfig deploy/iso/minikube-iso/board/coreos/minikube/linux_defconfig
+	cp $(BUILD_DIR)/buildroot/output/build/linux-$(KERNEL_VERSION)/defconfig deploy/iso/minikube-iso/board/minikube/$*/linux_$*_defconfig
 
-out/minikube.iso: $(shell find "deploy/iso/minikube-iso" -type f)
+out/minikube-%.iso: $(shell find "deploy/iso/minikube-iso" -type f)
 ifeq ($(IN_DOCKER),1)
-	$(MAKE) minikube_iso
+	$(MAKE) minikube-iso-$*
 else
 	docker run --rm --workdir /mnt --volume $(CURDIR):/mnt $(ISO_DOCKER_EXTRA_ARGS) \
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
@@ -321,9 +330,6 @@ iso_in_docker:
 		--user $(shell id -u):$(shell id -g) --env HOME=/tmp --env IN_DOCKER=1 \
 		$(ISO_BUILD_IMAGE) /bin/bash
 
-test-iso:
-	go test -v $(INTEGRATION_TESTS_TO_RUN) --tags=iso --minikube-start-args="--iso-url=file://$(shell pwd)/out/buildroot/output/images/rootfs.iso9660"
-
 .PHONY: test-pkg
 test-pkg/%: ## Trigger packaging test
 	go test -v -test.timeout=60m ./$* --tags="$(MINIKUBE_BUILD_TAGS)"
@@ -436,7 +442,7 @@ e2e-cross: e2e-linux-amd64 e2e-linux-arm64 e2e-darwin-amd64 e2e-windows-amd64.ex
 
 .PHONY: checksum
 checksum: ## Generate checksums
-	for f in out/minikube.iso out/minikube-linux-amd64 out/minikube-linux-arm \
+	for f in out/minikube-amd64.iso out/minikube-arm64.iso out/minikube-linux-amd64 out/minikube-linux-arm \
 		 out/minikube-linux-arm64 out/minikube-linux-ppc64le out/minikube-linux-s390x \
 		 out/minikube-darwin-amd64 out/minikube-darwin-arm64 out/minikube-windows-amd64.exe \
 		 out/docker-machine-driver-kvm2 out/docker-machine-driver-kvm2-amd64 out/docker-machine-driver-kvm2-arm64 \
@@ -776,9 +782,11 @@ push-gvisor-addon-image: gvisor-addon-image
 	$(MAKE) push-docker IMAGE=$(REGISTRY)/gvisor-addon:$(GVISOR_TAG)
 
 .PHONY: release-iso
-release-iso: minikube_iso checksum  ## Build and release .iso file
-	gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso
-	gsutil cp out/minikube.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso.sha256
+release-iso: minikube-iso-aarch64 minikube-iso-x86_64 checksum  ## Build and release .iso files
+	gsutil cp out/minikube-amd64.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-amd64.iso
+	gsutil cp out/minikube-amd64.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-amd64.iso.sha256
+	gsutil cp out/minikube-arm64.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-arm64.iso
+	gsutil cp out/minikube-arm64.iso.sha256 gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION)-arm64.iso.sha256
 
 .PHONY: release-minikube
 release-minikube: out/minikube checksum ## Minikube release
@@ -918,8 +926,9 @@ out/mkcmp:
 
 
 # auto pause binary to be used for ISO
-deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/bin/auto-pause: $(SOURCE_FILES) $(ASSET_FILES)
-	GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
+deploy/iso/minikube-iso/board/minikube/%/rootfs-overlay/usr/bin/auto-pause: $(SOURCE_FILES) $(ASSET_FILES)
+	@if [ "$*" != "x86_64" ] && [ "$*" != "aarch64" ]; then echo "Please enter a valid architecture. Choices are x86_64 and aarch64."; exit 1; fi
+	GOOS=linux GOARCH=$(subst x86_64,amd64,$(subst aarch64,arm64,$*)) go build -o $@ cmd/auto-pause/auto-pause.go
 
 
 .PHONY: deploy/addons/auto-pause/auto-pause-hook

deploy/iso/minikube-iso/arch/aarch64/Config.in.tmpl

@@ -0,0 +1,2 @@
+source "$BR2_EXTERNAL_MINIKUBE_PATH/package/Config.in"
+source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/Config.in" 

deploy/iso/minikube-iso/arch/aarch64/package/Config.in

@@ -0,0 +1,12 @@
+menu "System tools aarch64"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/buildkit-bin-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/runc-master-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/crio-bin-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/crictl-bin-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/docker-bin-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cri-dockerd-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cni-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/cni-plugins-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/containerd-bin-aarch64/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/arch/aarch64/package/podman-aarch64/Config.in"
+endmenu

deploy/iso/minikube-iso/arch/aarch64/package/buildkit-bin-aarch64/Config.in

@@ -0,0 +1,4 @@
+config BR2_PACKAGE_BUILDKIT_BIN_AARCH64
+	bool "buildkit-bin"
+	default y
+	depends on BR2_aarch64

deploy/iso/minikube-iso/arch/aarch64/package/buildkit-bin-aarch64/buildkit-bin.hash

@@ -0,0 +1 @@
+sha256 0a834a749e86525e0eb16e7e80ce726e7c6a8e8d26fc5cc36c712305024da4e7  buildkit-v0.9.0.linux-arm64.tar.gz

deploy/iso/minikube-iso/arch/aarch64/package/buildkit-bin-aarch64/buildkit-bin.mk

@@ -0,0 +1,52 @@
+################################################################################
+#
+# buildkit-bin
+#
+################################################################################
+
+BUILDKIT_BIN_AARCH64_VERSION = v0.9.0
+BUILDKIT_BIN_AARCH64_COMMIT = c8bb937807d405d92be91f06ce2629e6202ac7a9
+BUILDKIT_BIN_AARCH64_SITE = https://github.com/moby/buildkit/releases/download/$(BUILDKIT_BIN_AARCH64_VERSION)
+BUILDKIT_BIN_AARCH64_SOURCE = buildkit-$(BUILDKIT_BIN_AARCH64_VERSION).linux-arm64.tar.gz
+
+# https://github.com/opencontainers/runc.git
+BUILDKIT_RUNC_VERSION = 12644e614e25b05da6fd08a38ffa0cfe1903fdec
+
+define BUILDKIT_BIN_AARCH64_USERS
+	- -1 buildkit -1 - - - - -
+endef
+
+define BUILDKIT_BIN_AARCH64_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildctl \
+		$(TARGET_DIR)/usr/bin
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildkit-runc \
+		$(TARGET_DIR)/usr/sbin
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildkit-qemu-* \
+		$(TARGET_DIR)/usr/sbin
+	$(INSTALL) -D -m 0755 \
+		$(@D)/buildkitd \
+		$(TARGET_DIR)/usr/sbin
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.conf \
+		$(TARGET_DIR)/usr/lib/tmpfiles.d/buildkit.conf
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkitd.toml \
+		$(TARGET_DIR)/etc/buildkit/buildkitd.toml
+endef
+
+define BUILDKIT_BIN_AARCH64_INSTALL_INIT_SYSTEMD
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.service \
+		$(TARGET_DIR)/usr/lib/systemd/system/buildkit.service
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_AARCH64_PKGDIR)/buildkit.socket \
+		$(TARGET_DIR)/usr/lib/systemd/system/buildkit.socket
+	$(INSTALL) -D -m 644 \
+		$(BUILDKIT_BIN_AARCH64_PKGDIR)/51-buildkit.preset \
+		$(TARGET_DIR)/usr/lib/systemd/system-preset/51-buildkit.preset
+endef
+
+$(eval $(generic-package))

deploy/iso/minikube-iso/arch/aarch64/package/cni-aarch64/Config.in

@@ -0,0 +1,5 @@
+config BR2_PACKAGE_CNI_AARCH64
+	bool "cni"
+	default y
+	depends on BR2_aarch64
+	depends on BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS

deploy/iso/minikube-iso/arch/aarch64/package/cni-aarch64/cni.mk

@@ -0,0 +1,45 @@
+################################################################################
+#
+# cni
+#
+################################################################################
+
+CNI_AARCH64_VERSION = v0.7.1
+CNI_AARCH64_SITE = https://github.com/containernetworking/cni/archive
+CNI_AARCH64_SOURCE = $(CNI_AARCH64_VERSION).tar.gz
+CNI_AARCH64_LICENSE = Apache-2.0
+CNI_AARCH64_LICENSE_FILES = LICENSE
+
+CNI_AARCH64_DEPENDENCIES = host-go
+
+CNI_AARCH64_GOPATH = $(@D)/_output
+CNI_AARCH64_MAKE_ENV = \
+	$(GO_TARGET_ENV) \
+	CGO_ENABLED=0 \
+	GO111MODULE=off \
+	GOPATH="$(CNI_AARCH64_GOPATH)" \
+	PATH=$(CNI_AARCH64_GOPATH)/bin:$(BR_PATH) \
+	GOARCH=arm64
+
+CNI_AARCH64_BUILDFLAGS = -a --ldflags '-extldflags \"-static\"'
+
+define CNI_AARCH64_CONFIGURE_CMDS
+        mkdir -p $(CNI_AARCH64_GOPATH)/src/github.com/containernetworking
+        ln -sf $(@D) $(CNI_AARCH64_GOPATH)/src/github.com/containernetworking/cni
+endef
+
+define CNI_AARCH64_BUILD_CMDS
+	(cd $(@D); $(CNI_AARCH64_MAKE_ENV) go build -o bin/cnitool $(CNI_AARCH64_BUILDFLAGS) ./cnitool)
+endef
+
+define CNI_AARCH64_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bin/cnitool \
+		$(TARGET_DIR)/opt/cni/bin/cnitool
+
+	ln -sf \
+		../../opt/cni/bin/cnitool \
+		$(TARGET_DIR)/usr/bin/cnitool
+endef
+
+$(eval $(generic-package))

deploy/iso/minikube-iso/arch/aarch64/package/cni-plugins-aarch64/Config.in

@@ -0,0 +1,5 @@
+config BR2_PACKAGE_CNI_PLUGINS_AARCH64
+	bool "cni-plugins"
+	default y
+	depends on BR2_aarch64
+	depends on BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS

deploy/iso/minikube-iso/arch/aarch64/package/cni-plugins-aarch64/cni-plugins.hash

@@ -0,0 +1 @@
+sha256 a7881ec37e592c897bdfd2a225b4ed74caa981e3c4cdcf8f45574f8d2f111bce  cni-plugins-linux-arm64-v0.8.5.tgz

deploy/iso/minikube-iso/arch/aarch64/package/cni-plugins-aarch64/cni-plugins.mk

@@ -0,0 +1,120 @@
+################################################################################
+#
+# cni-plugins
+#
+################################################################################
+
+CNI_PLUGINS_AARCH64_VERSION = v0.8.5
+CNI_PLUGINS_AARCH64_SITE = https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_AARCH64_VERSION)
+CNI_PLUGINS_AARCH64_SOURCE = cni-plugins-linux-arm64-$(CNI_PLUGINS_AARCH64_VERSION).tgz
+CNI_PLUGINS_AARCH64_LICENSE = Apache-2.0
+CNI_PLUGINS_AARCH64_LICENSE_FILES = LICENSE
+
+define CNI_PLUGINS_AARCH64_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bandwidth \
+		$(TARGET_DIR)/opt/cni/bin/bandwidth
+
+	ln -sf \
+		../../opt/cni/bin/bandwidth \
+		$(TARGET_DIR)/usr/bin/bandwidth
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/bridge \
+		$(TARGET_DIR)/opt/cni/bin/bridge
+
+	ln -sf \
+		../../opt/cni/bin/bridge \
+		$(TARGET_DIR)/usr/bin/bridge
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/vlan \
+		$(TARGET_DIR)/opt/cni/bin/vlan
+
+	ln -sf \
+		../../opt/cni/bin/vlan \
+		$(TARGET_DIR)/usr/bin/vlan
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/tuning \
+		$(TARGET_DIR)/opt/cni/bin/tuning
+
+	ln -sf \
+		../../opt/cni/bin/tuning \
+		$(TARGET_DIR)/usr/bin/tuning
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/ptp \
+		$(TARGET_DIR)/opt/cni/bin/ptp
+
+	ln -sf \
+		../../opt/cni/bin/ptp \
+		$(TARGET_DIR)/usr/bin/ptp
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/portmap \
+		$(TARGET_DIR)/opt/cni/bin/portmap
+
+	ln -sf \
+		../../opt/cni/bin/portmap \
+		$(TARGET_DIR)/usr/bin/portmap
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/macvlan \
+		$(TARGET_DIR)/opt/cni/bin/macvlan
+
+	ln -sf \
+		../../opt/cni/bin/macvlan \
+		$(TARGET_DIR)/usr/bin/macvlan
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/loopback \
+		$(TARGET_DIR)/opt/cni/bin/loopback
+
+	ln -sf \
+		../../opt/cni/bin/loopback \
+		$(TARGET_DIR)/usr/bin/loopback
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/ipvlan \
+		$(TARGET_DIR)/opt/cni/bin/ipvlan
+
+	ln -sf \
+		../../opt/cni/bin/ipvlan \
+		$(TARGET_DIR)/usr/bin/ipvlan
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/host-local \
+		$(TARGET_DIR)/opt/cni/bin/host-local
+
+	ln -sf \
+		../../opt/cni/bin/host-local \
+		$(TARGET_DIR)/usr/bin/host-local
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/flannel \
+		$(TARGET_DIR)/opt/cni/bin/flannel
+
+	ln -sf \
+		../../opt/cni/bin/flannel \
+		$(TARGET_DIR)/usr/bin/flannel
+
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/dhcp \
+		$(TARGET_DIR)/opt/cni/bin/dhcp
+
+	ln -sf \
+		../../opt/cni/bin/dhcp \
+		$(TARGET_DIR)/usr/bin/dhcp
+
+	$(INSTALL) -D -m 0755 \
+		$(@D)/firewall \
+		$(TARGET_DIR)/opt/cni/bin/firewall
+
+	ln -sf \
+		../../opt/cni/bin/firewall \
+		$(TARGET_DIR)/usr/bin/firewall
+endef
+
+$(eval $(generic-package))