feat: Add MountPoint option to AdditionalVolumes

And update templates to use a dedicated binary image for `kernel.image` and an additional volume to provide the modules. In line with changes to our image-builder pipeline, which removes the embedded kernel modules from the OS image, this feature adds an option to include modules as an additional volume. Old images will remain available to not break those using earlier versions.
liquidmetal-dev · Jan 12, 2023 · a676deb · a676deb
1 parent 8d1221d
commit a676deb
Show file tree

Hide file tree

Showing 10 changed files with 96 additions and 41 deletions.
diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go
@@ -72,13 +72,17 @@ type Volume struct {
 	// ID is a unique identifier for this volume.
 	// +kubebuilder:validation:Required
 	ID string `json:"id"`
-	// Image is the container image to use for the volume.
+	// Image is the container image to use as the source for the volume.
 	// +kubebuilder:validation:Required
 	Image string `json:"image"`
 	// ReadOnly specifies that the volume is to be mounted readonly.
 	// +kubebuilder:default:=false
 	// +optional
 	ReadOnly bool `json:"readOnly,omitempty"`
+	// MountPoint specifies the guest mountpoint for the volume.
+	// This will only be applied to additional volumes.
+	// +optional
+	MountPoint string `json:"mountPoint,omitempty"`
 }
 
 // IfaceType is a type representing the network interface types.

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml
@@ -119,7 +119,12 @@ spec:
                     description: ID is a unique identifier for this volume.
                     type: string
                   image:
-                    description: Image is the container image to use for the volume.
+                    description: Image is the container image to use as the source
+                      for the volume.
+                    type: string
+                  mountPoint:
+                    description: MountPoint specifies the guest mountpoint for the
+                      volume. This will only be applied to additional volumes.
                     type: string
                   readOnly:
                     default: false
@@ -163,7 +168,12 @@ spec:
                       description: ID is a unique identifier for this volume.
                       type: string
                     image:
-                      description: Image is the container image to use for the volume.
+                      description: Image is the container image to use as the source
+                        for the volume.
+                      type: string
+                    mountPoint:
+                      description: MountPoint specifies the guest mountpoint for the
+                        volume. This will only be applied to additional volumes.
                       type: string
                     readOnly:
                       default: false

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml
@@ -155,8 +155,13 @@ spec:
                             description: ID is a unique identifier for this volume.
                             type: string
                           image:
-                            description: Image is the container image to use for the
-                              volume.
+                            description: Image is the container image to use as the
+                              source for the volume.
+                            type: string
+                          mountPoint:
+                            description: MountPoint specifies the guest mountpoint
+                              for the volume. This will only be applied to additional
+                              volumes.
                             type: string
                           readOnly:
                             default: false
@@ -203,8 +208,13 @@ spec:
                               description: ID is a unique identifier for this volume.
                               type: string
                             image:
-                              description: Image is the container image to use for
-                                the volume.
+                              description: Image is the container image to use as
+                                the source for the volume.
+                              type: string
+                            mountPoint:
+                              description: MountPoint specifies the guest mountpoint
+                                for the volume. This will only be applied to additional
+                                volumes.
                               type: string
                             readOnly:
                               default: false

diff --git a/docs/development-with-tilt.md b/docs/development-with-tilt.md
@@ -101,13 +101,14 @@ Create the declaration for your cluster. We will use the template in the repo.
 3. Create a cluster declaration from the template
 
     ```bash
-    export KUBERNETES_VERSION=v1.20.0
+    export KUBERNETES_VERSION=v1.23.5
     export CLUSTER_NAME=mvm-test
     export CONTROL_PLANE_MACHINE_COUNT=1
     export WORKER_MACHINE_COUNT=1
     export CONTROL_PLANE_VIP=192.168.8.15
-    export MVM_ROOT_IMAGE=docker.io/richardcase/ubuntu-bionic-test:cloudimage_v0.0.1
-    export MVM_KERNEL_IMAGE=docker.io/richardcase/ubuntu-bionic-kernel:0.0.11
+    export MVM_ROOT_IMAGE=ghcr.io/weaveworks-liquidmetal/capmvm-k8s-os:1.23.5
+    export MVM_KERNEL_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77
+    export MVM_KERNEL_MODULES_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77
     # NOTE: change 192.168.8.2 to be the IP address from step 2
     export HOST_ENDPOINT=192.168.8.2:9090
 

diff --git a/go.mod b/go.mod
@@ -7,8 +7,8 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.20.0
 	github.com/spf13/pflag v1.0.5
-	github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20220628141946-264f4544f49f
-	github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20220628141946-264f4544f49f
+	github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184
+	github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184
 	github.com/yitsushi/macpot v1.0.2
 	google.golang.org/grpc v1.48.0
 	google.golang.org/protobuf v1.28.1

diff --git a/go.sum b/go.sum
@@ -717,10 +717,10 @@ github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLr
 github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
 github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk=
 github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI=
-github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20220628141946-264f4544f49f h1:I+BieoFUnOC9Y7m9NwGfK6wtEwzSkPeBLPEQf3+8r+Y=
-github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20220628141946-264f4544f49f/go.mod h1:JPML9O56MoPKGX97jfj++BtuFFS84jm4T+jWQBjO5Uc=
-github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20220628141946-264f4544f49f h1:zrUZz+usVtrvojEcbfG6Le+gv4467gXrlwRQGniRTaA=
-github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20220628141946-264f4544f49f/go.mod h1:d65mpsT+pbMnMJZhUqeNtt7lcQnsX8cowl9dTRkniSI=
+github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184 h1:o0CJeY1Dbq+xxw5mx32S+5nOft5nf2NZMjIbHlLvFpQ=
+github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184/go.mod h1:JPML9O56MoPKGX97jfj++BtuFFS84jm4T+jWQBjO5Uc=
+github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184 h1:7vrApNvF61104ULFq5lzH1IGfIE+94MwV/SmXlYvss0=
+github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184/go.mod h1:d65mpsT+pbMnMJZhUqeNtt7lcQnsX8cowl9dTRkniSI=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/yitsushi/macpot v1.0.2 h1:LSiBfVKRhzrvUTBDO2GZYAQvS+gG9wdzPaaiNeR29KM=

diff --git a/internal/services/microvm/convert.go b/internal/services/microvm/convert.go
@@ -30,7 +30,7 @@ func convertToFlintlockAPI(machineScope *scope.MachineScope) *flintlocktypes.Mic
 			Filename:         &mvmSpec.Kernel.Filename,
 		},
 		RootVolume: &flintlocktypes.Volume{
-			Id:         "root",
+			Id:         mvmSpec.RootVolume.ID,
 			IsReadOnly: mvmSpec.RootVolume.ReadOnly,
 			Source: &flintlocktypes.VolumeSource{
 				ContainerSource: &mvmSpec.RootVolume.Image,
@@ -51,13 +51,19 @@ func convertToFlintlockAPI(machineScope *scope.MachineScope) *flintlocktypes.Mic
 	for i := range mvmSpec.AdditionalVolumes {
 		volume := mvmSpec.AdditionalVolumes[i]
 
-		apiVM.AdditionalVolumes = append(apiVM.AdditionalVolumes, &flintlocktypes.Volume{
+		addVol := &flintlocktypes.Volume{
 			Id:         volume.ID,
 			IsReadOnly: volume.ReadOnly,
 			Source: &flintlocktypes.VolumeSource{
 				ContainerSource: &volume.Image,
 			},
-		})
+		}
+
+		if volume.MountPoint != "" {
+			addVol.MountPoint = &volume.MountPoint
+		}
+
+		apiVM.AdditionalVolumes = append(apiVM.AdditionalVolumes, addVol)
 	}
 
 	apiVM.Interfaces = []*flintlocktypes.NetworkInterface{}

diff --git a/templates/cluster-template-cilium.yaml b/templates/cluster-template-cilium.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -76,10 +76,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -97,7 +101,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -119,10 +123,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"

diff --git a/templates/cluster-template-flannel.yaml b/templates/cluster-template-flannel.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
@@ -74,17 +74,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -99,7 +103,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
@@ -117,17 +121,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -40,7 +40,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -72,10 +72,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -93,7 +97,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -115,10 +119,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"