feat: Include kernel modules as AdditionalVolumes

In line with changes to our image-builder pipeline, which removes the embedded kernel modules from the OS image, this feature adds an option to include modules as an additional volume. Templates and docs have been updated. Old images and options will remain available to not break those using earlier versions.
liquidmetal-dev · Jan 6, 2023 · 60455eb · 60455eb
1 parent 8d1221d
commit 60455eb
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 34 deletions.
diff --git a/api/v1alpha1/types.go b/api/v1alpha1/types.go
@@ -72,13 +72,17 @@ type Volume struct {
 	// ID is a unique identifier for this volume.
 	// +kubebuilder:validation:Required
 	ID string `json:"id"`
-	// Image is the container image to use for the volume.
+	// Image is the container image to use as the source for the volume.
 	// +kubebuilder:validation:Required
 	Image string `json:"image"`
 	// ReadOnly specifies that the volume is to be mounted readonly.
 	// +kubebuilder:default:=false
 	// +optional
 	ReadOnly bool `json:"readOnly,omitempty"`
+	// MountPoint specifies the guest mountpoint for the volume.
+	// This will only be applied to additional volumes.
+	// +optional
+	MountPoint string `json:"destination,omitempty"`
 }
 
 // IfaceType is a type representing the network interface types.

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml
@@ -115,11 +115,16 @@ spec:
                 description: RootVolume specifies the volume to use for the root of
                   the microvm.
                 properties:
+                  destination:
+                    description: MountPoint specifies the guest mountpoint for the
+                      volume. This will only be applied to additional volumes.
+                    type: string
                   id:
                     description: ID is a unique identifier for this volume.
                     type: string
                   image:
-                    description: Image is the container image to use for the volume.
+                    description: Image is the container image to use as the source
+                      for the volume.
                     type: string
                   readOnly:
                     default: false
@@ -159,11 +164,16 @@ spec:
                 items:
                   description: Volume represents a volume to be attached to a microvm.
                   properties:
+                    destination:
+                      description: MountPoint specifies the guest mountpoint for the
+                        volume. This will only be applied to additional volumes.
+                      type: string
                     id:
                       description: ID is a unique identifier for this volume.
                       type: string
                     image:
-                      description: Image is the container image to use for the volume.
+                      description: Image is the container image to use as the source
+                        for the volume.
                       type: string
                     readOnly:
                       default: false

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml
@@ -151,12 +151,17 @@ spec:
                         description: RootVolume specifies the volume to use for the
                           root of the microvm.
                         properties:
+                          destination:
+                            description: MountPoint specifies the guest mountpoint
+                              for the volume. This will only be applied to additional
+                              volumes.
+                            type: string
                           id:
                             description: ID is a unique identifier for this volume.
                             type: string
                           image:
-                            description: Image is the container image to use for the
-                              volume.
+                            description: Image is the container image to use as the
+                              source for the volume.
                             type: string
                           readOnly:
                             default: false
@@ -199,12 +204,17 @@ spec:
                           description: Volume represents a volume to be attached to
                             a microvm.
                           properties:
+                            destination:
+                              description: MountPoint specifies the guest mountpoint
+                                for the volume. This will only be applied to additional
+                                volumes.
+                              type: string
                             id:
                               description: ID is a unique identifier for this volume.
                               type: string
                             image:
-                              description: Image is the container image to use for
-                                the volume.
+                              description: Image is the container image to use as
+                                the source for the volume.
                               type: string
                             readOnly:
                               default: false

diff --git a/docs/development-with-tilt.md b/docs/development-with-tilt.md
@@ -101,13 +101,14 @@ Create the declaration for your cluster. We will use the template in the repo.
 3. Create a cluster declaration from the template
 
     ```bash
-    export KUBERNETES_VERSION=v1.20.0
+    export KUBERNETES_VERSION=v1.23.5
     export CLUSTER_NAME=mvm-test
     export CONTROL_PLANE_MACHINE_COUNT=1
     export WORKER_MACHINE_COUNT=1
     export CONTROL_PLANE_VIP=192.168.8.15
-    export MVM_ROOT_IMAGE=docker.io/richardcase/ubuntu-bionic-test:cloudimage_v0.0.1
-    export MVM_KERNEL_IMAGE=docker.io/richardcase/ubuntu-bionic-kernel:0.0.11
+    export MVM_ROOT_IMAGE=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5
+    export MVM_KERNEL_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77
+    export MVM_KERNEL_MODULES_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77
     # NOTE: change 192.168.8.2 to be the IP address from step 2
     export HOST_ENDPOINT=192.168.8.2:9090
 

diff --git a/internal/services/microvm/convert.go b/internal/services/microvm/convert.go
@@ -51,13 +51,19 @@ func convertToFlintlockAPI(machineScope *scope.MachineScope) *flintlocktypes.Mic
 	for i := range mvmSpec.AdditionalVolumes {
 		volume := mvmSpec.AdditionalVolumes[i]
 
-		apiVM.AdditionalVolumes = append(apiVM.AdditionalVolumes, &flintlocktypes.Volume{
+		addVol := &flintlocktypes.Volume{
 			Id:         volume.ID,
 			IsReadOnly: volume.ReadOnly,
 			Source: &flintlocktypes.VolumeSource{
 				ContainerSource: &volume.Image,
 			},
-		})
+		}
+
+		if volume.MountPoint != "" {
+			// addVol.MountPoint = &volume.MountPoint
+		}
+
+		apiVM.AdditionalVolumes = append(apiVM.AdditionalVolumes, addVol)
 	}
 
 	apiVM.Interfaces = []*flintlocktypes.NetworkInterface{}

diff --git a/templates/cluster-template-cilium.yaml b/templates/cluster-template-cilium.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -76,10 +76,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -97,7 +101,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -119,10 +123,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"

diff --git a/templates/cluster-template-flannel.yaml b/templates/cluster-template-flannel.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
@@ -74,17 +74,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -99,7 +103,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
@@ -117,17 +121,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -40,7 +40,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -72,10 +72,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -93,7 +97,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -115,10 +119,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      additionalVolumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"