feat: Add MountPoint option to AdditionalVolumes

And update templates to use a dedicated binary image for `kernel.image` and an additional volume to provide the modules. In line with changes to our image-builder pipeline, which removes the embedded kernel modules from the OS image, this feature adds an option to include modules as an additional volume. Old images will remain available to not break those using earlier versions.
liquidmetal-dev · Jan 13, 2023 · a6d5124 · a6d5124
1 parent 37ae130
commit a6d5124
Show file tree

Hide file tree

Showing 9 changed files with 103 additions and 46 deletions.
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachines.yaml
@@ -68,6 +68,11 @@ spec:
                   kernel cmdline. Each MicroVM provider has its own recommended list,
                   they will be used automatically. This field is for additional values.
                 type: object
+              labels:
+                additionalProperties:
+                  type: string
+                description: Labels allow you to include extra data on the Microvm
+                type: object
               memoryMb:
                 description: MemoryMb is the amount of memory in megabytes that the
                   microvm will be allocated.
@@ -119,7 +124,12 @@ spec:
                     description: ID is a unique identifier for this volume.
                     type: string
                   image:
-                    description: Image is the container image to use for the volume.
+                    description: Image is the container image to use as the source
+                      for the volume.
+                    type: string
+                  mountPoint:
+                    description: MountPoint specifies the guest mountpoint for the
+                      volume. This will only be applied to additional volumes.
                     type: string
                   readOnly:
                     default: false
@@ -163,7 +173,12 @@ spec:
                       description: ID is a unique identifier for this volume.
                       type: string
                     image:
-                      description: Image is the container image to use for the volume.
+                      description: Image is the container image to use as the source
+                        for the volume.
+                      type: string
+                    mountPoint:
+                      description: MountPoint specifies the guest mountpoint for the
+                        volume. This will only be applied to additional volumes.
                       type: string
                     readOnly:
                       default: false

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_microvmmachinetemplates.yaml
@@ -103,6 +103,12 @@ spec:
                           recommended list, they will be used automatically. This
                           field is for additional values.
                         type: object
+                      labels:
+                        additionalProperties:
+                          type: string
+                        description: Labels allow you to include extra data on the
+                          Microvm
+                        type: object
                       memoryMb:
                         description: MemoryMb is the amount of memory in megabytes
                           that the microvm will be allocated.
@@ -155,8 +161,13 @@ spec:
                             description: ID is a unique identifier for this volume.
                             type: string
                           image:
-                            description: Image is the container image to use for the
-                              volume.
+                            description: Image is the container image to use as the
+                              source for the volume.
+                            type: string
+                          mountPoint:
+                            description: MountPoint specifies the guest mountpoint
+                              for the volume. This will only be applied to additional
+                              volumes.
                             type: string
                           readOnly:
                             default: false
@@ -203,8 +214,13 @@ spec:
                               description: ID is a unique identifier for this volume.
                               type: string
                             image:
-                              description: Image is the container image to use for
-                                the volume.
+                              description: Image is the container image to use as
+                                the source for the volume.
+                              type: string
+                            mountPoint:
+                              description: MountPoint specifies the guest mountpoint
+                                for the volume. This will only be applied to additional
+                                volumes.
                               type: string
                             readOnly:
                               default: false

diff --git a/docs/compatibility.md b/docs/compatibility.md
@@ -4,6 +4,7 @@ This table shows the compatibility between CAPMVM and Flintlock versions.
 
 | CAPMVM      | Flintlock              |
 | ----------- | ---------------------- |
+| `v0.8.0`    | `>= v0.5.0`            |
 | `v0.7.0`    | `>= v0.1.0`            |
 | `v0.6.0`    | `v0.1.0`               |
 | `v0.5.x`    | `v0.1.0-alpha.9`       |

diff --git a/docs/development-with-tilt.md b/docs/development-with-tilt.md
@@ -101,13 +101,14 @@ Create the declaration for your cluster. We will use the template in the repo.
 3. Create a cluster declaration from the template
 
     ```bash
-    export KUBERNETES_VERSION=v1.20.0
+    export KUBERNETES_VERSION=v1.23.5
     export CLUSTER_NAME=mvm-test
     export CONTROL_PLANE_MACHINE_COUNT=1
     export WORKER_MACHINE_COUNT=1
     export CONTROL_PLANE_VIP=192.168.8.15
-    export MVM_ROOT_IMAGE=docker.io/richardcase/ubuntu-bionic-test:cloudimage_v0.0.1
-    export MVM_KERNEL_IMAGE=docker.io/richardcase/ubuntu-bionic-kernel:0.0.11
+    export MVM_ROOT_IMAGE=ghcr.io/weaveworks-liquidmetal/capmvm-k8s-os:1.23.5
+    export MVM_KERNEL_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77
+    export MVM_KERNEL_MODULES_IMAGE=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77
     # NOTE: change 192.168.8.2 to be the IP address from step 2
     export HOST_ENDPOINT=192.168.8.2:9090
 

diff --git a/go.mod b/go.mod
@@ -7,11 +7,11 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.20.0
 	github.com/spf13/pflag v1.0.5
-	github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20221118161315-83de77687232
-	github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20221118161315-83de77687232
-	github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20221118161315-83de77687232
-	github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20221117153111-bd29de31356f
-	github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20221117153111-bd29de31356f
+	github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20230113164813-f1ccd249c9ba
+	github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20230113164813-f1ccd249c9ba
+	github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20230113164813-f1ccd249c9ba
+	github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184
+	github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184
 	github.com/yitsushi/macpot v1.0.2
 	google.golang.org/grpc v1.50.1
 	google.golang.org/protobuf v1.28.1

diff --git a/go.sum b/go.sum
@@ -610,16 +610,16 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
 github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
-github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20221118161315-83de77687232 h1:ODvIKIfUzOYEOsMiUkIOSjKF7CzveE29/L46GsDLp40=
-github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20221118161315-83de77687232/go.mod h1:0snthlwNQ731b4CPJtSZ8I76FgS02wAOfyar9ss+X+U=
-github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20221118161315-83de77687232 h1:Nsd0XLPPizVY8oXR+D34GR5JzTpu4XGC3oVFaxudH5Q=
-github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20221118161315-83de77687232/go.mod h1:jld9hBa9YZ4ujs9+mpQ+QWWRq/aQ+t7Rlv9C70I3bes=
-github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20221118161315-83de77687232 h1:896AcgVPh5xcPCAnvC5WR5wyPFOz53+rndKsoeFQ6Vg=
-github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20221118161315-83de77687232/go.mod h1:KPlbfZsmo9xlfzl98cFBqGYc9hpgSYljICR+LrqKJUA=
-github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20221117153111-bd29de31356f h1:xHtKo4eiR1pRZUrXxlGF3KDuoa6xYjBGuPP47M9eVWM=
-github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20221117153111-bd29de31356f/go.mod h1:JPML9O56MoPKGX97jfj++BtuFFS84jm4T+jWQBjO5Uc=
-github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20221117153111-bd29de31356f h1:xlBe6YZx8/cQMVAh0FaQ1IYaqeZPjP0kTOg3dHTcSdQ=
-github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20221117153111-bd29de31356f/go.mod h1:d65mpsT+pbMnMJZhUqeNtt7lcQnsX8cowl9dTRkniSI=
+github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20230113164813-f1ccd249c9ba h1:Hestt4C2MDRG1xEb2kVbCetmlBnPulUab8LuBtzXsl4=
+github.com/weaveworks-liquidmetal/controller-pkg/client v0.0.0-20230113164813-f1ccd249c9ba/go.mod h1:0snthlwNQ731b4CPJtSZ8I76FgS02wAOfyar9ss+X+U=
+github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20230113164813-f1ccd249c9ba h1:WreSgeHqwWSyvNHvsk1OHr9zx/a1XkQEkhWyuam3XCU=
+github.com/weaveworks-liquidmetal/controller-pkg/services/microvm v0.0.0-20230113164813-f1ccd249c9ba/go.mod h1:JPXXxIrs6HC1qmy4//kioH4myd4h1G9b02yvVl0GYEg=
+github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20230113164813-f1ccd249c9ba h1:czFGmavIfOHuyVt+UInQBfF7tCCI3Ov/ZigJh37qzok=
+github.com/weaveworks-liquidmetal/controller-pkg/types/microvm v0.0.0-20230113164813-f1ccd249c9ba/go.mod h1:KPlbfZsmo9xlfzl98cFBqGYc9hpgSYljICR+LrqKJUA=
+github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184 h1:o0CJeY1Dbq+xxw5mx32S+5nOft5nf2NZMjIbHlLvFpQ=
+github.com/weaveworks-liquidmetal/flintlock/api v0.0.0-20230109155544-97f4f12ca184/go.mod h1:JPML9O56MoPKGX97jfj++BtuFFS84jm4T+jWQBjO5Uc=
+github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184 h1:7vrApNvF61104ULFq5lzH1IGfIE+94MwV/SmXlYvss0=
+github.com/weaveworks-liquidmetal/flintlock/client v0.0.0-20230109155544-97f4f12ca184/go.mod h1:d65mpsT+pbMnMJZhUqeNtt7lcQnsX8cowl9dTRkniSI=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yitsushi/macpot v1.0.2 h1:LSiBfVKRhzrvUTBDO2GZYAQvS+gG9wdzPaaiNeR29KM=
 github.com/yitsushi/macpot v1.0.2/go.mod h1:7KBzokvUNbcsR1VcmKwmYRWB2FyAWcIll4L93b2A8q4=

diff --git a/templates/cluster-template-cilium.yaml b/templates/cluster-template-cilium.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -76,10 +76,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -97,7 +101,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -119,10 +123,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"

diff --git a/templates/cluster-template-flannel.yaml b/templates/cluster-template-flannel.yaml
@@ -44,7 +44,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
@@ -74,17 +74,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -99,7 +103,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
@@ -117,17 +121,21 @@ metadata:
 spec:
   template:
     spec:
+      rootVolume:
+        id: root
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: boot/vmlinux
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       memoryMb: 2048
       networkInterfaces:
       - guestDeviceName: eth1
         type: macvtap
-      rootVolume:
-        id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
       vcpu: 2
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -40,7 +40,7 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: "${KUBERNETES_VERSION:=v1.21.8}"
+  version: "${KUBERNETES_VERSION:=v1.23.5}"
   machineTemplate:
     infrastructureRef:
       kind: MicrovmMachineTemplate
@@ -72,10 +72,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"
@@ -93,7 +97,7 @@ spec:
   template:
     spec:
       clusterName: "${CLUSTER_NAME}"
-      version: "${KUBERNETES_VERSION:=v1.21.8}"
+      version: "${KUBERNETES_VERSION:=v1.23.5}"
       bootstrap:
         configRef:
           name: "${CLUSTER_NAME}-md-0"
@@ -115,10 +119,14 @@ spec:
       memoryMb: 2048
       rootVolume:
         id: root
-        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.21.8}"
+        image: "${MVM_ROOT_IMAGE:=ghcr.io/weaveworks-liquidmetal/capmvm-kubernetes:1.23.5}"
       kernel:
         filename: "boot/vmlinux"
-        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/flintlock-kernel:5.10.77}"
+        image: "${MVM_KERNEL_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-bin:5.10.77}"
+      volumes:
+      - id: modules
+        image: "${MVM_KERNEL_MODULES_IMAGE:=ghcr.io/weaveworks-liquidmetal/kernel-modules:5.10.77}"
+        mountPoint: /lib/modules/5.10.77
       kernelCmdline: {}
       networkInterfaces:
       - guestDeviceName: "eth1"