feat(hosts/k8s): kubevirt - set cpu-manager's policy to static

Author: ryan4yin

hosts/k8s/kubevirt-shoryu/default.nix

@@ -1,5 +1,5 @@
 {
-  config,
+  lib,
   pkgs,
   mylib,
   myvars,
@@ -22,6 +22,13 @@
     # use my own domain & kube-vip's virtual IP for the API server
     # so that the API server can always be accessed even if some nodes are down
     masterHost = "kubevirt-cluster-1.writefor.fun";
+    kubeletExtraArgs = [
+      "--cpu-manager-policy=static"
+      # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/
+      # we have to reserve some resources for for system daemons running as pods
+      # when cpu-manager's static policy is enabled
+      "--system-reserved=cpu=1,memory=1Gi,ephemeral-storage=2Gi"
+    ];
     nodeLabels = [
       "node-purpose=kubevirt"
     ];

lib/genK3sServerModule.nix

@@ -10,11 +10,13 @@
   # this can be a domain name or an IP address(such as kube-vip's virtual IP)
   masterHost,
   clusterInit ? false,
+  kubeletExtraArgs ? [],
   nodeLabels ? [],
   nodeTaints ? [],
   disableFlannel ? true,
   ...
 }: let
+  lib = pkgs.lib;
   package = pkgs.k3s_1_29;
 in {
   environment.systemPackages = with pkgs; [
@@ -59,9 +61,10 @@ in {
         ]
         ++ (map (label: "--node-label=${label}") nodeLabels)
         ++ (map (taint: "--node-taint=${taint}") nodeTaints)
-        ++ (pkgs.lib.optionals disableFlannel ["--flannel-backend=none"]);
+        ++ (map (arg: "--kubelet-arg=${arg}") kubeletExtraArgs)
+        ++ (lib.optionals disableFlannel ["--flannel-backend=none"]);
     in
-      pkgs.lib.concatStringsSep " " flagList;
+      lib.concatStringsSep " " flagList;
   };
 
   # create symlinks to link k3s's cni directory to the one used by almost all CNI plugins