Merge pull request #2986 from stmcginnis/cpu-policy-options

kubelet: Enable setting cpuManagerPolicyOptions
bottlerocket-os · Apr 24, 2023 · b1f7b44 · b1f7b44
2 parents 3e6bbf0 + 8cb2651
commit b1f7b44
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -418,6 +418,18 @@ The following settings are optional and allow you to further configure your clus
 * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container.
 * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated.
 * `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`.
+* `settings.kubernetes.cpu-manager-policy-options`: Policy options to apply when `cpu-manager-policy` is set to `static`. Currently `full-pcpus-only` is the only option.
+
+  For example:
+
+  ```toml
+  [settings.kubernetes]
+  cpu-manager-policy = "static"
+  cpu-manager-policy-options = [
+    "full-pcpus-only"
+  ]
+  ```
+
 * `settings.kubernetes.cpu-manager-reconcile-period`: Specifies the CPU manager reconcile period, which controls how often updated CPU assignments are written to cgroupfs. The value is a duration like `30s` for 30 seconds or `1h5m` for 1 hour and 5 minutes.
 * `settings.kubernetes.credential-providers`: Contains a collection of Kubelet image credential provider settings.
   Each name under `credential-providers` is the name of the plugin to configure.

diff --git a/Release.toml b/Release.toml
@@ -201,5 +201,6 @@ version = "1.14.0"
     "migrate_v1.13.4_add-hostname-override-metadata.lz4",
 ]
 "(1.13.4, 1.14.0)" = [
-    "migrate_v1.14.0_kubernetes-gc-percent-type-change.lz4"
+    "migrate_v1.14.0_kubernetes-gc-percent-type-change.lz4",
+    "migrate_v1.14.0_kubernetes-config-settings.lz4",
 ]
diff --git a/packages/kubernetes-1.22/kubelet-config b/packages/kubernetes-1.22/kubelet-config
@@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
 {{#if settings.kubernetes.cpu-manager-reconcile-period}}
 cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
 {{/if}}
+{{#if settings.kubernetes.cpu-manager-policy-options}}
+cpuManagerPolicyOptions:
+{{#each settings.kubernetes.cpu-manager-policy-options}}
+    {{this}}: "true"
+{{/each}}
+{{/if}}
 {{#if settings.kubernetes.topology-manager-scope}}
 topologyManagerScope: {{settings.kubernetes.topology-manager-scope}}
 {{/if}}

diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config
@@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
 {{#if settings.kubernetes.cpu-manager-reconcile-period}}
 cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
 {{/if}}
+{{#if settings.kubernetes.cpu-manager-policy-options}}
+cpuManagerPolicyOptions:
+{{#each settings.kubernetes.cpu-manager-policy-options}}
+    {{this}}: "true"
+{{/each}}
+{{/if}}
 {{#if settings.kubernetes.topology-manager-scope}}
 topologyManagerScope: {{settings.kubernetes.topology-manager-scope}}
 {{/if}}

diff --git a/packages/kubernetes-1.24/kubelet-config b/packages/kubernetes-1.24/kubelet-config
@@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
 {{#if settings.kubernetes.cpu-manager-reconcile-period}}
 cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
 {{/if}}
+{{#if settings.kubernetes.cpu-manager-policy-options}}
+cpuManagerPolicyOptions:
+{{#each settings.kubernetes.cpu-manager-policy-options}}
+    {{this}}: "true"
+{{/each}}
+{{/if}}
 {{#if settings.kubernetes.topology-manager-scope}}
 topologyManagerScope: {{settings.kubernetes.topology-manager-scope}}
 {{/if}}

diff --git a/packages/kubernetes-1.25/kubelet-config b/packages/kubernetes-1.25/kubelet-config
@@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
 {{#if settings.kubernetes.cpu-manager-reconcile-period}}
 cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
 {{/if}}
+{{#if settings.kubernetes.cpu-manager-policy-options}}
+cpuManagerPolicyOptions:
+{{#each settings.kubernetes.cpu-manager-policy-options}}
+    {{this}}: "true"
+{{/each}}
+{{/if}}
 {{#if settings.kubernetes.topology-manager-scope}}
 topologyManagerScope: {{settings.kubernetes.topology-manager-scope}}
 {{/if}}

diff --git a/packages/kubernetes-1.26/kubelet-config b/packages/kubernetes-1.26/kubelet-config
@@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}}
 {{#if settings.kubernetes.cpu-manager-reconcile-period}}
 cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}}
 {{/if}}
+{{#if settings.kubernetes.cpu-manager-policy-options}}
+cpuManagerPolicyOptions:
+{{#each settings.kubernetes.cpu-manager-policy-options}}
+    {{this}}: "true"
+{{/each}}
+{{/if}}
 {{#if settings.kubernetes.topology-manager-scope}}
 topologyManagerScope: {{settings.kubernetes.topology-manager-scope}}
 {{/if}}

diff --git a/sources/Cargo.lock b/sources/Cargo.lock
diff --git a/sources/Cargo.toml b/sources/Cargo.toml
@@ -45,6 +45,7 @@ members = [
     "api/migration/migrations/v1.13.4/add-hostname-override",
     "api/migration/migrations/v1.13.4/add-hostname-override-metadata",
     "api/migration/migrations/v1.14.0/kubernetes-gc-percent-type-change",
+    "api/migration/migrations/v1.14.0/kubelet-config-settings",
 
     "bottlerocket-release",
 

diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/Cargo.toml b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "kubelet-config-settings"
+version = "0.1.0"
+authors = ["Sean McGinnis <[email protected]>"]
+license = "Apache-2.0 OR MIT"
+edition = "2021"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers", version = "0.1.0"}
diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs
@@ -0,0 +1,19 @@
+use migration_helpers::{common_migrations::AddSettingsMigration, migrate, Result};
+use std::process;
+
+/// Additional `settings.kubernetes` options for this release.
+fn run() -> Result<()> {
+    migrate(AddSettingsMigration(&[
+        "settings.kubernetes.cpu-manager-policy-options",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}
diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs
@@ -172,6 +172,7 @@ mod variant;
 // The "de" module contains custom deserialization trait implementation for models.
 mod de;
 
+use modeled_types::KubernetesCPUManagerPolicyOption;
 pub use variant::*;
 
 // Types used to communicate between client and server for 'apiclient exec'.
@@ -243,6 +244,7 @@ struct KubernetesSettings {
     container_log_max_files: i32,
     cpu_manager_policy: CpuManagerPolicy,
     cpu_manager_reconcile_period: KubernetesDurationValue,
+    cpu_manager_policy_options: Vec<KubernetesCPUManagerPolicyOption>,
     topology_manager_scope: TopologyManagerScope,
     topology_manager_policy: TopologyManagerPolicy,
     pod_pids_limit: i64,

diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs
@@ -1,5 +1,6 @@
 use lazy_static::lazy_static;
 use regex::Regex;
+use scalar_derive::Scalar;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 // Just need serde's Error in scope to get its trait methods
 use super::error;
@@ -1328,3 +1329,32 @@ pub struct CredentialProvider {
     cache_duration: Option<KubernetesDurationValue>,
     environment: Option<EnvVarMap>,
 }
+
+// =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
+
+/// KubernetesCPUManagerPolicyOption values are the possible option names for the cpuManagerPolicyOptions.
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Scalar)]
+pub enum KubernetesCPUManagerPolicyOption {
+    #[serde(rename = "full-pcpus-only")]
+    FullPCPUsOnly,
+}
+
+#[cfg(test)]
+mod test_kubernetes_cpu_manager_policy_option {
+    use super::KubernetesCPUManagerPolicyOption;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn good_cpu_manager_policy_option() {
+        for ok in &["full-pcpus-only"] {
+            KubernetesCPUManagerPolicyOption::try_from(*ok).unwrap();
+        }
+    }
+
+    #[test]
+    fn bad_cpu_manager_policy_option() {
+        for err in &["fullPCPUSOnly", "", "align-by-socket"] {
+            KubernetesCPUManagerPolicyOption::try_from(*err).unwrap_err();
+        }
+    }
+}