diff --git a/README.md b/README.md index ce32db6c3bf..ede3ae058a1 100644 --- a/README.md +++ b/README.md @@ -418,6 +418,18 @@ The following settings are optional and allow you to further configure your clus * `settings.kubernetes.container-log-max-files`: The maximum number of container log files that can be present for a container. * `settings.kubernetes.container-log-max-size`: The maximum size of container log file before it is rotated. * `settings.kubernetes.cpu-manager-policy`: Specifies the CPU manager policy. Possible values are `static` and `none`. Defaults to `none`. If you want to allow pods with certain resource characteristics to be granted increased CPU affinity and exclusivity on the node, you can set this setting to `static`. You should reboot if you change this setting after startup - try `apiclient reboot`. +* `settings.kubernetes.cpu-manager-policy-options`: Policy options to apply when `cpu-manager-policy` is set to `static`. Currently `full-pcpus-only` is the only option. + + For example: + + ```toml + [settings.kubernetes] + cpu-manager-policy = "static" + cpu-manager-policy-options = [ + "full-pcpus-only" + ] + ``` + * `settings.kubernetes.cpu-manager-reconcile-period`: Specifies the CPU manager reconcile period, which controls how often updated CPU assignments are written to cgroupfs. The value is a duration like `30s` for 30 seconds or `1h5m` for 1 hour and 5 minutes. * `settings.kubernetes.credential-providers`: Contains a collection of Kubelet image credential provider settings. Each name under `credential-providers` is the name of the plugin to configure. diff --git a/Release.toml b/Release.toml index 7ce663eb065..011e1eabf09 100644 --- a/Release.toml +++ b/Release.toml @@ -201,5 +201,6 @@ version = "1.14.0" "migrate_v1.13.4_add-hostname-override-metadata.lz4", ] "(1.13.4, 1.14.0)" = [ - "migrate_v1.14.0_kubernetes-gc-percent-type-change.lz4" + "migrate_v1.14.0_kubernetes-gc-percent-type-change.lz4", + "migrate_v1.14.0_kubernetes-config-settings.lz4", ] diff --git a/packages/kubernetes-1.22/kubelet-config b/packages/kubernetes-1.22/kubelet-config index a24b0f98779..736263131cd 100644 --- a/packages/kubernetes-1.22/kubelet-config +++ b/packages/kubernetes-1.22/kubelet-config @@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} {{/if}} +{{#if settings.kubernetes.cpu-manager-policy-options}} +cpuManagerPolicyOptions: +{{#each settings.kubernetes.cpu-manager-policy-options}} + {{this}}: "true" +{{/each}} +{{/if}} {{#if settings.kubernetes.topology-manager-scope}} topologyManagerScope: {{settings.kubernetes.topology-manager-scope}} {{/if}} diff --git a/packages/kubernetes-1.23/kubelet-config b/packages/kubernetes-1.23/kubelet-config index 631b9d73849..991f22b48a4 100644 --- a/packages/kubernetes-1.23/kubelet-config +++ b/packages/kubernetes-1.23/kubelet-config @@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} {{/if}} +{{#if settings.kubernetes.cpu-manager-policy-options}} +cpuManagerPolicyOptions: +{{#each settings.kubernetes.cpu-manager-policy-options}} + {{this}}: "true" +{{/each}} +{{/if}} {{#if settings.kubernetes.topology-manager-scope}} topologyManagerScope: {{settings.kubernetes.topology-manager-scope}} {{/if}} diff --git a/packages/kubernetes-1.24/kubelet-config b/packages/kubernetes-1.24/kubelet-config index df0250882e2..d1e6871a1bb 100644 --- a/packages/kubernetes-1.24/kubelet-config +++ b/packages/kubernetes-1.24/kubelet-config @@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} {{/if}} +{{#if settings.kubernetes.cpu-manager-policy-options}} +cpuManagerPolicyOptions: +{{#each settings.kubernetes.cpu-manager-policy-options}} + {{this}}: "true" +{{/each}} +{{/if}} {{#if settings.kubernetes.topology-manager-scope}} topologyManagerScope: {{settings.kubernetes.topology-manager-scope}} {{/if}} diff --git a/packages/kubernetes-1.25/kubelet-config b/packages/kubernetes-1.25/kubelet-config index df0250882e2..d1e6871a1bb 100644 --- a/packages/kubernetes-1.25/kubelet-config +++ b/packages/kubernetes-1.25/kubelet-config @@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} {{/if}} +{{#if settings.kubernetes.cpu-manager-policy-options}} +cpuManagerPolicyOptions: +{{#each settings.kubernetes.cpu-manager-policy-options}} + {{this}}: "true" +{{/each}} +{{/if}} {{#if settings.kubernetes.topology-manager-scope}} topologyManagerScope: {{settings.kubernetes.topology-manager-scope}} {{/if}} diff --git a/packages/kubernetes-1.26/kubelet-config b/packages/kubernetes-1.26/kubelet-config index df0250882e2..d1e6871a1bb 100644 --- a/packages/kubernetes-1.26/kubelet-config +++ b/packages/kubernetes-1.26/kubelet-config @@ -80,6 +80,12 @@ cpuManagerPolicy: {{default "none" settings.kubernetes.cpu-manager-policy}} {{#if settings.kubernetes.cpu-manager-reconcile-period}} cpuManagerReconcilePeriod: {{settings.kubernetes.cpu-manager-reconcile-period}} {{/if}} +{{#if settings.kubernetes.cpu-manager-policy-options}} +cpuManagerPolicyOptions: +{{#each settings.kubernetes.cpu-manager-policy-options}} + {{this}}: "true" +{{/each}} +{{/if}} {{#if settings.kubernetes.topology-manager-scope}} topologyManagerScope: {{settings.kubernetes.topology-manager-scope}} {{/if}} diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 17b698a028d..257a7a3459f 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -2233,6 +2233,13 @@ dependencies = [ "migration-helpers", ] +[[package]] +name = "kubelet-config-settings" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubernetes-gc-percent-type-change" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index 11a026a461f..0aa074bd4fb 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -45,6 +45,7 @@ members = [ "api/migration/migrations/v1.13.4/add-hostname-override", "api/migration/migrations/v1.13.4/add-hostname-override-metadata", "api/migration/migrations/v1.14.0/kubernetes-gc-percent-type-change", + "api/migration/migrations/v1.14.0/kubelet-config-settings", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/Cargo.toml b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/Cargo.toml new file mode 100644 index 00000000000..c36f5277924 --- /dev/null +++ b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kubelet-config-settings" +version = "0.1.0" +authors = ["Sean McGinnis "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers", version = "0.1.0"} diff --git a/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs new file mode 100644 index 00000000000..06ba834ddc9 --- /dev/null +++ b/sources/api/migration/migrations/v1.14.0/kubelet-config-settings/src/main.rs @@ -0,0 +1,19 @@ +use migration_helpers::{common_migrations::AddSettingsMigration, migrate, Result}; +use std::process; + +/// Additional `settings.kubernetes` options for this release. +fn run() -> Result<()> { + migrate(AddSettingsMigration(&[ + "settings.kubernetes.cpu-manager-policy-options", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index bb18af1e723..7b731fc538e 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -172,6 +172,7 @@ mod variant; // The "de" module contains custom deserialization trait implementation for models. mod de; +use modeled_types::KubernetesCPUManagerPolicyOption; pub use variant::*; // Types used to communicate between client and server for 'apiclient exec'. @@ -243,6 +244,7 @@ struct KubernetesSettings { container_log_max_files: i32, cpu_manager_policy: CpuManagerPolicy, cpu_manager_reconcile_period: KubernetesDurationValue, + cpu_manager_policy_options: Vec, topology_manager_scope: TopologyManagerScope, topology_manager_policy: TopologyManagerPolicy, pod_pids_limit: i64, diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs index 45c2b5f6c54..45b4b42ff53 100644 --- a/sources/models/src/modeled_types/kubernetes.rs +++ b/sources/models/src/modeled_types/kubernetes.rs @@ -1,5 +1,6 @@ use lazy_static::lazy_static; use regex::Regex; +use scalar_derive::Scalar; use serde::{Deserialize, Deserializer, Serialize, Serializer}; // Just need serde's Error in scope to get its trait methods use super::error; @@ -1328,3 +1329,32 @@ pub struct CredentialProvider { cache_duration: Option, environment: Option, } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesCPUManagerPolicyOption values are the possible option names for the cpuManagerPolicyOptions. +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Scalar)] +pub enum KubernetesCPUManagerPolicyOption { + #[serde(rename = "full-pcpus-only")] + FullPCPUsOnly, +} + +#[cfg(test)] +mod test_kubernetes_cpu_manager_policy_option { + use super::KubernetesCPUManagerPolicyOption; + use std::convert::TryFrom; + + #[test] + fn good_cpu_manager_policy_option() { + for ok in &["full-pcpus-only"] { + KubernetesCPUManagerPolicyOption::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_cpu_manager_policy_option() { + for err in &["fullPCPUSOnly", "", "align-by-socket"] { + KubernetesCPUManagerPolicyOption::try_from(*err).unwrap_err(); + } + } +}