Add resctrl data collection support

Summary: Add support for collecting data from resctrl filesystem (see https://www.kernel.org/doc/html/v6.4/arch/x86/resctrl.html). Reviewed By: dschatzberg Differential Revision: D51767787 fbshipit-source-id: 385024d71d5ceccfe43f03e69e81359b35667b74
facebookincubator · Dec 6, 2023 · 3a85501 · 3a85501
1 parent 5151c5a
commit 3a85501
Show file tree

Hide file tree

Showing 9 changed files with 408 additions and 1 deletion.
diff --git a/below/config/src/lib.rs b/below/config/src/lib.rs
@@ -46,6 +46,7 @@ pub struct BelowConfig {
     pub btrfs_samples: u64,
     pub btrfs_min_pct: f64,
     pub enable_ethtool_stats: bool,
+    pub enable_resctrl_stats: bool,
 }
 
 impl Default for BelowConfig {
@@ -61,6 +62,7 @@ impl Default for BelowConfig {
             btrfs_samples: btrfs::DEFAULT_SAMPLES,
             btrfs_min_pct: btrfs::DEFAULT_MIN_PCT,
             enable_ethtool_stats: false,
+            enable_resctrl_stats: false,
         }
     }
 }

diff --git a/below/dump/src/test.rs b/below/dump/src/test.rs
@@ -986,6 +986,7 @@ fn test_dump_queue_content() {
         process: model::ProcessModel::default(),
         network,
         gpu: None,
+        resctrl: None,
     };
 
     let mut opts: GeneralOpt = Default::default();

diff --git a/below/model/Cargo.toml b/below/model/Cargo.toml
@@ -23,6 +23,7 @@ hostname = "0.3"
 os_info = "3.0.7"
 procfs = { package = "fb_procfs", version = "0.7.1", path = "../procfs" }
 regex = "1.9.2"
+resctrlfs = { version = "0.7.1", path = "../resctrlfs" }
 serde = { version = "1.0.185", features = ["derive", "rc"] }
 serde_json = { version = "1.0.100", features = ["float_roundtrip", "unbounded_depth"] }
 slog = { version = "2.7", features = ["max_level_trace", "nested-values"] }

diff --git a/below/model/src/collector.rs b/below/model/src/collector.rs
@@ -30,6 +30,7 @@ pub struct CollectorOptions {
     pub disable_disk_stat: bool,
     pub enable_btrfs_stats: bool,
     pub enable_ethtool_stats: bool,
+    pub enable_resctrl_stats: bool,
     pub btrfs_samples: u64,
     pub btrfs_min_pct: f64,
     pub cgroup_re: Option<Regex>,
@@ -46,6 +47,7 @@ impl Default for CollectorOptions {
             disable_disk_stat: false,
             enable_btrfs_stats: false,
             enable_ethtool_stats: false,
+            enable_resctrl_stats: false,
             btrfs_samples: btrfs::DEFAULT_SAMPLES,
             btrfs_min_pct: btrfs::DEFAULT_MIN_PCT,
             cgroup_re: None,
@@ -287,6 +289,24 @@ fn collect_sample(logger: &slog::Logger, options: &CollectorOptions) -> Result<S
                 }
             }
         },
+        resctrl: if !options.enable_resctrl_stats {
+            None
+        } else {
+            match resctrlfs::ResctrlReader::root() {
+                Ok(resctrl_reader) => match resctrl_reader.read_all() {
+                    Ok(resctrl) => Some(resctrl),
+                    Err(e) => {
+                        error!(logger, "{:#}", e);
+                        None
+                    }
+                },
+                Err(_e) => {
+                    // ResctrlReader only fails to initialize if resctrlfs is
+                    // not mounted. In this case we ignore.
+                    None
+                }
+            }
+        },
     })
 }
 

diff --git a/below/model/src/common_field_ids.rs b/below/model/src/common_field_ids.rs
@@ -23,7 +23,7 @@
 ///
 /// This list also servers as documentation for available field ids that could
 /// be used in other below crates. A test ensures that this list is up-to-date.
-pub const COMMON_MODEL_FIELD_IDS: [&str; 374] = [
+pub const COMMON_MODEL_FIELD_IDS: [&str; 408] = [
     "system.hostname",
     "system.kernel_version",
     "system.os_release",
@@ -254,6 +254,40 @@ pub const COMMON_MODEL_FIELD_IDS: [&str; 374] = [
     "cgroup.[path:/<cgroup_path>/.]mem_numa.<key>.workingset_restore_anon",
     "cgroup.[path:/<cgroup_path>/.]mem_numa.<key>.workingset_restore_file",
     "cgroup.[path:/<cgroup_path>/.]mem_numa.<key>.workingset_nodereclaim",
+    "resctrl.cpuset",
+    "resctrl.ctrl_mon_groups.<key>.cpuset",
+    "resctrl.ctrl_mon_groups.<key>.full_path",
+    "resctrl.ctrl_mon_groups.<key>.mode",
+    "resctrl.ctrl_mon_groups.<key>.mon.per_l3.<key>.llc_occupancy_bytes",
+    "resctrl.ctrl_mon_groups.<key>.mon.per_l3.<key>.mbm_local_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon.per_l3.<key>.mbm_total_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon.total.llc_occupancy_bytes",
+    "resctrl.ctrl_mon_groups.<key>.mon.total.mbm_local_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon.total.mbm_total_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.full_path",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.per_l3.<key>.llc_occupancy_bytes",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.per_l3.<key>.mbm_local_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.per_l3.<key>.mbm_total_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.total.llc_occupancy_bytes",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.total.mbm_local_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.mon.total.mbm_total_bytes_per_sec",
+    "resctrl.ctrl_mon_groups.<key>.mon_groups.<key>.name",
+    "resctrl.ctrl_mon_groups.<key>.name",
+    "resctrl.mode",
+    "resctrl.mon.per_l3.<key>.llc_occupancy_bytes",
+    "resctrl.mon.per_l3.<key>.mbm_local_bytes_per_sec",
+    "resctrl.mon.per_l3.<key>.mbm_total_bytes_per_sec",
+    "resctrl.mon.total.llc_occupancy_bytes",
+    "resctrl.mon.total.mbm_local_bytes_per_sec",
+    "resctrl.mon.total.mbm_total_bytes_per_sec",
+    "resctrl.mon_groups.<key>.full_path",
+    "resctrl.mon_groups.<key>.mon.per_l3.<key>.llc_occupancy_bytes",
+    "resctrl.mon_groups.<key>.mon.per_l3.<key>.mbm_local_bytes_per_sec",
+    "resctrl.mon_groups.<key>.mon.per_l3.<key>.mbm_total_bytes_per_sec",
+    "resctrl.mon_groups.<key>.mon.total.llc_occupancy_bytes",
+    "resctrl.mon_groups.<key>.mon.total.mbm_local_bytes_per_sec",
+    "resctrl.mon_groups.<key>.mon.total.mbm_total_bytes_per_sec",
+    "resctrl.mon_groups.<key>.name",
     "process.processes.<key>.pid",
     "process.processes.<key>.ppid",
     "process.processes.<key>.ns_tgid",

diff --git a/below/model/src/lib.rs b/below/model/src/lib.rs
@@ -36,6 +36,7 @@ pub mod collector_plugin;
 mod common_field_ids;
 pub mod network;
 pub mod process;
+pub mod resctrl;
 pub mod sample;
 mod sample_model;
 pub mod system;
@@ -46,6 +47,7 @@ pub use cgroup::*;
 pub use collector::*;
 pub use network::*;
 pub use process::*;
+pub use resctrl::*;
 pub use sample::*;
 pub use system::*;
 
@@ -66,6 +68,8 @@ pub enum Field {
     StrU64Map(BTreeMap<String, u64>),
     Cpuset(cgroupfs::Cpuset),
     MemNodes(cgroupfs::MemNodes),
+    ResctrlCpuset(resctrlfs::Cpuset),
+    ResctrlGroupMode(resctrlfs::GroupMode),
 }
 
 impl From<Field> for u64 {
@@ -196,6 +200,18 @@ impl From<cgroupfs::MemNodes> for Field {
     }
 }
 
+impl From<resctrlfs::Cpuset> for Field {
+    fn from(v: resctrlfs::Cpuset) -> Self {
+        Field::ResctrlCpuset(v)
+    }
+}
+
+impl From<resctrlfs::GroupMode> for Field {
+    fn from(v: resctrlfs::GroupMode) -> Self {
+        Field::ResctrlGroupMode(v)
+    }
+}
+
 impl<T: Into<Field> + Clone> From<&T> for Field {
     fn from(v: &T) -> Self {
         v.clone().into()
@@ -284,6 +300,8 @@ impl fmt::Display for Field {
             )),
             Field::Cpuset(v) => v.fmt(f),
             Field::MemNodes(v) => v.fmt(f),
+            Field::ResctrlCpuset(v) => v.fmt(f),
+            Field::ResctrlGroupMode(v) => v.fmt(f),
         }
     }
 }
@@ -515,6 +533,8 @@ pub struct Model {
     pub network: NetworkModel,
     #[queriable(subquery)]
     pub gpu: Option<GpuModel>,
+    #[queriable(subquery)]
+    pub resctrl: Option<ResctrlModel>,
 }
 
 impl Model {
@@ -563,6 +583,16 @@ impl Model {
                     }
                 })
             }),
+            resctrl: sample.resctrl.as_ref().map(|r| {
+                ResctrlModel::new(
+                    r,
+                    if let Some((s, d)) = last {
+                        s.resctrl.as_ref().map(|r| (r, d))
+                    } else {
+                        None
+                    },
+                )
+            }),
         }
     }
 }