From 2eb8d9295fff2db2b1b74c3bb09206b22f1c2f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 9 Aug 2023 20:56:11 +0200 Subject: [PATCH] Add artifact size and step duration summaries from `opt-dist` to github job summary --- Cargo.lock | 60 ++++++++++++++++++ src/ci/docker/run.sh | 6 ++ src/tools/opt-dist/Cargo.toml | 1 + src/tools/opt-dist/src/main.rs | 12 +++- src/tools/opt-dist/src/utils/artifact_size.rs | 61 +++++++++++++++++++ src/tools/opt-dist/src/utils/mod.rs | 51 ++++++++-------- 6 files changed, 162 insertions(+), 29 deletions(-) create mode 100644 src/tools/opt-dist/src/utils/artifact_size.rs diff --git a/Cargo.lock b/Cargo.lock index 3b270c14bb5c9..ce0740e2ff02b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,6 +2547,7 @@ dependencies = [ "serde", "serde_json", "sysinfo", + "tabled", "tar", "tempfile", "xz", @@ -2598,6 +2599,17 @@ dependencies = [ "unwind", ] +[[package]] +name = "papergrid" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ccbe15f2b6db62f9a9871642746427e297b0ceb85f9a7f1ee5ff47d184d0c8" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2790,6 +2802,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -5023,6 +5059,30 @@ dependencies = [ "test", ] +[[package]] +name = "tabled" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d38d39c754ae037a9bc3ca1580a985db7371cd14f1229172d1db9093feb6739" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tar" version = "0.4.38" diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 87db964a15f0a..b7e21f7f5992b 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -247,6 +247,9 @@ else BASE_COMMIT="" fi +SUMMARY_FILE=github-summary.md +touch $objdir/${SUMMARY_FILE} + docker \ run \ --workdir /checkout/obj \ @@ -258,6 +261,7 @@ docker \ --env CI \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ + --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ @@ -272,6 +276,8 @@ docker \ rust-ci \ $command +cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" + if [ -f /.dockerenv ]; then rm -rf $objdir docker cp checkout:/checkout/obj $objdir diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index 3f7dba81c3a9f..5f4fc5ca52828 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -21,3 +21,4 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" glob = "0.3" tempfile = "3.5" +tabled = "0.13" diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs index f6edd008845ce..cd1c483f140df 100644 --- a/src/tools/opt-dist/src/main.rs +++ b/src/tools/opt-dist/src/main.rs @@ -1,17 +1,19 @@ -use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use anyhow::Context; use log::LevelFilter; + use utils::io; +use crate::bolt::{bolt_optimize, with_bolt_instrumented}; use crate::environment::{create_environment, Environment}; use crate::exec::Bootstrap; use crate::tests::run_tests; use crate::timer::Timer; use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles}; +use crate::utils::artifact_size::print_binary_sizes; use crate::utils::io::reset_directory; use crate::utils::{ - clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space, - with_log_group, + clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group, + write_timer_to_summary, }; mod bolt; @@ -207,6 +209,10 @@ fn main() -> anyhow::Result<()> { let result = execute_pipeline(env.as_ref(), &mut timer, build_args); log::info!("Timer results\n{}", timer.format_stats()); + if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") { + write_timer_to_summary(&summary_path, &timer)?; + } + print_free_disk_space()?; result.context("Optimized build pipeline has failed")?; print_binary_sizes(env.as_ref())?; diff --git a/src/tools/opt-dist/src/utils/artifact_size.rs b/src/tools/opt-dist/src/utils/artifact_size.rs new file mode 100644 index 0000000000000..c587c4793f05e --- /dev/null +++ b/src/tools/opt-dist/src/utils/artifact_size.rs @@ -0,0 +1,61 @@ +use std::io::Write; + +use tabled::builder::Builder; +use tabled::settings::object::Columns; +use tabled::settings::style::{BorderChar, Offset}; +use tabled::settings::{Modify, Style}; + +use crate::environment::Environment; +use crate::utils::io::get_files_from_dir; + +pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { + use humansize::format_size; + use humansize::BINARY; + use std::fmt::Write; + + let root = env.build_artifacts().join("stage2"); + + let mut files = get_files_from_dir(&root.join("bin"), None)?; + files.extend(get_files_from_dir(&root.join("lib"), Some(".so"))?); + files.sort_unstable(); + + let items: Vec<_> = files + .into_iter() + .map(|file| { + let size = std::fs::metadata(file.as_std_path()).map(|m| m.len()).unwrap_or(0); + let size_formatted = format_size(size, BINARY); + let name = file.file_name().unwrap().to_string(); + (name, size_formatted) + }) + .collect(); + + // Write to log + let mut output = String::new(); + for (name, size_formatted) in items.iter() { + let name = format!("{}:", name); + writeln!(output, "{name:<50}{size_formatted:>10}")?; + } + log::info!("Rustc artifact size\n{output}"); + + // Write to GitHub summary + if let Ok(summary_path) = std::env::var("GITHUB_STEP_SUMMARY") { + let mut builder = Builder::default(); + for (name, size_formatted) in items { + builder.push_record(vec![name, size_formatted]); + } + + builder.set_header(vec!["Artifact", "Size"]); + let mut table = builder.build(); + + let mut file = std::fs::File::options().append(true).create(true).open(summary_path)?; + writeln!( + file, + "# Artifact size\n{}\n", + table.with(Style::markdown()).with( + Modify::new(Columns::single(1)).with(BorderChar::horizontal(':', Offset::End(0))), + ) + )?; + } + + Ok(()) +} diff --git a/src/tools/opt-dist/src/utils/mod.rs b/src/tools/opt-dist/src/utils/mod.rs index 9a3df15e302bb..9440da4bf0981 100644 --- a/src/tools/opt-dist/src/utils/mod.rs +++ b/src/tools/opt-dist/src/utils/mod.rs @@ -1,9 +1,12 @@ -pub mod io; +use humansize::BINARY; +use sysinfo::{DiskExt, RefreshKind, System, SystemExt}; use crate::environment::Environment; -use crate::utils::io::{delete_directory, get_files_from_dir}; -use humansize::{format_size, BINARY}; -use sysinfo::{DiskExt, RefreshKind, System, SystemExt}; +use crate::timer::Timer; +use crate::utils::io::delete_directory; + +pub mod artifact_size; +pub mod io; pub fn format_env_variables() -> String { let vars = std::env::vars().map(|(key, value)| format!("{key}={value}")).collect::>(); @@ -25,28 +28,6 @@ pub fn print_free_disk_space() -> anyhow::Result<()> { Ok(()) } -pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> { - use std::fmt::Write; - - let root = env.build_artifacts().join("stage2"); - - let mut files = get_files_from_dir(&root.join("bin"), None)?; - files.extend(get_files_from_dir(&root.join("lib"), Some(".so"))?); - files.sort_unstable(); - - let mut output = String::new(); - for file in files { - let size = std::fs::metadata(file.as_std_path())?.len(); - let size_formatted = format_size(size, BINARY); - let name = format!("{}:", file.file_name().unwrap()); - writeln!(output, "{name:<50}{size_formatted:>10}")?; - } - - log::info!("Rustc artifact size\n{output}"); - - Ok(()) -} - pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { // Bootstrap currently doesn't support rebuilding LLVM when PGO options // change (or any other llvm-related options); so just clear out the relevant @@ -57,6 +38,24 @@ pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> { Ok(()) } +/// Write the formatted statistics of the timer to a Github Actions summary. +pub fn write_timer_to_summary(path: &str, timer: &Timer) -> anyhow::Result<()> { + use std::io::Write; + + let mut file = std::fs::File::options().append(true).create(true).open(path)?; + writeln!( + file, + r#"# Step durations + +``` +{} +``` +"#, + timer.format_stats() + )?; + Ok(()) +} + /// Wraps all output produced within the `func` closure in a CI output group, if we're running in /// CI. pub fn with_log_group R, R>(group: &str, func: F) -> R {