diff --git a/.gitignore b/.gitignore index ea8c4bf..b5099e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /target +/benchmark/*.tsv + diff --git a/Cargo.toml b/Cargo.toml index 15d7757..c151524 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0" version = "0.3.2" edition = "2021" rust-version = "1.70.0" -exclude = [".github", ".gitignore", "*.sh", "*.tsv"] +exclude = [".github", ".gitignore", "*.sh", "benchmark/**/*"] [dependencies] thiserror = "1.0.56" diff --git a/benchmark.tsv b/benchmark.tsv deleted file mode 100644 index 0103b7d..0000000 --- a/benchmark.tsv +++ /dev/null @@ -1,20 +0,0 @@ -term iterations min_ns max_ns mean_ns -GNOME Console 10000 8157828 56823240 20656316 -XTerm 10000 33550 295990 39926 -Alacritty 10000 40340 414961 57569 -Konsole 10000 34110 3652145 38094 -VSCode 10000 24164008 140036258 26061349 -foot 10000 26130 248260 31825 -Terminal.app 10000 196143 25064408 241916 -iTerm2 10000 4065856 49872777 28259948 -IntelliJ IDEA 10000 71267 2453094 154491 -Hyper 10000 16287473 57534790 20040066 -QTerminal 10000 26220 4127641 37446 -linux 10000 15470 75190 16107 -WezTerm 10000 1174129 6400318 3461548 -kitty 10000 1412243 6343324 3137705 -Rio 10000 36940 1626094 56959 -rxvt-unicode 10000 27900 11974885 37092 -QMLKonsole 10000 25010 969482 27317 -cool-retro-term 10000 28070 3457008 35218 -Terminology 10000 30570 620351 36248 diff --git a/benchmark/measurements.Rmd b/benchmark/measurements.Rmd new file mode 100644 index 0000000..d1ee028 --- /dev/null +++ b/benchmark/measurements.Rmd @@ -0,0 +1,146 @@ +--- +title: "Measurements" +author: "Jan Hohenheim" +date: "`r Sys.Date()`" +header-includes: + - \usepackage{fontspec} +output: + pdf_document: + latex_engine: xelatex +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = TRUE) +``` + +## R Markdown + +```{r} +library(tidyverse); +library(ggthemes); +library(svglite) +theme_set(theme_solarized_2(light = TRUE)); +``` + +```{r} +dat_raw <- read_tsv("raw.tsv"); +dat_raw$term <- as.factor(dat_raw$term); +dat_raw$machine <- as.factor(dat_raw$machine) +dat_raw$supported <- as.logical(dat_raw$supported); +``` + +```{r} +message("Raw data"); +dat_raw |> summary(maxsum = max(lengths(lapply(dat_raw, unique)))) + +dat_raw |> + group_by(term) |> + summarise( + "mean [ns]" = mean(duration_ns), + "median [ns]" = median(duration_ns), + "sd [ns]" = sd(duration_ns), + ); +``` + +```{r} +message("Filtered data"); +alpha <- 0.05; +dat <- dat_raw |> + filter(duration_ns > quantile(duration_ns, alpha / 2) & duration_ns < quantile(duration_ns, 1 - alpha / 2)) |> + mutate(duration_us = duration_ns / 1000) |> + select(-duration_ns); + +dat$machine <- dat$machine |> + recode( + "linux" = "Linux Desktop", + "macbook" = "MacBook Pro" + ); + + +dat |> summary(maxsum = max(lengths(lapply(dat, unique)))); + +dat |> + group_by(term) |> + summarise( + "mean [μs]" = mean(duration_us), + "median [μs]" = median(duration_us), + "sd [μs]" = sd(duration_us), + ); + +``` + +## Violin plots + + +```{r} +for (current_term in unique(dat$term)) { + machine <- dat |> + filter(term == current_term) |> + pull(machine) |> + unique(); + plt <- dat |> + filter(term == current_term) |> + ggplot(aes(x = term, y = duration_us)) + + geom_violin() + + ggtitle(glue::glue("Violin plot for {current_term} on {machine}")) + + ylab("Duration [μs]"); + print(plt); +} +``` + + +## Histograms + +```{r} +for (current_term in unique(dat$term)) { + machine <- dat |> + filter(term == current_term) |> + pull(machine) |> + unique(); + plt <- dat |> + filter(term == current_term) |> + ggplot(aes(x = duration_us)) + + geom_histogram(bins = 200) + + ggtitle(glue::glue("Histogram for {current_term} on {machine}")) + + xlab("Duration [μs]"); + print(plt); +} +``` + +## Median plot + +```{r} +dat.median <- dat |> + group_by(term, machine) |> + summarise( + median = median(duration_us), + supported = ifelse(first(supported), "True", "False"), + fast = median(duration_us) < 2000, + .groups = "keep", + ); + +dat.median |> + filter(fast) |> + ggplot(aes(x = term, y = median, fill = supported)) + + geom_bar(stat = "identity", position = "dodge") + + ggtitle("Median duration per terminal for fast terminals") + + ylab("Median duration [μs]") + + xlab("Term") + + scale_fill_manual(values = c(True = "steelblue", False = "coral2")) + + theme(axis.text.x = element_text(angle = 45, hjust = 1)); + +ggsave("measurements_fast.svg", width = 10, height = 8) + +dat.median |> + filter(!fast) |> + ggplot(aes(x = term, y = median, fill = supported)) + + geom_bar(stat = "identity", position = "dodge") + + ggtitle("Median duration per terminal for slow terminals") + + ylab("Median duration [μs]") + + xlab("Term") + + scale_fill_manual(values = c(True = "steelblue", False = "coral2")) + + theme(axis.text.x = element_text(angle = 45, hjust = 1)); + +ggsave("measurements_slow.svg", width = 10, height = 8) +``` + diff --git a/benchmark/measurements.pdf b/benchmark/measurements.pdf new file mode 100644 index 0000000..0fff119 Binary files /dev/null and b/benchmark/measurements.pdf differ diff --git a/benchmark/measurements_fast_dark.svg b/benchmark/measurements_fast_dark.svg new file mode 100644 index 0000000..6576004 --- /dev/null +++ b/benchmark/measurements_fast_dark.svg @@ -0,0 +1 @@ +050100150200Alacrittycool-retro-termFleetfootIntelliJ IDEAKonsolelinuxQMLKonsoleQTerminalRiorxvt-unicodeTerminal.appTerminologyxtermTermMedian duration [μs]supportedFalseTrueMedian duration per terminal for fast terminals \ No newline at end of file diff --git a/benchmark/measurements_fast_light.svg b/benchmark/measurements_fast_light.svg new file mode 100644 index 0000000..20776b5 --- /dev/null +++ b/benchmark/measurements_fast_light.svg @@ -0,0 +1 @@ +050100150200Alacrittycool-retro-termFleetfootIntelliJ IDEAKonsolelinuxQMLKonsoleQTerminalRiorxvt-unicodeTerminal.appTerminologyxtermTermMedian duration [μs]supportedFalseTrueMedian duration per terminal for fast terminals \ No newline at end of file diff --git a/benchmark/measurements_slow_dark.svg b/benchmark/measurements_slow_dark.svg new file mode 100644 index 0000000..ce55ec6 --- /dev/null +++ b/benchmark/measurements_slow_dark.svg @@ -0,0 +1 @@ +01000020000HyperiTerm2kittyVSCodevteWezTermTermMedian duration [μs]supportedTrueMedian duration per terminal for slow terminals \ No newline at end of file diff --git a/benchmark/measurements_slow_light.svg b/benchmark/measurements_slow_light.svg new file mode 100644 index 0000000..626bec6 --- /dev/null +++ b/benchmark/measurements_slow_light.svg @@ -0,0 +1 @@ +01000020000HyperiTerm2kittyVSCodevteWezTermTermMedian duration [μs]supportedTrueMedian duration per terminal for slow terminals \ No newline at end of file diff --git a/benchmark/raw.tsv.gz b/benchmark/raw.tsv.gz new file mode 100644 index 0000000..7f1ad8f Binary files /dev/null and b/benchmark/raw.tsv.gz differ diff --git a/doc/latency.md b/doc/latency.md index 4d13f35..43a9d56 100644 --- a/doc/latency.md +++ b/doc/latency.md @@ -4,27 +4,18 @@ Measurements generated using [examples/benchmark](../examples/benchmark/src/main cargo run --release -p benchmark '' ``` -| Terminal | Iterations | min | max | mean | supported | -|---------------------|------------|--------------|---------------|--------------|-----------| -| foot | 10000 | 26.130 µs | 248.260 µs | 31.825 µs | yes | -| XTerm | 10000 | 33.550 µs | 295.990 µs | 39.926 µs | yes | -| Konsole | 10000 | 34.110 µs | 3.652145 ms | 38.094 µs | yes | -| Alacritty | 10000 | 40.340 µs | 414.961 µs | 57.569 µs | yes | -| IntelliJ IDEA | 10000 | 71.267 µs | 2.453094 ms | 154.491 µs | yes | -| Terminal.app | 10000 | 196.143 µs | 25.064408 ms | 241.916 µs | yes | -| Hyper | 10000 | 16.287473 ms | 57.534790 ms | 20.040066 ms | yes | -| GNOME Console (vte) | 10000 | 8.157828 ms | 56.823240 ms | 20.656316 ms | yes | -| VSCode | 10000 | 24.164008 ms | 140.036258 ms | 26.061349 ms | yes | -| iTerm2 | 10000 | 4.065856 ms | 49.872777 ms | 28.259948 ms | yes | -| QTerminal | 10000 | 26.22 µs | 4.127641 ms | 37.446 µs | no | -| linux | 10000 | 15.47 µs | 75.19 µs | 16.107 µs | no | -| WezTerm | 10000 | 1.174129 ms | 6.400318 ms | 3.461548 ms | yes | -| kitty | 10000 | 1.412243 ms | 6.343324 ms | 3.137705 ms | yes | -| Rio | 10000 | 36.94 µs | 1.626094 ms | 56.959 µs | yes | -| rxvt-unicode | 10000 | 27.9 µs | 11.97489 ms | 37.092 µs | yes | -| QMLKonsole | 10000 | 25.01 µs | 0.969482 ms | 27.317 µs | no | -| cool-retro-term | 10000 | 28.07 µs | 3.457008 ms | 35.218 µs | no | -| Terminology | 10000 | 30.57 µs | 0.620351 ms | 36.248 µs | yes | +## Fast Terminals + + + + + +## Slow Terminals + + + + + **ℹ️ Note:** The macOS terminals were not tested on the same machine as the Linux terminals. diff --git a/examples/benchmark/src/main.rs b/examples/benchmark/src/main.rs index 6e74f36..b4efa15 100644 --- a/examples/benchmark/src/main.rs +++ b/examples/benchmark/src/main.rs @@ -56,15 +56,9 @@ fn save_results(results: &[Duration], term: String) -> io::Result<()> { let mut file = OpenOptions::new() .append(true) .create(true) - .open("benchmark.tsv")?; - writeln!( - file, - "{}\t{}\t{}\t{}\t{}", - term, - results.len(), - results.iter().min().unwrap().as_nanos(), - results.iter().max().unwrap().as_nanos(), - (results.iter().sum::() / results.len() as u32).as_nanos(), - )?; + .open("benchmark/raw.tsv")?; + for result in results { + writeln!(file, "{}\t{}", term, result.as_nanos())?; + } Ok(()) }