diff --git a/.cz.toml b/.cz.toml index f876f4b..d03ee28 100644 --- a/.cz.toml +++ b/.cz.toml @@ -12,7 +12,7 @@ schema = ": " schema_pattern = "(feat|fix):(\\s.*)" [[tool.commitizen.customize.questions]] -choices = ["feat", "fix", "doc", "ci", "break", "doc"] # short version +choices = ["feat", "fix", "doc", "ci", "break", "doc", "chore"] # short version message = "Select the type of change you are committing" name = "change_type" type = "list" diff --git a/.github/workflows/bump.yaml b/.github/workflows/bump.yaml index f67eaf8..a7c0b8c 100644 --- a/.github/workflows/bump.yaml +++ b/.github/workflows/bump.yaml @@ -14,8 +14,6 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest name: Bump - outputs: - should-publish: ${{ steps.add-and-commit.outputs.committed }} steps: - name: Check out uses: actions/checkout@v2 @@ -54,21 +52,7 @@ jobs: tag_name: ${{ env.REVISION }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - release-crate: - runs-on: ubuntu-latest - needs: bump - if: ${{ needs.bump.outputs.should-publish }} - name: Release Crate - steps: - - name: Check out - uses: actions/checkout@v2 - with: - token: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" - fetch-depth: 0 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - uses: Swatinem/rust-cache@v1 - uses: katyo/publish-crates@v1 + if: ${{ steps.add-and-commit.outputs.committed }} with: registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index fe8fca4..d7c69ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,6 +32,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + [[package]] name = "arrayvec" version = "0.4.12" @@ -41,6 +47,12 @@ dependencies = [ "nodrop", ] +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "async-trait" version = "0.1.52" @@ -81,6 +93,17 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec 0.5.2", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -102,6 +125,18 @@ dependencies = [ "byte-tools", ] +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "lazy_static", + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.8.0" @@ -182,6 +217,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.9.2" @@ -252,6 +293,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "dashmap" version = "5.0.0" @@ -272,6 +335,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -307,14 +381,16 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fatigue" -version = "0.2.2" +version = "0.2.3" dependencies = [ "clap", "console", "crossterm", + "humantime", "itertools", "libfatigue", "num-format", + "prettytable-rs", "serde", "serde_json", "thiserror", @@ -462,6 +538,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.3" @@ -470,7 +557,7 @@ checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -706,7 +793,7 @@ checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "libfatigue" -version = "0.2.2" +version = "0.2.3" dependencies = [ "async-trait", "dashmap", @@ -918,7 +1005,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ - "arrayvec", + "arrayvec 0.4.12", "itoa 0.4.8", ] @@ -1016,7 +1103,7 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.10", "smallvec", "winapi", ] @@ -1105,6 +1192,20 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" +[[package]] +name = "prettytable-rs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd04b170004fa2daccf418a7f8253aaf033c27760b5f225889024cf66d7ac2e" +dependencies = [ + "atty", + "csv", + "encode_unicode", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1205,7 +1306,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.3", ] [[package]] @@ -1217,6 +1318,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1226,6 +1333,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + [[package]] name = "regex" version = "1.5.4" @@ -1237,6 +1355,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" version = "0.6.25" @@ -1287,6 +1411,18 @@ dependencies = [ "winreg", ] +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + [[package]] name = "ryu" version = "1.0.9" @@ -1501,11 +1637,22 @@ dependencies = [ "cfg-if", "libc", "rand", - "redox_syscall", + "redox_syscall 0.2.10", "remove_dir_all", "winapi", ] +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi", +] + [[package]] name = "term_size" version = "0.3.2" @@ -1752,6 +1899,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" diff --git a/examples/simple_plan.yaml b/examples/simple_plan.yaml index 0d62aa6..f40429e 100644 --- a/examples/simple_plan.yaml +++ b/examples/simple_plan.yaml @@ -1,9 +1,9 @@ run: base_url: http://localhost:8000 - concurrency: 2 + concurrency: 70 duration: timed: - duration: 20s + duration: 2m warm_up: 1s iterations: 1 static_context: diff --git a/fatigue/Cargo.toml b/fatigue/Cargo.toml index 70ac614..447b7bf 100644 --- a/fatigue/Cargo.toml +++ b/fatigue/Cargo.toml @@ -14,9 +14,11 @@ license = "Apache-2.0" clap = { version = "2.34.0", features = ["suggestions", "color", "wrap_help"] } console = "0.15.0" crossterm = "0.22.1" +humantime = "2.1.0" itertools = "0.10.3" libfatigue = { path = "../libfatigue", version = "0.2.4"} num-format = "0.4.0" +prettytable-rs = "0.8.0" serde = "1.0.133" serde_json = "1.0.74" thiserror = "1.0.30" diff --git a/fatigue/src/main.rs b/fatigue/src/main.rs index fb5ec9b..55e6d5b 100644 --- a/fatigue/src/main.rs +++ b/fatigue/src/main.rs @@ -2,6 +2,8 @@ extern crate thiserror; #[macro_use] extern crate clap; +#[macro_use] +extern crate prettytable; use crate::output::{get_output_formatter, OutputFormatter}; use clap::{App, Arg}; diff --git a/fatigue/src/output/pretty.rs b/fatigue/src/output/pretty.rs index 6574f0d..40829ca 100644 --- a/fatigue/src/output/pretty.rs +++ b/fatigue/src/output/pretty.rs @@ -1,15 +1,20 @@ use crate::output::OutputFormatter; use console::Style; -use crossterm::cursor::MoveTo; -use crossterm::style::Print; +use crossterm::cursor::{Hide, MoveDown, MoveTo}; +use crossterm::style::{Attribute, Print, SetAttribute}; use crossterm::terminal::{Clear, ClearType}; use crossterm::{ExecutableCommand, QueueableCommand}; +use humantime::format_duration; use itertools::Itertools; -use libfatigue::context::TestResult; +use libfatigue::context::{TestDurationStatus, TestResult}; use libfatigue::FatigueTestError; use num_format::{Locale, ToFormattedString}; +use prettytable::format::consts::FORMAT_CLEAN; +use prettytable::format::Alignment; +use prettytable::{row, Cell, Row, Table}; use std::io::{stdout, Stdout, Write}; use std::sync::Mutex; +use std::time::Duration; pub(crate) struct PrettyOutputFormatter { inner: Mutex, @@ -19,7 +24,7 @@ struct PrettyInner { cyan: Style, blue: Style, _red: Style, - _yellow: Style, + yellow: Style, _green: Style, out: Stdout, } @@ -29,6 +34,7 @@ impl PrettyOutputFormatter { let mut out = stdout(); out.execute(Clear(ClearType::All)) .expect("todo: result handling"); + out.execute(Hide).expect("todo: result handling"); let cyan = Style::new().cyan(); let blue = Style::new().blue(); let red = Style::new().red(); @@ -38,7 +44,7 @@ impl PrettyOutputFormatter { cyan, blue, _red: red, - _yellow: yellow, + yellow, _green: green, out, }; @@ -71,29 +77,86 @@ impl PrettyInner { self.out .queue(Clear(ClearType::All)) .expect("todo: result handling"); - for test_name in val.timings.keys().sorted() { - self.out - .queue(Print(format!( - "* {}\n", - self.cyan.apply_to(test_name.as_str()) - ))) - .expect("todo: result handling"); + self.display_status_row(val); + + let table = self.build_output_table(val); + self.out.queue(Print(table)).expect("todo: result handling"); + self.out.flush().expect("todo: result handling"); + } + + fn display_status_row(&mut self, val: &TestResult) { + let rps = format!( + "rps: {:.2}\n", + self.yellow.apply_to(val.requests_per_second) + ); + let mut rps = Cell::new(rps.as_str()); + rps.align(Alignment::LEFT); + + let remaining = match &val.duration { + None => String::new(), + Some(duration_status) => match duration_status { + TestDurationStatus::Iteration { current, until } => { + format!( + "Iterations: {} / {}", + current.to_formatted_string(&Locale::en), + until.to_formatted_string(&Locale::en) + ) + } + TestDurationStatus::Timed { remaining } => { + let cleansed = Duration::new(remaining.as_secs(), 0); + let readable = format_duration(cleansed); + + format!("Remaining: {}", readable) + } + }, + }; + let mut remaining = Cell::new(remaining.as_str()); + remaining.align(Alignment::RIGHT); + let remaining = remaining.with_hspan(20); + + let mut table = Table::new(); + table.set_format(*FORMAT_CLEAN); + + table.add_row(Row::new(vec![rps, remaining])); + self.out + .queue(SetAttribute(Attribute::Bold)) + .expect("todo: result handling"); + self.out.queue(Print(table)).expect("todo: result handling"); + self.out + .queue(SetAttribute(Attribute::Reset)) + .expect("todo: result handling"); + self.out.queue(MoveDown(1)).expect("todo: result handling"); + } + + fn build_output_table(&self, val: &TestResult) -> Table { + let mut table = Table::new(); + table.set_format(*FORMAT_CLEAN); + table.set_titles(row![ + "Test", + "Status", + "# Requests", + "Median ms", + "95th % ms", + "99.999% ms" + ]); + + for test_name in val.timings.keys().sorted() { + table.add_row(row![self.cyan.apply_to(test_name.as_str())]); let status_map = &val.timings[test_name]; for status_code in status_map.keys().sorted() { let timings = &status_map[status_code]; - self.out - .queue(Print(format!( - "\t- {}\t{}\t{}\t{}\n", - self.blue.apply_to(&status_code.as_str()[..3]), - timings.metric_len.to_formatted_string(&Locale::en), - timings.pct_995_ms, - timings.median_ms - ))) - .expect("todo: result handling"); + table.add_row(row![ + "", + self.blue.apply_to(&status_code.as_str()), + timings.metric_len.to_formatted_string(&Locale::en), + timings.median_ms, + timings.pct_995_ms, + timings.pct_99999_ms + ]); } } - self.out.flush().expect("todo: result handling"); + table } } diff --git a/libfatigue/src/context/iteration.rs b/libfatigue/src/context/iteration.rs index dc3d2d6..a9bd522 100644 --- a/libfatigue/src/context/iteration.rs +++ b/libfatigue/src/context/iteration.rs @@ -1,4 +1,4 @@ -use crate::context::TestDurationTracker; +use crate::context::{TestDurationStatus, TestDurationTracker}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use tokio::sync::RwLock; @@ -44,6 +44,13 @@ impl TestDurationTracker for IterationDurationTracker { async fn should_track_iteration(&self) -> bool { self.warm_up.is_done().await } + + fn get_status(&self) -> TestDurationStatus { + TestDurationStatus::Iteration { + until: self.iterations, + current: self.count.load(Ordering::Relaxed), + } + } } pub(crate) struct TimedDurationTracker { @@ -86,6 +93,12 @@ impl TestDurationTracker for TimedDurationTracker { async fn should_track_iteration(&self) -> bool { self.warm_up.is_done().await } + + fn get_status(&self) -> TestDurationStatus { + TestDurationStatus::Timed { + remaining: self.ends_at - Instant::now(), + } + } } struct WarmUpTracker { diff --git a/libfatigue/src/context/mod.rs b/libfatigue/src/context/mod.rs index 2702a7c..eaea6b2 100644 --- a/libfatigue/src/context/mod.rs +++ b/libfatigue/src/context/mod.rs @@ -8,6 +8,7 @@ use hdrhistogram::Histogram; use liquid::model::{to_value, Value}; use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use tokio::sync::RwLock; pub mod actions; @@ -87,6 +88,14 @@ pub(crate) enum IterationResult { #[derive(Serialize, Debug, Default, Clone)] pub struct TestResult { pub timings: HashMap>, + pub requests_per_second: f64, + pub duration: Option, +} + +#[derive(Serialize, Debug, Clone)] +pub enum TestDurationStatus { + Iteration { until: u64, current: u64 }, + Timed { remaining: Duration }, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -185,6 +194,9 @@ impl TestRunContext { pub(crate) async fn get_test_results(&self) -> TestResult { self.result_builder.build().await } + pub(crate) fn get_duration_status(&self) -> TestDurationStatus { + self.duration_tracker.get_status() + } pub(crate) fn _mark_exit(&self) { self.duration_tracker.mark_exit() @@ -239,6 +251,7 @@ pub(crate) trait TestDurationTracker { fn mark_exit(&self); fn is_done(&self) -> bool; async fn should_track_iteration(&self) -> bool; + fn get_status(&self) -> TestDurationStatus; } fn get_duration_tracker( diff --git a/libfatigue/src/context/result.rs b/libfatigue/src/context/result.rs index 9e9b02f..3d6eb02 100644 --- a/libfatigue/src/context/result.rs +++ b/libfatigue/src/context/result.rs @@ -6,6 +6,7 @@ use reqwest::StatusCode; use std::collections::HashMap; use std::time::Duration; use tokio::sync::Mutex; +use tokio::time::Instant; pub(crate) struct TestResultBuilder { inner: Mutex, @@ -45,30 +46,23 @@ impl TestResultBuilder { } } -struct TestResultLogItem { - _actions: Vec, - _context: IterationContext, -} - struct TestResultBuilderInner { http_timings: HashMap>>, - _timings: HashMap>, - full_log: Vec, + started_at: Instant, } impl TestResultBuilderInner { fn new() -> Self { TestResultBuilderInner { http_timings: HashMap::new(), - _timings: HashMap::new(), - full_log: Vec::with_capacity(512), + started_at: Instant::now(), } } fn mark_success_iteration( &mut self, actions: Vec, - context: IterationContext, + _context: IterationContext, ) { for action in &actions { match &action.internal { @@ -76,12 +70,6 @@ impl TestResultBuilderInner { Err(_) => {} } } - - let log_item = TestResultLogItem { - _actions: actions, - _context: context, - }; - self.full_log.push(log_item); } fn mark_success_action(&mut self, info: &ActionExecutionInfo, action: &InternalActionResult) { @@ -125,11 +113,19 @@ impl TestResultBuilderInner { timings_log.record(duration_as_ns).unwrap(); } - fn build_http_timings(&self) -> HashMap> { + fn build_http_timings( + &self, + ) -> ( + HashMap>, + u64, + ) { let mut res = HashMap::with_capacity(self.http_timings.len()); + let mut call_count = 0; for (name, log) in &self.http_timings { let mut current_group = HashMap::new(); for (status, hist) in log { + let log_item = TestResultTimingLogItem::map_from_histogram(hist); + call_count += log_item.metric_len; current_group.insert( status.to_string(), TestResultTimingLogItem::map_from_histogram(hist), @@ -139,11 +135,17 @@ impl TestResultBuilderInner { res.insert(name.to_string(), current_group); } - res + (res, call_count) } pub fn build(&self) -> TestResult { - let timings = self.build_http_timings(); - TestResult { timings } + let (timings, requests) = self.build_http_timings(); + let time_since_started = Instant::now() - self.started_at; + let requests_per_second = (requests as f64) / time_since_started.as_secs_f64(); + TestResult { + timings, + requests_per_second, + duration: None, + } } } diff --git a/libfatigue/src/runner.rs b/libfatigue/src/runner.rs index d8f2faf..7079070 100644 --- a/libfatigue/src/runner.rs +++ b/libfatigue/src/runner.rs @@ -98,7 +98,9 @@ fn start_test_run_watch_handler( ) -> JoinHandle<()> { spawn(async move { while ctx.is_not_done() { - let results = ctx.get_test_results().await; + let mut results = ctx.get_test_results().await; + let status = ctx.get_duration_status(); + results.duration = Some(status); let send_res = sender.send(results); // todo: probably handle this better?