diff --git a/docs/command-line.md b/docs/command-line.md index 742d84b660..7ec2fbb31d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -24,6 +24,7 @@ [Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)
[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)
[Disable statistical analysis of collected benchmark samples](#disable-statistical-analysis-of-collected-benchmark-samples)
+[Specify the amount of time in milliseconds spent on warming up each test](#specify-the-amount-of-time-in-milliseconds-spent-on-warming-up-each-test)
[Usage](#usage)
[Specify the section to run](#specify-the-section-to-run)
[Filenames as tags](#filenames-as-tags)
@@ -64,6 +65,7 @@ Click one of the following links to take you straight to that option - or scroll ` --benchmark-resamples`
` --benchmark-confidence-interval`
` --benchmark-no-analysis`
+ ` --benchmark-warmup-time`
` --use-colour`

@@ -317,6 +319,14 @@ Must be between 0 and 1 and defaults to 0.95. When this flag is specified no bootstrapping or any other statistical analysis is performed. Instead the user code is only measured and the plain mean from the samples is reported. + +## Specify the amount of time in milliseconds spent on warming up each test +
--benchmark-warmup-time
+ +> [Introduced](https://github.com/catchorg/Catch2/pull/1844) in Catch X.Y.Z. + +Configure the amount of time spent warming up each test. + ## Usage 
-h, -?, --help
diff --git a/docs/configuration.md b/docs/configuration.md index 599bb9ae4a..d09e043f9f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -16,6 +16,7 @@ [Windows header clutter](#windows-header-clutter)
[Enabling stringification](#enabling-stringification)
[Disabling exceptions](#disabling-exceptions)
+[Overriding Catch's debug break (`-b`)](#overriding-catchs-debug-break--b)
Catch is designed to "just work" as much as possible. For most people the only configuration needed is telling Catch which source file should host all the implementation code (```CATCH_CONFIG_MAIN```). @@ -257,6 +258,18 @@ namespace Catch { } ``` +## Overriding Catch's debug break (`-b`) + +> [Introduced](https://github.com/catchorg/Catch2/pull/1846) in Catch X.Y.Z. + +You can override Catch2's break-into-debugger code by defining the +`CATCH_BREAK_INTO_DEBUGGER()` macro. This can be used if e.g. Catch2 does +not know your platform, or your platform is misdetected. + +The macro will be used as is, that is, `CATCH_BREAK_INTO_DEBUGGER();` +must compile and must break into debugger. + + --- [Home](Readme.md#top) diff --git a/include/internal/benchmark/catch_benchmark.hpp b/include/internal/benchmark/catch_benchmark.hpp index d9887eba75..ec8dde0861 100644 --- a/include/internal/benchmark/catch_benchmark.hpp +++ b/include/internal/benchmark/catch_benchmark.hpp @@ -44,10 +44,10 @@ namespace Catch { template ExecutionPlan> prepare(const IConfig &cfg, Environment> env) const { auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; - auto run_time = std::max(min_time, std::chrono::duration_cast(Detail::warmup_time)); + auto run_time = std::max(min_time, std::chrono::duration_cast(cfg.benchmarkWarmupTime())); auto&& test = Detail::run_for_at_least(std::chrono::duration_cast>(run_time), 1, fun); int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); - return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(Detail::warmup_time), Detail::warmup_iterations }; + return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; } template diff --git a/include/internal/catch_commandline.cpp b/include/internal/catch_commandline.cpp index ecb7dc2f3b..b0412d50d8 100644 --- a/include/internal/catch_commandline.cpp +++ b/include/internal/catch_commandline.cpp @@ -213,6 +213,9 @@ namespace Catch { | Opt( config.benchmarkNoAnalysis ) ["--benchmark-no-analysis"] ( "perform only measurements; do not perform any analysis" ) + | Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" ) + ["--benchmark-warmup-time"] + ( "amount of time in milliseconds spent on warming up each test (default: 100)" ) | Arg( config.testsOrTags, "test name|pattern|tags" ) ( "which test or tests to use" ); diff --git a/include/internal/catch_config.cpp b/include/internal/catch_config.cpp index e222328b97..21cd62965b 100644 --- a/include/internal/catch_config.cpp +++ b/include/internal/catch_config.cpp @@ -72,10 +72,11 @@ namespace Catch { bool Config::showInvisibles() const { return m_data.showInvisibles; } Verbosity Config::verbosity() const { return m_data.verbosity; } - bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } - int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } - double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } - unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } + int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } + double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } + unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + std::chrono::milliseconds Config::benchmarkWarmupTime() const { return std::chrono::milliseconds(m_data.benchmarkWarmupTime); } IStream const* Config::openStream() { return Catch::makeStream(m_data.outputFilename); diff --git a/include/internal/catch_config.hpp b/include/internal/catch_config.hpp index 95b67d25fc..64d2c035f9 100644 --- a/include/internal/catch_config.hpp +++ b/include/internal/catch_config.hpp @@ -47,6 +47,7 @@ namespace Catch { unsigned int benchmarkSamples = 100; double benchmarkConfidenceInterval = 0.95; unsigned int benchmarkResamples = 100000; + std::chrono::milliseconds::rep benchmarkWarmupTime = 100; Verbosity verbosity = Verbosity::Normal; WarnAbout::What warnings = WarnAbout::Nothing; @@ -113,6 +114,7 @@ namespace Catch { int benchmarkSamples() const override; double benchmarkConfidenceInterval() const override; unsigned int benchmarkResamples() const override; + std::chrono::milliseconds benchmarkWarmupTime() const override; private: diff --git a/include/internal/catch_debugger.h b/include/internal/catch_debugger.h index fb1850a23b..001b46457b 100644 --- a/include/internal/catch_debugger.h +++ b/include/internal/catch_debugger.h @@ -48,10 +48,12 @@ namespace Catch { #define CATCH_TRAP() DebugBreak() #endif -#ifdef CATCH_TRAP - #define CATCH_BREAK_INTO_DEBUGGER() []{ if( Catch::isDebuggerActive() ) { CATCH_TRAP(); } }() -#else - #define CATCH_BREAK_INTO_DEBUGGER() []{}() +#ifndef CATCH_BREAK_INTO_DEBUGGER + #ifdef CATCH_TRAP + #define CATCH_BREAK_INTO_DEBUGGER() []{ if( Catch::isDebuggerActive() ) { CATCH_TRAP(); } }() + #else + #define CATCH_BREAK_INTO_DEBUGGER() []{}() + #endif #endif #endif // TWOBLUECUBES_CATCH_DEBUGGER_H_INCLUDED diff --git a/include/internal/catch_interfaces_config.h b/include/internal/catch_interfaces_config.h index f8cbf71c86..8fb986be46 100644 --- a/include/internal/catch_interfaces_config.h +++ b/include/internal/catch_interfaces_config.h @@ -11,6 +11,7 @@ #include "catch_common.h" #include "catch_option.hpp" +#include #include #include #include @@ -81,6 +82,7 @@ namespace Catch { virtual int benchmarkSamples() const = 0; virtual double benchmarkConfidenceInterval() const = 0; virtual unsigned int benchmarkResamples() const = 0; + virtual std::chrono::milliseconds benchmarkWarmupTime() const = 0; }; using IConfigPtr = std::shared_ptr; diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index 8bfe5ee6d0..628c6bdf49 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -1116,6 +1116,8 @@ CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-confid CmdLine.tests.cpp:: passed: config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99) for: 0.99 == Approx( 0.99 ) CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} CmdLine.tests.cpp:: passed: config.benchmarkNoAnalysis for: true +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkWarmupTime == 10 for: 10 == 10 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 3 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 2 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 1 >= 1 diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 4132242535..74e576a97d 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -1381,5 +1381,5 @@ due to unexpected exception with message: =============================================================================== test cases: 305 | 231 passed | 70 failed | 4 failed as expected -assertions: 1662 | 1510 passed | 131 failed | 21 failed as expected +assertions: 1664 | 1512 passed | 131 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 1cc7893680..25a9232087 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -8046,7 +8046,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + confidence-interval ------------------------------------------------------------------------------- CmdLine.tests.cpp: ............................................................................... @@ -8064,7 +8064,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + no-analysis ------------------------------------------------------------------------------- CmdLine.tests.cpp: ............................................................................... @@ -8079,6 +8079,24 @@ CmdLine.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + warmup-time +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkWarmupTime == 10 ) +with expansion: + 10 == 10 + ------------------------------------------------------------------------------- Product with differing arities - std::tuple ------------------------------------------------------------------------------- @@ -13285,5 +13303,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 305 | 215 passed | 86 failed | 4 failed as expected -assertions: 1679 | 1510 passed | 148 failed | 21 failed as expected +assertions: 1681 | 1512 passed | 148 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index d32809ff1d..e0b7c1b8ef 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1015,8 +1015,9 @@ Message.tests.cpp: - - + + + diff --git a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt index 47dabc2b25..d6f6b0905d 100644 --- a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -64,8 +64,9 @@ - - + + + diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 55c160a269..6d06368e34 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -10140,7 +10140,7 @@ Nor would this
-
+
cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) @@ -10162,7 +10162,7 @@ Nor would this
-
+
cli.parse({ "test", "--benchmark-no-analysis" }) @@ -10183,6 +10183,28 @@ Nor would this
+
+
+ + + cli.parse({ "test", "--benchmark-warmup-time=10" }) + + + {?} + + + + + config.benchmarkWarmupTime == 10 + + + 10 == 10 + + + +
+ +
@@ -15871,7 +15893,7 @@ loose text artifact
- + - + diff --git a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index c1fd90e3a9..641f1b2941 100644 --- a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -503,17 +503,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" REQUIRE(config.benchmarkResamples == 20000); } - SECTION("resamples") { + SECTION("confidence-interval") { CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" })); REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99)); } - SECTION("resamples") { + SECTION("no-analysis") { CHECK(cli.parse({ "test", "--benchmark-no-analysis" })); REQUIRE(config.benchmarkNoAnalysis); } + + SECTION("warmup-time") { + CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" })); + + REQUIRE(config.benchmarkWarmupTime == 10); + } } }