Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tests: Avoid dialog boxes, prevent stdout from being lost #906

Merged
merged 5 commits into from
Jun 20, 2020
Merged

tests: Avoid dialog boxes, prevent stdout from being lost #906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d12e3c5
Print randomized test failures to stderr.
StephanTLavavej Jun 17, 2020
61af9d0
Change wcout to wcerr in P0218R1_filesystem.
StephanTLavavej Jun 17, 2020
ebe33b8
Fix GH-781 (dialogs) and fix GH-789 (stdout).
StephanTLavavej Jun 17, 2020
36df0bb
Update include_each_header_alone_matrix.lst.
StephanTLavavej Jun 18, 2020
2b4a28c
NO_TEST_ENVIRONMENT_PREPARER escape hatch.
StephanTLavavej Jun 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions tests/std/include/force_include.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#pragma once

#ifndef NO_TEST_ENVIRONMENT_PREPARER

#include <crtdbg.h>
#include <stdio.h>
#include <stdlib.h>

struct TestEnvironmentPreparer {
TestEnvironmentPreparer() noexcept {
// avoid assertion dialog boxes; see GH-781
_CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE);
_CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
_CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
_CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
_set_abort_behavior(0, _CALL_REPORTFAULT);

// set stdout to be unbuffered; see GH-789
setvbuf(stdout, nullptr, _IONBF, 0);
}
};

const TestEnvironmentPreparer test_environment_preparer{};

#endif // NO_TEST_ENVIRONMENT_PREPARER
8 changes: 4 additions & 4 deletions tests/std/tests/P0024R2_parallel_algorithms_set_difference/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ using namespace std::execution;
template <class T>
void assert_message_vector(const bool b, const char* const msg, const T seedValue) {
if (!b) {
cout << msg << " failed for seed value: " << seedValue << "\n";
cout << "This is a randomized test.\n";
cout << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cout << "You must report it to the STL maintainers.\n";
cerr << msg << " failed for seed value: " << seedValue << "\n";
cerr << "This is a randomized test.\n";
cerr << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cerr << "You must report it to the STL maintainers.\n";
abort();
}
}
Expand Down
8 changes: 4 additions & 4 deletions tests/std/tests/P0024R2_parallel_algorithms_set_intersection/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ using namespace std::execution;
template <class T>
void assert_message_vector(const bool b, const char* const msg, const T seedValue) {
if (!b) {
cout << msg << " failed for seed value: " << seedValue << "\n";
cout << "This is a randomized test.\n";
cout << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cout << "You must report it to the STL maintainers.\n";
cerr << msg << " failed for seed value: " << seedValue << "\n";
cerr << "This is a randomized test.\n";
cerr << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cerr << "You must report it to the STL maintainers.\n";
abort();
}
}
Expand Down
16 changes: 8 additions & 8 deletions tests/std/tests/P0067R5_charconv/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,20 +564,20 @@ void all_integer_tests() {

void assert_message_bits(const bool b, const char* const msg, const uint32_t bits) {
if (!b) {
printf("%s failed for 0x%08X\n", msg, bits);
puts("This is a randomized test.");
puts("DO NOT IGNORE/RERUN THIS FAILURE.");
puts("You must report it to the STL maintainers.");
fprintf(stderr, "%s failed for 0x%08X\n", msg, bits);
fprintf(stderr, "This is a randomized test.\n");
fprintf(stderr, "DO NOT IGNORE/RERUN THIS FAILURE.\n");
fprintf(stderr, "You must report it to the STL maintainers.\n");
abort();
}
}

void assert_message_bits(const bool b, const char* const msg, const uint64_t bits) {
if (!b) {
printf("%s failed for 0x%016llX\n", msg, bits);
puts("This is a randomized test.");
puts("DO NOT IGNORE/RERUN THIS FAILURE.");
puts("You must report it to the STL maintainers.");
fprintf(stderr, "%s failed for 0x%016llX\n", msg, bits);
fprintf(stderr, "This is a randomized test.\n");
fprintf(stderr, "DO NOT IGNORE/RERUN THIS FAILURE.\n");
fprintf(stderr, "You must report it to the STL maintainers.\n");
abort();
}
}
Expand Down
78 changes: 39 additions & 39 deletions tests/std/tests/P0218R1_filesystem/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,19 +58,19 @@ struct test_temp_directory {
explicit test_temp_directory(const string_view testName) : directoryPath(get_new_test_directory(testName)) {
remove_all(directoryPath, ec);
if (ec) {
wcout << L"Warning, couldn't clean up " << directoryPath << L" before test.\n";
wcerr << L"Warning, couldn't clean up " << directoryPath << L" before test.\n";
} else {
create_directories(directoryPath, ec);
if (ec) {
wcout << L"Warning, couldn't create test directory " << directoryPath << L" before test.\n";
wcerr << L"Warning, couldn't create test directory " << directoryPath << L" before test.\n";
}
}
}

~test_temp_directory() noexcept {
remove_all(directoryPath, ec);
if (ec) {
wcout << L"Warning, couldn't clean up " << directoryPath << L" after test.\n";
wcerr << L"Warning, couldn't clean up " << directoryPath << L" after test.\n";
}
}
};
Expand All @@ -79,7 +79,7 @@ bool pass = true;

bool expect(const bool b, const char* const func, const int line, const char* const message) {
if (!b) {
wcout << func << L" @ " << line << L": check failed: " << message << L'\n';
wcerr << func << L" @ " << line << L": check failed: " << message << L'\n';
pass = false;
}
return b;
Expand All @@ -90,12 +90,12 @@ bool expect(const bool b, const char* const func, const int line, const char* co
bool good(const error_code& ec) {
bool overall = true;
if (ec.value() != 0) {
wcout << L"Unexpected error " << ec.value() << L" " << ec.message().c_str() << L"\n";
wcerr << L"Unexpected error " << ec.value() << L" " << ec.message().c_str() << L"\n";
overall = false;
}

if (ec.category() != system_category()) {
wcout << L"Unexpected category " << ec.category().name() << L"\n";
wcerr << L"Unexpected category " << ec.category().name() << L"\n";
overall = false;
}

Expand Down Expand Up @@ -362,7 +362,7 @@ bool run_decomp_test_case(const decomposition_test_case& testCase) {
return true;
}

wcout << L"Test failure:\n" << testCase << actual;
wcerr << L"Test failure:\n" << testCase << actual;

return false;
}
Expand Down Expand Up @@ -406,7 +406,7 @@ bool run_stem_test_case(const stem_test_case& testCase) {
return true;
}

wcout << L"Expected " << p.native() << L" to have stem() " << testCase.stem << L" and extension() "
wcerr << L"Expected " << p.native() << L" to have stem() " << testCase.stem << L" and extension() "
<< testCase.extension << L", but it actually has stem() " << p.stem().native() << L" and extension() "
<< p.extension().native() << L"\n";

Expand Down Expand Up @@ -523,7 +523,7 @@ bool run_compare_test_case(const compare_test_case& testCase) {
return true;
}

wcout << L"Unexpected comparison result:\nLeft: " << testCase.left << L"\nRight: " << testCase.right
wcerr << L"Unexpected comparison result:\nLeft: " << testCase.left << L"\nRight: " << testCase.right
<< L"\nExpected: " << testCase.expected << L"\n Actual: " << actual << L"\n";

return false;
Expand Down Expand Up @@ -564,7 +564,7 @@ bool run_slash_test_case(const slash_test_case& testCase) {
return true;
}

wcout << L"Expected " << testCase.a << L" / " << testCase.b << L" to be " << testCase.expected << L" but it was "
wcerr << L"Expected " << testCase.a << L" / " << testCase.b << L" to be " << testCase.expected << L" but it was "
<< p.native() << L"\n";
return false;
}
Expand Down Expand Up @@ -811,7 +811,7 @@ void test_remove_filename_and_sep() {
path p{before};
p._Remove_filename_and_separator();
if (p.native() != after) { // look for exact match
wcout << L"_Remove_filename_and_separator('" << before << L"') => '" << p.native() << L"' expected '"
wcerr << L"_Remove_filename_and_separator('" << before << L"') => '" << p.native() << L"' expected '"
<< after << L"'\n";
pass = false;
}
Expand Down Expand Up @@ -879,7 +879,7 @@ void test_filesystem_error() {
void test_file_status() {
auto check = [](const file_status& x, file_type ft, perms p, const char* name) {
if (x.type() != ft || x.permissions() != p) {
wcout << L"test_file_status failed: " << name << L'\n';
wcerr << L"test_file_status failed: " << name << L'\n';
pass = false;
}
};
Expand Down Expand Up @@ -947,25 +947,25 @@ void test_file_status() {

void check_symlink_permissions(const error_code& ec, const wchar_t* const function_id) {
if (ec.category() != system_category()) {
wcout << L"Incorrect error category from " << function_id << L"\n";
wcerr << L"Incorrect error category from " << function_id << L"\n";
pass = false;
}

#ifdef WINDOWS_XP
if (ec.value() != 50) {
wcout << L"Expected ERROR_NOT_SUPPORTED from " << function_id << L" but it returned " << ec.message().c_str()
wcerr << L"Expected ERROR_NOT_SUPPORTED from " << function_id << L" but it returned " << ec.message().c_str()
<< L"\n";
pass = false;
}
#else // ^^^ WINDOWS_XP ^^^ // vvv !WINDOWS_XP vvv
if (ec.value() != 1314) {
wcout << L"Expected ERROR_PRIVILEGE_NOT_HELD from " << function_id << L" but it returned "
wcerr << L"Expected ERROR_PRIVILEGE_NOT_HELD from " << function_id << L" but it returned "
<< ec.message().c_str() << L"\n";
pass = false;
}
#endif // WINDOWS_XP

wcout << L"Warning: could not test " << function_id
wcerr << L"Warning: could not test " << function_id
<< L" due to symlink creation failure, do you have admin rights?\n";
}

Expand Down Expand Up @@ -1517,12 +1517,12 @@ void expect_absolute(const path& input, const wstring_view expected) {
error_code ec(-1, generic_category());
const path actual = absolute(input, ec);
if (actual.native() != expected || ec || ec.category() != system_category()) {
wcout << L"Expected absolute(" << input.native() << L") to be " << expected << L"\n";
wcerr << L"Expected absolute(" << input.native() << L") to be " << expected << L"\n";
if (actual.native() != expected) {
wcout << L"Actual result: " << actual.native() << L"\n";
wcerr << L"Actual result: " << actual.native() << L"\n";
}
if (ec) {
wcout << L"The call failed.\n";
wcerr << L"The call failed.\n";
}

pass = false;
Expand All @@ -1543,7 +1543,7 @@ void test_absolute() {
EXPECT(throws_filesystem_error([&] { return absolute(longPath); }, "absolute"sv, longPath));
EXPECT(absolute(longPath, ec).empty());
if (ec.value() != 206) {
wcout << L"Warning: Expected absolute on a >32k long path to report ERROR_FILENAME_EXCED_RANGE, "
wcerr << L"Warning: Expected absolute on a >32k long path to report ERROR_FILENAME_EXCED_RANGE, "
L"but it reported "
<< ec.value() << L"\n";
}
Expand Down Expand Up @@ -2266,11 +2266,11 @@ void equivalent_failure_test_case(const path& left, const path& right) {

error_code ec;
if (equivalent(left, right, ec)) {
wcout << L"Expected equivalent(" << left << L", " << right << L") to fail but it returned true\n";
wcerr << L"Expected equivalent(" << left << L", " << right << L") to fail but it returned true\n";
pass = false;
} else if (!ec) {
EXPECT(ec.category() == system_category());
wcout << L"Expected equivalent(" << left << L", " << right
wcerr << L"Expected equivalent(" << left << L", " << right
<< L") to fail but it "
L"returned no failure code\n";
pass = false;
Expand All @@ -2282,7 +2282,7 @@ void equivalent_test_case(const path& left, const path& right, const bool expect
const bool actual = equivalent(left, right, ec);
EXPECT(good(ec));
if (expected != actual) {
wcout << boolalpha << L"Expected equivalent(" << left << L", " << right << L") to be " << expected
wcerr << boolalpha << L"Expected equivalent(" << left << L", " << right << L") to be " << expected
<< L" but it was " << actual << L"\n";
pass = false;
}
Expand Down Expand Up @@ -3789,15 +3789,15 @@ void test_file_time_type() {

if (file_time_tick_count + tolerance < system_clock_tick_count
|| file_time_tick_count - tolerance > system_clock_tick_count) {
wcout << L"test_file_time_type failed: " << file_time_tick_count << L" ticks too different from "
wcerr << L"test_file_time_type failed: " << file_time_tick_count << L" ticks too different from "
<< system_clock_tick_count << L" ticks from system_clock\n";
pass = false;
}
}

template <typename DirIter>
void interactive_dir_iter(wstring_view p) {
wcout << L"iterate over: \"" << p << L"\":\n";
wcerr << L"iterate over: \"" << p << L"\":\n";

directory_options opts = {};
// ! => skip_permission_denied
Expand All @@ -3816,9 +3816,9 @@ void interactive_dir_iter(wstring_view p) {
}

for (const auto& entry : DirIter(p, opts)) {
wcout << entry.path().native() << L'\n';
wcerr << entry.path().native() << L'\n';
}
wcout << L"---- iteration complete -----\n";
wcerr << L"---- iteration complete -----\n";
}

template <typename Elem, typename Traits>
Expand Down Expand Up @@ -3874,32 +3874,32 @@ void run_interactive_tests(int argc, wchar_t* argv[]) {
for (int i = 1; i < argc; ++i) {
const wstring_view arg = argv[i];
if (arg == L"-?"sv) {
wcout << usage;
wcerr << usage;
} else if (starts_with(arg, L"-recdir:"sv)) {
interactive_dir_iter<recursive_directory_iterator>(the_rest);
} else if (starts_with(arg, L"-dir:"sv)) {
interactive_dir_iter<directory_iterator>(the_rest);
} else if (starts_with(arg, L"-lstat:"sv)) {
wcout << quoted(arg) << L" => " << symlink_status(the_rest) << "\n";
wcerr << quoted(arg) << L" => " << symlink_status(the_rest) << "\n";
} else if (starts_with(arg, L"-stat:"sv)) {
wcout << quoted(arg) << L" => " << status(the_rest) << "\n";
wcerr << quoted(arg) << L" => " << status(the_rest) << "\n";
} else if (starts_with(arg, L"-de:"sv)) {
wcout << quoted(arg) << L" => " << directory_entry(the_rest) << "\n";
wcerr << quoted(arg) << L" => " << directory_entry(the_rest) << "\n";
} else if (starts_with(arg, L"-mkdir:"sv)) {
wcout << L"create_directory => " << create_directory(the_rest) << "\n";
wcerr << L"create_directory => " << create_directory(the_rest) << "\n";
} else if (starts_with(arg, L"-mkdirs:"sv)) {
wcout << L"create_directory => " << create_directories(the_rest) << "\n";
wcerr << L"create_directory => " << create_directories(the_rest) << "\n";
} else if (starts_with(arg, L"-now"sv)) {
wcout << L" system_clock: " << system_clock::now().time_since_epoch().count() << L'\n';
wcout << L"file_time_type::clock: " << file_time_type::clock::now().time_since_epoch().count() << L'\n';
wcerr << L" system_clock: " << system_clock::now().time_since_epoch().count() << L'\n';
wcerr << L"file_time_type::clock: " << file_time_type::clock::now().time_since_epoch().count() << L'\n';
} else if (starts_with(arg, L"-rm:"sv)) {
wcout << L"remove => " << remove(the_rest) << "\n";
wcerr << L"remove => " << remove(the_rest) << "\n";
} else if (starts_with(arg, L"-rmall:"sv)) {
wcout << L"remove_all => " << remove_all(the_rest) << "\n";
wcerr << L"remove_all => " << remove_all(the_rest) << "\n";
} else if (starts_with(arg, L"-sz:"sv)) {
wcout << L"file_size => " << file_size(the_rest) << "\n";
wcerr << L"file_size => " << file_size(the_rest) << "\n";
} else {
wcout << usage;
wcerr << usage;
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions tests/std/tests/P0220R1_searchers/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,10 @@ void test_case_BM_unicode32() {
}

void report_randomized_failure(const string_view searcher, const string_view needle, const string_view haystack) {
cout << searcher << " failed for needle \"" << needle << "\" and haystack \"" << haystack << "\".\n";
cout << "This is a randomized test.\n";
cout << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cout << "You must report it to the STL maintainers.\n";
cerr << searcher << " failed for needle \"" << needle << "\" and haystack \"" << haystack << "\".\n";
cerr << "This is a randomized test.\n";
cerr << "DO NOT IGNORE/RERUN THIS FAILURE.\n";
cerr << "You must report it to the STL maintainers.\n";
assert(false);
}

Expand Down
Loading