Commit 5d233fae authored by Dominik Charousset's avatar Dominik Charousset

Fix and improve some issues with the testing DSL

parent 4bbbb5a2
...@@ -5,10 +5,34 @@ is based on [Keep a Changelog](https://keepachangelog.com). ...@@ -5,10 +5,34 @@ is based on [Keep a Changelog](https://keepachangelog.com).
## [Unreleased] ## [Unreleased]
### Added
- When adding CAF with exceptions enabled (default), the unit test framework now
offers new check macros:
- `CAF_CHECK_NOTHROW(expr)`
- `CAF_CHECK_THROWS_AS(expr, type)`
- `CAF_CHECK_THROWS_WITH(expr, str)`
- `CAF_CHECK_THROWS_WITH_AS(expr, str, type)`
### Fixed
- The DSL function `run_until` miscounted the number executed events, also
causing `run_once` to report a wrong value. Both functions now return the
correct result.
- Using `allow(...).with(...)` in unit tests without a matching message crashed
the program. By adding a missing NULL-check, `allow` is now always safe to
use.
### Changed ### Changed
- Since support of Qt 5 expired, we have ported the Qt examples to version 6. - Since support of Qt 5 expired, we have ported the Qt examples to version 6.
Hence, building the Qt examples now requires Qt in version 6. Hence, building the Qt examples now requires Qt in version 6.
- When compiling CAF with exceptions enabled (default), `REQUIRE*` macros,
`expect` and `disallow` no longer call `abort()`. Instead, they throw an
exception that only stops the current test instead of stopping the entire test
program.
- Reporting of several unit test macros has been improved to show correct line
numbers and provide better diagnostic of test errors.
## [0.18.5] - 2021-07-16 ## [0.18.5] - 2021-07-16
......
...@@ -270,6 +270,7 @@ caf_add_component( ...@@ -270,6 +270,7 @@ caf_add_component(
detail.unique_function detail.unique_function
detail.unordered_flat_map detail.unordered_flat_map
dictionary dictionary
dsl
dynamic_spawn dynamic_spawn
error error
expected expected
......
...@@ -46,14 +46,18 @@ public: ...@@ -46,14 +46,18 @@ public:
} }
/// Peeks into the mailbox of `next_job<scheduled_actor>()`. /// Peeks into the mailbox of `next_job<scheduled_actor>()`.
template <class T> template <class... Ts>
const T& peek() { decltype(auto) peek() {
auto ptr = next_job<scheduled_actor>().mailbox().peek(); auto ptr = next_job<scheduled_actor>().mailbox().peek();
CAF_ASSERT(ptr != nullptr); CAF_ASSERT(ptr != nullptr);
auto view = make_typed_message_view<T>(ptr->content()); if (auto view = make_const_typed_message_view<Ts...>(ptr->payload)) {
if (!view) if constexpr (sizeof...(Ts) == 1)
CAF_RAISE_ERROR("Mailbox element does not match T."); return get<0>(view);
return get<0>(view); else
return to_tuple(view);
} else {
CAF_RAISE_ERROR("Mailbox element does not match.");
}
} }
/// Puts `x` at the front of the queue unless it cannot be found in the queue. /// Puts `x` at the front of the queue unless it cannot be found in the queue.
......
This diff is collapsed.
...@@ -37,6 +37,16 @@ ...@@ -37,6 +37,16 @@
#define CHECK_GT(lhs, rhs) CAF_CHECK_GREATER(lhs, rhs) #define CHECK_GT(lhs, rhs) CAF_CHECK_GREATER(lhs, rhs)
#define CHECK_GE(lhs, rhs) CAF_CHECK_GREATER_OR_EQUAL(lhs, rhs) #define CHECK_GE(lhs, rhs) CAF_CHECK_GREATER_OR_EQUAL(lhs, rhs)
#ifdef CAF_ENABLE_EXCEPTIONS
# define CHECK_NOTHROW(expr) CAF_CHECK_NOTHROW(expr)
# define CHECK_THROWS_AS(expr, type) CAF_CHECK_THROWS_AS(expr, type)
# define CHECK_THROWS_WITH(expr, msg) CAF_CHECK_THROWS_WITH(expr, msg)
# define CHECK_THROWS_WITH_AS(expr, msg, type) \
CAF_CHECK_THROWS_WITH_AS(expr, msg, type)
#endif // CAF_ENABLE_EXCEPTIONS
#define REQUIRE(what) CAF_REQUIRE(what) #define REQUIRE(what) CAF_REQUIRE(what)
#define REQUIRE_EQ(lhs, rhs) CAF_REQUIRE_EQUAL(lhs, rhs) #define REQUIRE_EQ(lhs, rhs) CAF_REQUIRE_EQUAL(lhs, rhs)
#define REQUIRE_NE(lhs, rhs) CAF_REQUIRE_NOT_EQUAL(lhs, rhs) #define REQUIRE_NE(lhs, rhs) CAF_REQUIRE_NOT_EQUAL(lhs, rhs)
......
This diff is collapsed.
...@@ -55,9 +55,9 @@ public: ...@@ -55,9 +55,9 @@ public:
/// @param fun A function object for delegating to the parent's `exec_all`. /// @param fun A function object for delegating to the parent's `exec_all`.
test_node_fixture(run_all_nodes_fun fun) test_node_fixture(run_all_nodes_fun fun)
: mm(this->sys.middleman()), : mm(this->sys.middleman()),
mpx(dynamic_cast<caf::io::network::test_multiplexer&>(mm.backend())), mpx(dynamic_cast<caf::io::network::test_multiplexer&>(mm.backend())),
run_all_nodes(std::move(fun)) { run_all_nodes(std::move(fun)) {
bb = mm.named_broker<caf::io::basp_broker>("BASP"); bb = mm.named_broker<caf::io::basp_broker>("BASP");
} }
...@@ -124,9 +124,7 @@ void exec_all_fixtures(Iterator first, Iterator last) { ...@@ -124,9 +124,7 @@ void exec_all_fixtures(Iterator first, Iterator last) {
return x->sched.try_run_once() || x->mpx.read_data() return x->sched.try_run_once() || x->mpx.read_data()
|| x->mpx.try_exec_runnable() || x->mpx.try_accept_connection(); || x->mpx.try_exec_runnable() || x->mpx.try_accept_connection();
}; };
auto trigger_timeouts = [](fixture_ptr x) { auto trigger_timeouts = [](fixture_ptr x) { x->sched.trigger_timeouts(); };
x->sched.trigger_timeouts();
};
for (;;) { for (;;) {
// Exhaust all messages in the system. // Exhaust all messages in the system.
while (std::any_of(first, last, advance)) while (std::any_of(first, last, advance))
...@@ -167,9 +165,8 @@ public: ...@@ -167,9 +165,8 @@ public:
/// (calls `publish`). /// (calls `publish`).
/// @returns randomly picked connection handles for the server and the client. /// @returns randomly picked connection handles for the server and the client.
std::pair<connection_handle, connection_handle> std::pair<connection_handle, connection_handle>
prepare_connection(PlanetType& server, PlanetType& client, prepare_connection(PlanetType& server, PlanetType& client, std::string host,
std::string host, uint16_t port, uint16_t port, accept_handle server_accept_hdl) {
accept_handle server_accept_hdl) {
auto server_hdl = next_connection_handle(); auto server_hdl = next_connection_handle();
auto client_hdl = next_connection_handle(); auto client_hdl = next_connection_handle();
server.mpx.prepare_connection(server_accept_hdl, server_hdl, client.mpx, server.mpx.prepare_connection(server_accept_hdl, server_hdl, client.mpx,
...@@ -181,8 +178,8 @@ public: ...@@ -181,8 +178,8 @@ public:
/// (calls `publish`). /// (calls `publish`).
/// @returns randomly picked connection handles for the server and the client. /// @returns randomly picked connection handles for the server and the client.
std::pair<connection_handle, connection_handle> std::pair<connection_handle, connection_handle>
prepare_connection(PlanetType& server, PlanetType& client, prepare_connection(PlanetType& server, PlanetType& client, std::string host,
std::string host, uint16_t port) { uint16_t port) {
return prepare_connection(server, client, std::move(host), port, return prepare_connection(server, client, std::move(host), port,
next_accept_handle()); next_accept_handle());
} }
...@@ -206,7 +203,7 @@ public: ...@@ -206,7 +203,7 @@ public:
} }
/// Type-erased callback for calling `exec_all`. /// Type-erased callback for calling `exec_all`.
std::function<void ()> exec_all_callback() { std::function<void()> exec_all_callback() {
return [&] { exec_all(); }; return [&] { exec_all(); };
} }
...@@ -273,13 +270,9 @@ public: ...@@ -273,13 +270,9 @@ public:
}; };
#define expect_on(where, types, fields) \ #define expect_on(where, types, fields) \
do { \ (expect_clause<CAF_EXPAND(CAF_DSL_LIST types)>{where.sched, __LINE__} \
CAF_MESSAGE(#where << ": expect" << #types << "." << #fields); \ .fields.eval(#types, #fields))
expect_clause<CAF_EXPAND(CAF_DSL_LIST types)>{where.sched}.fields; \
} while (false)
#define disallow_on(where, types, fields) \ #define disallow_on(where, types, fields) \
do { \ (disallow_clause<CAF_EXPAND(CAF_DSL_LIST types)>{where.sched, __LINE__} \
CAF_MESSAGE(#where << ": disallow" << #types << "." << #fields); \ .fields.eval(#types, #fields))
disallow_clause<CAF_EXPAND(CAF_DSL_LIST types)>{where.sched}.fields; \
} while (false)
This diff is collapsed.
...@@ -29,6 +29,14 @@ ...@@ -29,6 +29,14 @@
namespace caf::test { namespace caf::test {
requirement_error::requirement_error(std::string msg) : what_(std::move(msg)) {
// nop
}
const char* requirement_error::what() const noexcept {
return what_.c_str();
}
class watchdog { class watchdog {
public: public:
static void start(int secs); static void start(int secs);
...@@ -121,6 +129,7 @@ namespace detail { ...@@ -121,6 +129,7 @@ namespace detail {
<< term::yellow << ":" << term::cyan << engine::last_check_line() << term::yellow << ":" << term::cyan << engine::last_check_line()
<< term::reset << detail::fill(engine::last_check_line()) << term::reset << detail::fill(engine::last_check_line())
<< "had last successful check" << '\n'; << "had last successful check" << '\n';
CAF_RAISE_ERROR(requirement_error, msg.c_str());
abort(); abort();
} }
...@@ -200,6 +209,59 @@ bool check_bin(bool result, const char* file, size_t line, const char* expr, ...@@ -200,6 +209,59 @@ bool check_bin(bool result, const char* file, size_t line, const char* expr,
return result; return result;
} }
void require_un(bool result, const char* file, size_t line, const char* expr) {
string_view rel_up = "../";
while (strncmp(file, rel_up.data(), rel_up.size()) == 0)
file += rel_up.size();
auto parent = engine::current_test();
auto out = logger::instance().massive();
if (result) {
out << term::green << "** " << term::blue << file << term::yellow << ":"
<< term::blue << line << fill(line) << term::reset << "passed" << '\n';
parent->pass();
engine::last_check_file(file);
engine::last_check_line(line);
} else {
out << term::red << "!! " << term::blue << file << term::yellow << ":"
<< term::blue << line << fill(line) << term::red
<< "requirement failed: " << expr << term::reset << '\n';
parent->fail(false);
std::string msg = "requirement failed in ";
msg += file;
msg += " line ";
msg += std::to_string(line);
requirement_failed(std::move(msg));
}
}
void require_bin(bool result, const char* file, size_t line, const char* expr,
const std::string& lhs, const std::string& rhs) {
string_view rel_up = "../";
while (strncmp(file, rel_up.data(), rel_up.size()) == 0)
file += rel_up.size();
auto parent = engine::current_test();
auto out = logger::instance().massive();
if (result) {
out << term::green << "** " << term::blue << file << term::yellow << ":"
<< term::blue << line << fill(line) << term::reset << "passed" << '\n';
parent->pass();
engine::last_check_file(file);
engine::last_check_line(line);
} else {
out << term::red << "!! " << term::blue << file << term::yellow << ":"
<< term::blue << line << fill(line) << term::red
<< "requirement failed: " << expr << term::reset << '\n'
<< " lhs: " << lhs << '\n'
<< " rhs: " << rhs << '\n';
parent->fail(false);
std::string msg = "requirement failed in ";
msg += file;
msg += " line ";
msg += std::to_string(line);
requirement_failed(std::move(msg));
}
}
} // namespace detail } // namespace detail
logger::stream::stream(logger& parent, logger::level lvl) logger::stream::stream(logger& parent, logger::level lvl)
...@@ -319,36 +381,6 @@ bool engine::run(bool colorize, const std::string& log_file, ...@@ -319,36 +381,6 @@ bool engine::run(bool colorize, const std::string& log_file,
size_t total_bad = 0; size_t total_bad = 0;
size_t total_bad_expected = 0; size_t total_bad_expected = 0;
auto bar = '+' + std::string(70, '-') + '+'; auto bar = '+' + std::string(70, '-') + '+';
#if (!defined(__clang__) && defined(__GNUC__) && __GNUC__ == 4 \
&& __GNUC_MINOR__ < 9) \
|| (defined(__clang__) && !defined(_LIBCPP_VERSION))
// regex implementation is broken prior to 4.9
using strvec = std::vector<std::string>;
using whitelist_type = strvec;
using blacklist_type = strvec;
auto from_psv = [](const std::string& psv) -> strvec {
// psv == pipe-separated-values
strvec result;
if (psv != ".*") {
split(result, psv, "|", token_compress_on);
std::sort(result.begin(), result.end());
}
return result;
};
auto suites = from_psv(suites_str);
auto not_suites = from_psv(not_suites_str);
auto tests = from_psv(tests_str);
auto not_tests = from_psv(not_tests_str);
auto enabled = [](const strvec& whitelist, const strvec& blacklist,
const std::string& x) {
// an empty whitelist means original input was ".*", i.e., enable all
return !std::binary_search(blacklist.begin(), blacklist.end(), x)
&& (whitelist.empty()
|| std::binary_search(whitelist.begin(), whitelist.end(), x));
};
#else
using whitelist_type = std::regex;
using blacklist_type = std::regex;
std::regex suites{suites_str}; std::regex suites{suites_str};
std::regex tests{tests_str}; std::regex tests{tests_str};
std::regex not_suites; std::regex not_suites;
...@@ -358,18 +390,16 @@ bool engine::run(bool colorize, const std::string& log_file, ...@@ -358,18 +390,16 @@ bool engine::run(bool colorize, const std::string& log_file,
not_suites.assign(not_suites_str); not_suites.assign(not_suites_str);
if (!not_tests_str.empty()) if (!not_tests_str.empty())
not_tests.assign(not_tests_str); not_tests.assign(not_tests_str);
auto enabled = [](const std::regex& whitelist, const std::regex& blacklist, auto enabled = [](const std::regex& selected, const std::regex& blocked,
const std::string& x) { const std::string& x) {
// an empty whitelist means original input was "*", i.e., enable all return std::regex_search(x, selected) && !std::regex_search(x, blocked);
return std::regex_search(x, whitelist) && !std::regex_search(x, blacklist);
}; };
#endif auto test_enabled = [&](const std::regex& selected, const std::regex& blocked,
auto test_enabled = [&](const whitelist_type& whitelist, const test& x) {
const blacklist_type& blacklist, const test& x) {
// Disabled tests run if explicitly requested by the user, i.e., // Disabled tests run if explicitly requested by the user, i.e.,
// tests_str is not the ".*" catch-all default. // tests_str is not the ".*" catch-all default.
return (!x.disabled() || tests_str != ".*") return (!x.disabled() || tests_str != ".*")
&& enabled(whitelist, blacklist, x.name()); && enabled(selected, blocked, x.name());
}; };
std::vector<std::string> failed_tests; std::vector<std::string> failed_tests;
for (auto& p : instance().suites_) { for (auto& p : instance().suites_) {
...@@ -393,7 +423,23 @@ bool engine::run(bool colorize, const std::string& log_file, ...@@ -393,7 +423,23 @@ bool engine::run(bool colorize, const std::string& log_file,
log.verbose() << term::yellow << "- " << term::reset << t->name() << '\n'; log.verbose() << term::yellow << "- " << term::reset << t->name() << '\n';
auto start = std::chrono::high_resolution_clock::now(); auto start = std::chrono::high_resolution_clock::now();
watchdog::start(max_runtime()); watchdog::start(max_runtime());
#ifdef CAF_ENABLE_EXCEPTIONS
try {
t->run_test_impl();
} catch (requirement_error&) {
// nop
} catch (std::exception& ex) {
t->fail(false);
log.error() << term::red << "!! uncaught exception, what: " << ex.what()
<< term::reset_endl;
} catch (...) {
t->fail(false);
log.error() << term::red << "!! uncaught exception of unknown type"
<< term::reset_endl;
}
#else
t->run_test_impl(); t->run_test_impl();
#endif // CAF_ENABLE_EXCEPTIONS
watchdog::stop(); watchdog::stop();
auto stop = std::chrono::high_resolution_clock::now(); auto stop = std::chrono::high_resolution_clock::now();
auto elapsed auto elapsed
...@@ -515,8 +561,8 @@ std::string engine::render(std::chrono::microseconds t) { ...@@ -515,8 +561,8 @@ std::string engine::render(std::chrono::microseconds t) {
return t.count() > 1000000 return t.count() > 1000000
? (std::to_string(t.count() / 1000000) + '.' ? (std::to_string(t.count() / 1000000) + '.'
+ std::to_string((t.count() % 1000000) / 10000) + " s") + std::to_string((t.count() % 1000000) / 10000) + " s")
: t.count() > 1000 ? (std::to_string(t.count() / 1000) + " ms") : t.count() > 1000 ? (std::to_string(t.count() / 1000) + " ms")
: (std::to_string(t.count()) + " us"); : (std::to_string(t.count()) + " us");
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment