Commit b78cb1a3 authored by Dominik Charousset's avatar Dominik Charousset

Wait for BASP workers on MM shutdown

parent 6ac92c6b
...@@ -205,6 +205,10 @@ public: ...@@ -205,6 +205,10 @@ public:
return this_node_; return this_node_;
} }
worker_hub& hub() {
return hub_;
}
actor_system& system() { actor_system& system() {
return callee_.proxies().system(); return callee_.proxies().system();
} }
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
#pragma once #pragma once
#include <atomic> #include <atomic>
#include <mutex>
#include "caf/fwd.hpp" #include "caf/fwd.hpp"
#include "caf/io/basp/fwd.hpp" #include "caf/io/basp/fwd.hpp"
...@@ -47,7 +48,7 @@ public: ...@@ -47,7 +48,7 @@ public:
// -- properties ------------------------------------------------------------- // -- properties -------------------------------------------------------------
/// Creates a new worker and adds it to the hub. /// Creates a new worker and adds it to the hub.
void push_new_worker(message_queue&, proxy_registry&); void add_new_worker(message_queue&, proxy_registry&);
/// Add a worker to the hub. /// Add a worker to the hub.
void push(pointer ptr); void push(pointer ptr);
...@@ -62,10 +63,19 @@ public: ...@@ -62,10 +63,19 @@ public:
/// hub is currently empty. /// hub is currently empty.
pointer peek(); pointer peek();
/// Waits until all workers are back at the hub.
void await_workers();
private: private:
// -- member variables ------------------------------------------------------- // -- member variables -------------------------------------------------------
std::atomic<pointer> head_; std::atomic<pointer> head_;
std::atomic<size_t> running_;
std::mutex mtx_;
std::condition_variable cv_;
}; };
} // namespace basp } // namespace basp
......
...@@ -73,6 +73,11 @@ basp_broker::~basp_broker() { ...@@ -73,6 +73,11 @@ basp_broker::~basp_broker() {
// -- implementation of local_actor/broker ------------------------------------- // -- implementation of local_actor/broker -------------------------------------
void basp_broker::on_exit() { void basp_broker::on_exit() {
// Wait until all pending messages of workers have been shipped.
// TODO: this blocks the calling thread. This is only safe because we know
// that the middleman calls this in its stop() function. However,
// ultimately we should find a nonblocking solution here.
instance.hub().await_workers();
// Release any obsolete state. // Release any obsolete state.
ctx.clear(); ctx.clear();
// Make sure all spawn servers are down before clearing the container. // Make sure all spawn servers are down before clearing the container.
......
...@@ -49,7 +49,7 @@ instance::instance(abstract_broker* parent, callee& lstnr) ...@@ -49,7 +49,7 @@ instance::instance(abstract_broker* parent, callee& lstnr)
auto workers = get_or(config(), "middleman.workers", auto workers = get_or(config(), "middleman.workers",
defaults::middleman::workers); defaults::middleman::workers);
for (size_t i = 0; i < workers; ++i) for (size_t i = 0; i < workers; ++i)
hub_.push_new_worker(queue_, proxies()); hub_.add_new_worker(queue_, proxies());
} }
connection_state instance::handle(execution_unit* ctx, connection_state instance::handle(execution_unit* ctx,
......
...@@ -329,18 +329,12 @@ void middleman::stop() { ...@@ -329,18 +329,12 @@ void middleman::stop() {
while (backend().try_run_once()) while (backend().try_run_once())
; // nop ; // nop
} }
named_brokers_.clear();
scoped_actor self{system(), true}; scoped_actor self{system(), true};
self->send_exit(manager_, exit_reason::kill); self->send_exit(manager_, exit_reason::kill);
if (!get_or(config(), "middleman.attach-utility-actors", false)) if (!get_or(config(), "middleman.attach-utility-actors", false))
self->wait_for(manager_); self->wait_for(manager_);
destroy(manager_); destroy(manager_);
// Note: we intentionally don't call `named_brokers_.clear()` here. The BASP
// broker must outlive the scheduler threads. The scheduler is stopped
// *after* the MM. However, the BASP workers still need to return to their
// hub safely, should some of them are still running at this point. By not
// clearing the container, we keep the BASP broker (and thus the BASP
// instance) alive until the MM module gets destroyed. At that point, all
// BASP workers are safe in their hub.
} }
void middleman::init(actor_system_config& cfg) { void middleman::init(actor_system_config& cfg) {
......
...@@ -26,11 +26,12 @@ namespace basp { ...@@ -26,11 +26,12 @@ namespace basp {
// -- constructors, destructors, and assignment operators ---------------------- // -- constructors, destructors, and assignment operators ----------------------
worker_hub::worker_hub() : head_(nullptr) { worker_hub::worker_hub() : head_(nullptr), running_(0) {
// nop // nop
} }
worker_hub::~worker_hub() { worker_hub::~worker_hub() {
await_workers();
auto head = head_.load(); auto head = head_.load();
while (head != nullptr) { while (head != nullptr) {
auto next = head->next_.load(); auto next = head->next_.load();
...@@ -41,17 +42,27 @@ worker_hub::~worker_hub() { ...@@ -41,17 +42,27 @@ worker_hub::~worker_hub() {
// -- properties --------------------------------------------------------------- // -- properties ---------------------------------------------------------------
void worker_hub::push_new_worker(message_queue& queue, void worker_hub::add_new_worker(message_queue& queue, proxy_registry& proxies) {
proxy_registry& proxies) { auto ptr = new worker(*this, queue, proxies);
push(new worker(*this, queue, proxies)); auto next = head_.load();
for (;;) {
ptr->next_ = next;
if (head_.compare_exchange_strong(next, ptr))
return;
}
} }
void worker_hub::push(pointer ptr) { void worker_hub::push(pointer ptr) {
auto next = head_.load(); auto next = head_.load();
for (;;) { for (;;) {
ptr->next_ = next; ptr->next_ = next;
if (head_.compare_exchange_strong(next, ptr)) if (head_.compare_exchange_strong(next, ptr)) {
if (--running_ == 0) {
std::unique_lock<std::mutex> guard{mtx_};
cv_.notify_all();
}
return; return;
}
} }
} }
...@@ -61,8 +72,11 @@ worker_hub::pointer worker_hub::pop() { ...@@ -61,8 +72,11 @@ worker_hub::pointer worker_hub::pop() {
return nullptr; return nullptr;
for (;;) { for (;;) {
auto next = result->next_.load(); auto next = result->next_.load();
if (head_.compare_exchange_strong(result, next)) if (head_.compare_exchange_strong(result, next)) {
if (result != nullptr)
++running_;
return result; return result;
}
} }
} }
...@@ -70,6 +84,12 @@ worker_hub::pointer worker_hub::peek() { ...@@ -70,6 +84,12 @@ worker_hub::pointer worker_hub::peek() {
return head_.load(); return head_.load();
} }
void worker_hub::await_workers() {
std::unique_lock<std::mutex> guard{mtx_};
while (running_ != 0)
cv_.wait(guard);
}
} // namespace basp } // namespace basp
} // namespace io } // namespace io
} // namespace caf } // namespace caf
...@@ -105,7 +105,7 @@ CAF_TEST_FIXTURE_SCOPE(worker_tests, fixture) ...@@ -105,7 +105,7 @@ CAF_TEST_FIXTURE_SCOPE(worker_tests, fixture)
CAF_TEST(deliver serialized message) { CAF_TEST(deliver serialized message) {
CAF_MESSAGE("create the BASP worker"); CAF_MESSAGE("create the BASP worker");
CAF_REQUIRE_EQUAL(hub.peek(), nullptr); CAF_REQUIRE_EQUAL(hub.peek(), nullptr);
hub.push_new_worker(queue, proxies); hub.add_new_worker(queue, proxies);
CAF_REQUIRE_NOT_EQUAL(hub.peek(), nullptr); CAF_REQUIRE_NOT_EQUAL(hub.peek(), nullptr);
auto w = hub.pop(); auto w = hub.pop();
CAF_MESSAGE("create a fake message + BASP header"); CAF_MESSAGE("create a fake message + BASP header");
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment