Commit 808dc849 authored by Dominik Charousset's avatar Dominik Charousset

Use policy-based design for scheduler

This patch separates interface and implementation of scheduling coordinator and
worker. The implementation is now outsourced into two policies: a steal policy
and a (de)queue policy. The previously hardwired implementation is split into
the two policy classes `iterative_stealing` and `fork_join`.
parent 1f9aa431
......@@ -22,24 +22,11 @@
#include <atomic>
#include <cstddef> // size_t
namespace cppa {
class node_id;
namespace scheduler {
class coordinator;
}
} // namespace cppa
#include "cppa/fwd.hpp"
namespace cppa {
namespace detail {
class logging;
class message_data;
class group_manager;
class actor_registry;
class uniform_type_info_map;
class abstract_singleton {
public:
......@@ -72,7 +59,7 @@ class singletons {
static node_id get_node_id();
static scheduler::coordinator* get_scheduling_coordinator();
static scheduler::abstract_coordinator* get_scheduling_coordinator();
static group_manager* get_group_manager();
......
......@@ -23,7 +23,7 @@ namespace cppa {
class resumable;
/*
/**
* @brief Identifies an execution unit, e.g., a worker thread of the scheduler.
*/
class execution_unit {
......@@ -32,7 +32,7 @@ class execution_unit {
virtual ~execution_unit();
/*
/**
* @brief Enqueues @p ptr to the job list of the execution unit.
* @warning Must only be called from a {@link resumable} currently
* executed by this execution unit.
......
......@@ -29,11 +29,11 @@ class intrusive_ptr;
// classes
class actor;
class group;
class message;
class channel;
class node_id;
class behavior;
class resumable;
class message;
class actor_addr;
class local_actor;
class actor_proxy;
......@@ -64,10 +64,23 @@ template<typename T, typename U>
T actor_cast(const U&);
namespace io {
class broker;
class middleman;
} // namespace io
class broker;
namespace scheduler {
class abstract_worker;
class abstract_coordinator;
} // namespace scheduler
} // namespace io
namespace detail {
class logging;
class singletons;
class message_data;
class group_manager;
class actor_registry;
class uniform_type_info_map;
} // namespace detail
} // namespace cppa
......
/******************************************************************************\
* ___ __ *
* /\_ \ __/\ \ *
* \//\ \ /\_\ \ \____ ___ _____ _____ __ *
* \ \ \ \/\ \ \ '__`\ /'___\/\ '__`\/\ '__`\ /'__`\ *
* \_\ \_\ \ \ \ \L\ \/\ \__/\ \ \L\ \ \ \L\ \/\ \L\.\_ *
* /\____\\ \_\ \_,__/\ \____\\ \ ,__/\ \ ,__/\ \__/.\_\ *
* \/____/ \/_/\/___/ \/____/ \ \ \/ \ \ \/ \/__/\/_/ *
* \ \_\ \ \_\ *
* \/_/ \/_/ *
* *
* Copyright (C) 2011 - 2014 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the Boost Software License, Version 1.0. See *
* accompanying file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt *
\******************************************************************************/
#ifndef CPPA_POLICY_FORK_JOIN_HPP
#define CPPA_POLICY_FORK_JOIN_HPP
#include <chrono>
#include <vector>
#include <thread>
#include "cppa/resumable.hpp"
#include "cppa/detail/producer_consumer_list.hpp"
namespace cppa {
namespace policy {
/**
* @brief An implementation of the {@link job_queue_policy} concept for
* fork-join like processing of actors.
*
* This work-stealing fork-join implementation uses two queues: a
* synchronized queue accessible by other threads and an internal queue.
* Access to the synchronized queue is minimized.
* The reasoning behind this design decision is that it
* has been shown that stealing actually is very rare for most workloads [1].
* Hence, implementations should focus on the performance in
* the non-stealing case. For this reason, each worker has an exposed
* job queue that can be accessed by the central scheduler instance as
* well as other workers, but it also has a private job list it is
* currently working on. To account for the load balancing aspect, each
* worker makes sure that at least one job is left in its exposed queue
* to allow other workers to steal it.
*
* @relates job_queue_policy
*/
class fork_join {
public:
fork_join() = default;
fork_join(fork_join&& other) {
// delegate to move assignment operator
*this = std::move(other);
}
fork_join& operator=(fork_join&& other) {
m_private_queue = std::move(other.m_private_queue);
auto next = [&] { return other.m_exposed_queue.try_pop(); };
for (auto j = next(); j != nullptr; j = next()) {
m_exposed_queue.push_back(j);
}
return *this;
}
/**
* @brief A thead-safe queue implementation.
*/
using sync_queue = detail::producer_consumer_list<resumable>;
/**
* @brief A queue implementation supporting fast push and pop
* operations. Note that we do dequeue from the back of the
* queue.
*/
using priv_queue = std::vector<resumable*>;
template<class Worker>
inline void external_enqueue(Worker*, resumable* job) {
m_exposed_queue.push_back(job);
}
template<class Worker>
inline void internal_enqueue(Worker*, resumable* job) {
// give others the opportunity to steal from us
if (m_exposed_queue.empty()) {
if (m_private_queue.empty()) {
m_exposed_queue.push_back(job);
} else {
m_exposed_queue.push_back(m_private_queue.front());
m_private_queue.erase(m_private_queue.begin());
m_private_queue.push_back(job);
}
} else {
m_private_queue.push_back(job);
}
}
template<class Worker>
inline resumable* try_external_dequeue(Worker*) {
return m_exposed_queue.try_pop();
}
template<class Worker>
inline resumable* internal_dequeue(Worker* self) {
resumable* job;
auto local_poll = [&]() -> bool {
if (!m_private_queue.empty()) {
job = m_private_queue.back();
m_private_queue.pop_back();
return true;
}
return false;
};
auto aggressive_poll = [&]() -> bool {
for (int i = 1; i < 101; ++i) {
job = m_exposed_queue.try_pop();
if (job) {
return true;
}
// try to steal every 10 poll attempts
if ((i % 10) == 0) {
job = self->raid();
if (job) {
return true;
}
}
std::this_thread::yield();
}
return false;
};
auto moderate_poll = [&]() -> bool {
for (int i = 1; i < 550; ++i) {
job = m_exposed_queue.try_pop();
if (job) {
return true;
}
// try to steal every 5 poll attempts
if ((i % 5) == 0) {
job = self->raid();
if (job) {
return true;
}
}
std::this_thread::sleep_for(std::chrono::microseconds(50));
}
return false;
};
auto relaxed_poll = [&]() -> bool {
for (;;) {
job = m_exposed_queue.try_pop();
if (job) {
return true;
}
// always try to steal at this stage
job = self->raid();
if (job) {
return true;
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
};
local_poll() || aggressive_poll() || moderate_poll() || relaxed_poll();
return job;
}
template<class Worker>
void clear_internal_queue(Worker*) {
// give others the opportunity to steal unfinished jobs
for (auto ptr : m_private_queue) {
m_exposed_queue.push_back(ptr);
}
m_private_queue.clear();
}
template<class Worker>
void assert_stealable(Worker*) {
// give others the opportunity to steal from us
if (m_private_queue.size() > 1 && m_exposed_queue.empty()) {
m_exposed_queue.push_back(m_private_queue.front());
m_private_queue.erase(m_private_queue.begin());
}
}
template<class Worker, typename UnaryFunction>
void consume_all(Worker*, UnaryFunction f) {
for (auto job : m_private_queue) {
f(job);
}
m_private_queue.clear();
auto next = [&] { return m_exposed_queue.try_pop(); };
for (auto job = next(); job != nullptr; job = next()) {
f(job);
}
}
private:
// this queue is exposed to others, i.e., other workers
// may attempt to steal jobs from it and the central scheduling
// unit can push new jobs to the queue
sync_queue m_exposed_queue;
// internal job queue
priv_queue m_private_queue;
};
} // namespace policy
} // namespace cppa
#endif // CPPA_POLICY_FORK_JOIN_HPP
/******************************************************************************\
* ___ __ *
* /\_ \ __/\ \ *
* \//\ \ /\_\ \ \____ ___ _____ _____ __ *
* \ \ \ \/\ \ \ '__`\ /'___\/\ '__`\/\ '__`\ /'__`\ *
* \_\ \_\ \ \ \ \L\ \/\ \__/\ \ \L\ \ \ \L\ \/\ \L\.\_ *
* /\____\\ \_\ \_,__/\ \____\\ \ ,__/\ \ ,__/\ \__/.\_\ *
* \/____/ \/_/\/___/ \/____/ \ \ \/ \ \ \/ \/__/\/_/ *
* \ \_\ \ \_\ *
* \/_/ \/_/ *
* *
* Copyright (C) 2011 - 2014 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the Boost Software License, Version 1.0. See *
* accompanying file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt *
\******************************************************************************/
#ifndef CPPA_POLICY_ITERATIVE_STEALING_HPP
#define CPPA_POLICY_ITERATIVE_STEALING_HPP
#include <cstddef>
#include "cppa/fwd.hpp"
namespace cppa {
namespace policy {
/**
* @brief An implementation of the {@link steal_policy} concept
* that iterates over all other workers when stealing.
*
* @relates steal_policy
*/
class iterative_stealing {
public:
constexpr iterative_stealing() : m_victim(0) { }
template<class Worker>
resumable* raid(Worker* self) {
// try once to steal from anyone
auto inc = [](size_t arg) -> size_t {
return arg + 1;
};
auto dec = [](size_t arg) -> size_t {
return arg - 1;
};
// reduce probability of 'steal collisions' by letting
// half the workers pick victims by increasing IDs and
// the other half by decreasing IDs
size_t (*next)(size_t) = (self->id() % 2) == 0 ? inc : dec;
auto n = self->parent()->num_workers();
for (size_t i = 0; i < n; ++i) {
m_victim = next(m_victim) % n;
if (m_victim != self->id()) {
auto job = self->parent()->worker_by_id(m_victim)->try_steal();
if (job) {
return job;
}
}
}
return nullptr;
}
private:
size_t m_victim;
};
} // namespace policy
} // namespace cppa
#endif // CPPA_POLICY_ITERATIVE_STEALING_HPP
/******************************************************************************\
* ___ __ *
* /\_ \ __/\ \ *
* \//\ \ /\_\ \ \____ ___ _____ _____ __ *
* \ \ \ \/\ \ \ '__`\ /'___\/\ '__`\/\ '__`\ /'__`\ *
* \_\ \_\ \ \ \ \L\ \/\ \__/\ \ \L\ \ \ \L\ \/\ \L\.\_ *
* /\____\\ \_\ \_,__/\ \____\\ \ ,__/\ \ ,__/\ \__/.\_\ *
* \/____/ \/_/\/___/ \/____/ \ \ \/ \ \ \/ \/__/\/_/ *
* \ \_\ \ \_\ *
* \/_/ \/_/ *
* *
* Copyright (C) 2011 - 2014 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the Boost Software License, Version 1.0. See *
* accompanying file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt *
\******************************************************************************/
#ifndef CPPA_POLICY_JOB_QUEUE_POLICY_HPP
#define CPPA_POLICY_JOB_QUEUE_POLICY_HPP
#include "cppa/fwd.hpp"
namespace cppa {
namespace policy {
/**
* @brief This concept class describes the interface of a policy class
* for managing the queue(s) of a scheduler worker.
*/
class job_queue_policy {
public:
/**
* @brief Enqueues a new job to the worker's queue from an
* external source, i.e., from any other thread.
*/
template<class Worker>
void external_enqueue(Worker* self, resumable* job);
/**
* @brief Enqueues a new job to the worker's queue from an
* internal source, i.e., from the same thread.
*/
template<class Worker>
void internal_enqueue(Worker* self, resumable* job);
/**
* @brief Called by external sources to try to dequeue an element.
* Returns @p nullptr if no element could be dequeued immediately.
*/
template<class Worker>
resumable* try_external_dequeue(Worker* self);
/**
* @brief Called by the worker itself to acquire a new job.
* Blocks until a job could be dequeued.
*/
template<class Worker>
resumable* internal_dequeue(Worker* self);
/**
* @brief Moves all elements form the internal queue to the external queue.
*/
template<class Worker>
void clear_internal_queue(Worker* self);
/**
* @brief Tries to move at least one element from the internal queue ot
* the external queue if possible to allow others to steal from us.
*/
template<class Worker>
void assert_stealable(Worker* self);
/**
* @brief Applies given functor to all elements in all queues and
* clears all queues afterwards.
*/
template<class Worker, typename UnaryFunction>
void consume_all(Worker* self, UnaryFunction f);
};
} // namespace policy
} // namespace cppa
#endif // CPPA_POLICY_JOB_QUEUE_POLICY_HPP
/******************************************************************************\
* ___ __ *
* /\_ \ __/\ \ *
* \//\ \ /\_\ \ \____ ___ _____ _____ __ *
* \ \ \ \/\ \ \ '__`\ /'___\/\ '__`\/\ '__`\ /'__`\ *
* \_\ \_\ \ \ \ \L\ \/\ \__/\ \ \L\ \ \ \L\ \/\ \L\.\_ *
* /\____\\ \_\ \_,__/\ \____\\ \ ,__/\ \ ,__/\ \__/.\_\ *
* \/____/ \/_/\/___/ \/____/ \ \ \/ \ \ \/ \/__/\/_/ *
* \ \_\ \ \_\ *
* \/_/ \/_/ *
* *
* Copyright (C) 2011 - 2014 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the Boost Software License, Version 1.0. See *
* accompanying file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt *
\******************************************************************************/
#ifndef CPPA_POLICY_STEAL_POLICY_HPP
#define CPPA_POLICY_STEAL_POLICY_HPP
#include <cstddef>
#include "cppa/fwd.hpp"
namespace cppa {
namespace policy {
/**
* @brief This concept class describes the interface of a policy class
* for stealing jobs from other workers.
*/
class steal_policy {
public:
/**
* @brief Go on a raid in quest for a shiny new job. Returns @p nullptr
* if no other worker provided any work to steal.
*/
template<class Worker>
resumable* raid(Worker* self);
};
} // namespace policy
} // namespace cppa
#endif // CPPA_POLICY_STEAL_POLICY_HPP
This diff is collapsed.
This diff is collapsed.
......@@ -36,7 +36,7 @@ namespace detail {
namespace {
std::atomic<abstract_singleton*> s_plugins[singletons::max_plugin_singletons];
std::atomic<scheduler::coordinator*> s_scheduling_coordinator;
std::atomic<scheduler::abstract_coordinator*> s_scheduling_coordinator;
std::atomic<uniform_type_info_map*> s_uniform_type_info_map;
std::atomic<actor_registry*> s_actor_registry;
std::atomic<group_manager*> s_group_manager;
......@@ -84,7 +84,7 @@ group_manager* singletons::get_group_manager() {
return lazy_get(s_group_manager);
}
scheduler::coordinator* singletons::get_scheduling_coordinator() {
scheduler::abstract_coordinator* singletons::get_scheduling_coordinator() {
return lazy_get(s_scheduling_coordinator);
}
......
......@@ -304,6 +304,7 @@ struct master : event_based_actor {
behavior make_behavior() override {
return (
on(atom("done")) >> [=] {
CPPA_PRINT("master: received done");
quit(exit_reason::user_shutdown);
}
);
......@@ -316,7 +317,12 @@ struct slave : event_based_actor {
behavior make_behavior() override {
link_to(master);
trap_exit(true);
return (
[=](const exit_msg& msg) {
CPPA_PRINT("slave: received exit message");
quit(msg.reason);
},
others() >> CPPA_UNEXPECTED_MSG_CB(this)
);
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment