Commit fa5f795c authored by Sebastian Woelke's avatar Sebastian Woelke

Add cache support for locality guided scheduling

parent 58015d19
...@@ -70,7 +70,7 @@ set (LIBCAF_CORE_SRCS ...@@ -70,7 +70,7 @@ set (LIBCAF_CORE_SRCS
src/monitorable_actor.cpp src/monitorable_actor.cpp
src/node_id.cpp src/node_id.cpp
src/outbound_path.cpp src/outbound_path.cpp
src/numa_aware_work_stealing.cpp src/locality_guided_scheduling.cpp
src/parse_ini.cpp src/parse_ini.cpp
src/pretty_type_name.cpp src/pretty_type_name.cpp
src/private_thread.cpp src/private_thread.cpp
......
...@@ -264,9 +264,10 @@ public: ...@@ -264,9 +264,10 @@ public:
size_t work_stealing_relaxed_steal_interval; size_t work_stealing_relaxed_steal_interval;
size_t work_stealing_relaxed_sleep_duration_us; size_t work_stealing_relaxed_sleep_duration_us;
// -- config parameters for numa aware work-stealing ------------------------- // -- config parameters for locality guided scheduling (LGS) -----------------
size_t numa_aware_work_stealing_neighborhood_level; atom_value lgs_actor_pinning_entity;
atom_value lgs_weighted_work_stealing_start_entity;
// -- config parameters for the logger --------------------------------------- // -- config parameters for the logger ---------------------------------------
......
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
#include "caf/policy/work_sharing.hpp" #include "caf/policy/work_sharing.hpp"
#include "caf/policy/work_stealing.hpp" #include "caf/policy/work_stealing.hpp"
#include "caf/policy/numa_aware_work_stealing.hpp" #include "caf/policy/locality_guided_scheduling.hpp"
#include "caf/scheduler/coordinator.hpp" #include "caf/scheduler/coordinator.hpp"
#include "caf/scheduler/test_coordinator.hpp" #include "caf/scheduler/test_coordinator.hpp"
...@@ -230,7 +230,7 @@ actor_system::actor_system(actor_system_config& cfg) ...@@ -230,7 +230,7 @@ actor_system::actor_system(actor_system_config& cfg)
using test = scheduler::test_coordinator; using test = scheduler::test_coordinator;
using share = scheduler::coordinator<policy::work_sharing>; using share = scheduler::coordinator<policy::work_sharing>;
using steal = scheduler::coordinator<policy::work_stealing>; using steal = scheduler::coordinator<policy::work_stealing>;
using numa_steal = scheduler::coordinator<policy::numa_aware_work_stealing>; using weighted_steal = scheduler::coordinator<policy::locality_guided_scheduling>;
using profiled_share = scheduler::profiled_coordinator<policy::profiled<policy::work_sharing>>; using profiled_share = scheduler::profiled_coordinator<policy::profiled<policy::work_sharing>>;
using profiled_steal = scheduler::profiled_coordinator<policy::profiled<policy::work_stealing>>; using profiled_steal = scheduler::profiled_coordinator<policy::profiled<policy::work_stealing>>;
// set scheduler only if not explicitly loaded by user // set scheduler only if not explicitly loaded by user
...@@ -239,12 +239,12 @@ actor_system::actor_system(actor_system_config& cfg) ...@@ -239,12 +239,12 @@ actor_system::actor_system(actor_system_config& cfg)
stealing = 0x0001, stealing = 0x0001,
sharing = 0x0002, sharing = 0x0002,
testing = 0x0003, testing = 0x0003,
numa_stealing = 0x0004, weighted_stealing = 0x0004,
profiled = 0x0100, profiled = 0x0100,
profiled_stealing = 0x0101, profiled_stealing = 0x0101,
profiled_sharing = 0x0102 profiled_sharing = 0x0102
}; };
sched_conf sc = numa_stealing; sched_conf sc = weighted_stealing;
if (cfg.scheduler_policy == atom("stealing")) if (cfg.scheduler_policy == atom("stealing"))
sc = stealing; sc = stealing;
...@@ -252,17 +252,17 @@ actor_system::actor_system(actor_system_config& cfg) ...@@ -252,17 +252,17 @@ actor_system::actor_system(actor_system_config& cfg)
sc = sharing; sc = sharing;
else if (cfg.scheduler_policy == atom("testing")) else if (cfg.scheduler_policy == atom("testing"))
sc = testing; sc = testing;
else if (cfg.scheduler_policy != atom("numa-steal")) else if (cfg.scheduler_policy != atom("w-stealing"))
std::cerr std::cerr
<< "[WARNING] " << deep_to_string(cfg.scheduler_policy) << "[WARNING] " << deep_to_string(cfg.scheduler_policy)
<< " is an unrecognized scheduler pollicy, " << " is an unrecognized scheduler pollicy, "
"falling back to 'numa-steal' (i.e. numa aware work-stealing)" "falling back to 'w-stealing' (i.e. weighted work stealing)"
<< std::endl; << std::endl;
if (cfg.scheduler_enable_profiling) if (cfg.scheduler_enable_profiling)
sc = static_cast<sched_conf>(sc | profiled); sc = static_cast<sched_conf>(sc | profiled);
switch (sc) { switch (sc) {
default: // any invalid configuration falls back to numa work stealing default: // any invalid configuration falls back to numa work stealing
sched.reset(new numa_steal(*this)); sched.reset(new weighted_steal(*this));
break; break;
case sharing: case sharing:
sched.reset(new share(*this)); sched.reset(new share(*this));
......
...@@ -112,7 +112,7 @@ actor_system_config::actor_system_config() ...@@ -112,7 +112,7 @@ actor_system_config::actor_system_config()
add_message_type_impl<std::vector<atom_value>>("std::vector<@atom>"); add_message_type_impl<std::vector<atom_value>>("std::vector<@atom>");
add_message_type_impl<std::vector<message>>("std::vector<@message>"); add_message_type_impl<std::vector<message>>("std::vector<@message>");
// (1) hard-coded defaults // (1) hard-coded defaults
scheduler_policy = atom("numa-steal"); scheduler_policy = atom("w-stealing");
scheduler_max_threads = std::max(std::thread::hardware_concurrency(), scheduler_max_threads = std::max(std::thread::hardware_concurrency(),
unsigned{4}); unsigned{4});
scheduler_max_throughput = std::numeric_limits<size_t>::max(); scheduler_max_throughput = std::numeric_limits<size_t>::max();
...@@ -125,7 +125,8 @@ actor_system_config::actor_system_config() ...@@ -125,7 +125,8 @@ actor_system_config::actor_system_config()
work_stealing_moderate_sleep_duration_us = 50; work_stealing_moderate_sleep_duration_us = 50;
work_stealing_relaxed_steal_interval = 1; work_stealing_relaxed_steal_interval = 1;
work_stealing_relaxed_sleep_duration_us = 10000; work_stealing_relaxed_sleep_duration_us = 10000;
numa_aware_work_stealing_neighborhood_level = 1; lgs_actor_pinning_entity = atom("node");
lgs_weighted_work_stealing_start_entity = atom("cache");
logger_file_name = "actor_log_[PID]_[TIMESTAMP]_[NODE].log"; logger_file_name = "actor_log_[PID]_[TIMESTAMP]_[NODE].log";
logger_file_format = "%r %c %p %a %t %C %M %F:%L %m%n"; logger_file_format = "%r %c %p %a %t %C %M %F:%L %m%n";
logger_console = atom("none"); logger_console = atom("none");
...@@ -139,18 +140,20 @@ actor_system_config::actor_system_config() ...@@ -139,18 +140,20 @@ actor_system_config::actor_system_config()
middleman_detach_multiplexer = true; middleman_detach_multiplexer = true;
// fill our options vector for creating INI and CLI parsers // fill our options vector for creating INI and CLI parsers
opt_group{options_, "scheduler"} opt_group{options_, "scheduler"}
.add(scheduler_policy, "policy", .add(scheduler_policy, "policy",
"sets the scheduling policy to either 'stealing' (default) or 'sharing'") "sets the scheduling policy to either 'w-stealing' (default), "
.add(scheduler_max_threads, "max-threads", "'stealing' or 'sharing'")
"sets a fixed number of worker threads for the scheduler") .add(scheduler_max_threads, "max-threads",
.add(scheduler_max_throughput, "max-throughput", "sets a fixed number of worker threads for the scheduler")
"sets the maximum number of messages an actor consumes before yielding") .add(
.add(scheduler_enable_profiling, "enable-profiling", scheduler_max_throughput, "max-throughput",
"enables or disables profiler output") "sets the maximum number of messages an actor consumes before yielding")
.add(scheduler_profiling_ms_resolution, "profiling-ms-resolution", .add(scheduler_enable_profiling, "enable-profiling",
"sets the rate in ms in which the profiler collects data") "enables or disables profiler output")
.add(scheduler_profiling_output_file, "profiling-output-file", .add(scheduler_profiling_ms_resolution, "profiling-ms-resolution",
"sets the output file for the profiler"); "sets the rate in ms in which the profiler collects data")
.add(scheduler_profiling_output_file, "profiling-output-file",
"sets the output file for the profiler");
opt_group(options_, "work-stealing") opt_group(options_, "work-stealing")
.add(work_stealing_aggressive_poll_attempts, "aggressive-poll-attempts", .add(work_stealing_aggressive_poll_attempts, "aggressive-poll-attempts",
"sets the number of zero-sleep-interval polling attempts") "sets the number of zero-sleep-interval polling attempts")
...@@ -166,9 +169,12 @@ actor_system_config::actor_system_config() ...@@ -166,9 +169,12 @@ actor_system_config::actor_system_config()
"sets the frequency of steal attempts during relaxed polling") "sets the frequency of steal attempts during relaxed polling")
.add(work_stealing_relaxed_sleep_duration_us, "relaxed-sleep-duration", .add(work_stealing_relaxed_sleep_duration_us, "relaxed-sleep-duration",
"sets the sleep interval between poll attempts during relaxed polling"); "sets the sleep interval between poll attempts during relaxed polling");
opt_group{options_, "numa"} opt_group{options_, "lgs"}
.add(numa_aware_work_stealing_neighborhood_level, "neighborhood-level", .add(lgs_actor_pinning_entity, "actor-pinning-entity",
"defines the neighborhood radius (0=all, 1=next smaller group, 2=...)"); "defines the actor pinning entity (pu, cache, node, system)")
.add(
lgs_weighted_work_stealing_start_entity, "w-stealing-entity",
"defines the weighted work stealing start entity (cache, node, system)");
opt_group{options_, "logger"} opt_group{options_, "logger"}
.add(logger_file_name, "file-name", .add(logger_file_name, "file-name",
"sets the filesystem path of the log file") "sets the filesystem path of the log file")
...@@ -436,7 +442,7 @@ actor_system_config& actor_system_config::parse(message& args, ...@@ -436,7 +442,7 @@ actor_system_config& actor_system_config::parse(message& args,
atom("asio") atom("asio")
# endif # endif
}, middleman_network_backend, "middleman.network-backend"); }, middleman_network_backend, "middleman.network-backend");
verify_atom_opt({atom("stealing"), atom("sharing"), atom("numa-steal")}, verify_atom_opt({atom("stealing"), atom("sharing"), atom("w-stealing")},
scheduler_policy, "scheduler.policy "); scheduler_policy, "scheduler.policy ");
if (res.opts.count("caf#dump-config") != 0u) { if (res.opts.count("caf#dump-config") != 0u) {
cli_helptext_printed = true; cli_helptext_printed = true;
......
...@@ -17,18 +17,18 @@ ...@@ -17,18 +17,18 @@
* http://www.boost.org/LICENSE_1_0.txt. * * http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/ ******************************************************************************/
#include "caf/policy/numa_aware_work_stealing.hpp" #include "caf/policy/locality_guided_scheduling.hpp"
namespace caf { namespace caf {
namespace policy { namespace policy {
numa_aware_work_stealing::~numa_aware_work_stealing() { locality_guided_scheduling::~locality_guided_scheduling() {
// nop // nop
} }
std::ostream& std::ostream&
operator<<(std::ostream& s, operator<<(std::ostream& s,
const numa_aware_work_stealing::hwloc_bitmap_wrapper& w) { const locality_guided_scheduling::bitmap_wrapper_t& w) {
char* tmp = nullptr; char* tmp = nullptr;
hwloc_bitmap_asprintf(&tmp, w.get()); hwloc_bitmap_asprintf(&tmp, w.get());
s << std::string(tmp); s << std::string(tmp);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment