Commit 503bc2f6 authored by Dominik Charousset's avatar Dominik Charousset

Merge branch 'topic/improve-autoconnect'

parents b092a645 e781635d
...@@ -123,12 +123,15 @@ public: ...@@ -123,12 +123,15 @@ public:
friend class abstract_actor; friend class abstract_actor;
/// The number of actors implictly spawned by the actor system on startup. /// The number of actors implictly spawned by the actor system on startup.
static constexpr size_t num_internal_actors = 2; static constexpr size_t num_internal_actors = 3;
/// Returns the ID of an internal actor by its name. /// Returns the ID of an internal actor by its name.
/// @pre x in {'SpawnServ', 'ConfigServ', 'StreamServ'} /// @pre x in {'SpawnServ', 'ConfigServ', 'StreamServ', 'PeerServ'}
static constexpr size_t internal_actor_id(atom_value x) { static constexpr size_t internal_actor_id(atom_value x) {
return x == atom("SpawnServ") ? 0 : (x == atom("ConfigServ") ? 1 : 2); return x == atom("SpawnServ") ? 0
: (x == atom("ConfigServ") ? 1
: (x == atom("PeerServ") ? 2
: 3));
} }
/// Returns the internal actor for dynamic spawn operations. /// Returns the internal actor for dynamic spawn operations.
...@@ -142,6 +145,11 @@ public: ...@@ -142,6 +145,11 @@ public:
return internal_actors_[internal_actor_id(atom("ConfigServ"))]; return internal_actors_[internal_actor_id(atom("ConfigServ"))];
} }
/// Returns the internal actor for storing the addresses of its peers.
inline const strong_actor_ptr& peer_serv() const {
return internal_actors_[internal_actor_id(atom("PeerServ"))];
}
actor_system() = delete; actor_system() = delete;
actor_system(const actor_system&) = delete; actor_system(const actor_system&) = delete;
actor_system& operator=(const actor_system&) = delete; actor_system& operator=(const actor_system&) = delete;
...@@ -571,6 +579,11 @@ private: ...@@ -571,6 +579,11 @@ private:
// -- member variables ------------------------------------------------------- // -- member variables -------------------------------------------------------
/// Sets the internal actor for storing the peer addresses.
void peer_serv(strong_actor_ptr x) {
internal_actors_[internal_actor_id(atom("PeerServ"))] = std::move(x);
}
/// Used to generate ascending actor IDs. /// Used to generate ascending actor IDs.
std::atomic<size_t> ids_; std::atomic<size_t> ids_;
......
...@@ -166,6 +166,98 @@ behavior spawn_serv_impl(stateful_actor<spawn_serv_state>* self) { ...@@ -166,6 +166,98 @@ behavior spawn_serv_impl(stateful_actor<spawn_serv_state>* self) {
}; };
} }
// -- peer server --------------------------------------------------------------
// A peer server keepy track of the addresses to reach its peers. All addresses
// for a given node are stored under the string representation of iits node id.
// When an entry is requested that does not exist, the requester is subscribed
// to the key and sent a message as soon as an entry is set and then removed
// from the subscribers.
struct peer_state {
using key_type = std::string;
using mapped_type = message;
using subscriber_set = std::unordered_set<strong_actor_ptr>;
using topic_set = std::unordered_set<std::string>;
std::unordered_map<key_type, std::pair<mapped_type, subscriber_set>> data;
std::unordered_map<strong_actor_ptr, topic_set> subscribers;
static const char* name;
template <class Processor>
friend void serialize(Processor& proc, peer_state& x, unsigned int) {
proc & x.data;
proc & x.subscribers;
}
};
const char* peer_state::name = "peer_server";
behavior peer_serv_impl(stateful_actor<peer_state>* self) {
CAF_LOG_TRACE("");
std::string wildcard = "*";
auto unsubscribe_all = [=](actor subscriber) {
auto& subscribers = self->state.subscribers;
auto ptr = actor_cast<strong_actor_ptr>(subscriber);
auto i = subscribers.find(ptr);
if (i == subscribers.end())
return;
for (auto& key : i->second)
self->state.data[key].second.erase(ptr);
subscribers.erase(i);
};
self->set_down_handler([=](down_msg& dm) {
CAF_LOG_TRACE(CAF_ARG(dm));
auto ptr = actor_cast<strong_actor_ptr>(dm.source);
if (ptr)
unsubscribe_all(actor_cast<actor>(std::move(ptr)));
});
return {
// Set a key/value pair.
[=](put_atom, const std::string& key, message& msg) {
CAF_LOG_TRACE(CAF_ARG(key) << CAF_ARG(msg));
if (key == wildcard || key.empty())
return;
auto& vp = self->state.data[key];
vp.first = std::move(msg);
for (auto& subscriber_ptr : vp.second) {
// We never put a nullptr in our map.
auto subscriber = actor_cast<actor>(subscriber_ptr);
if (subscriber != self->current_sender()) {
self->send(subscriber, key, vp.first);
self->state.subscribers[subscriber_ptr].erase(key);
}
}
self->state.data[key].second.clear();
},
// Get a key/value pair.
[=](get_atom, std::string& key) {
auto sender = actor_cast<strong_actor_ptr>(self->current_sender());
if (sender) {
CAF_LOG_TRACE(CAF_ARG(key));
// Get the value ...
if (key == wildcard || key.empty())
return;
auto d = self->state.data.find(key);
if (d != self->state.data.end()) {
self->send(actor_cast<actor>(sender), std::move(key),
d->second.first);
return;
}
// ... or sub if it is not available.
CAF_LOG_TRACE(CAF_ARG(key) << CAF_ARG(sender));
self->state.data[key].second.insert(sender);
auto& subscribers = self->state.subscribers;
auto s = subscribers.find(sender);
if (s != subscribers.end()) {
s->second.insert(key);
} else {
self->monitor(sender);
subscribers.emplace(sender, peer_state::topic_set{key});
}
}
}
};
}
// -- stream server ------------------------------------------------------------ // -- stream server ------------------------------------------------------------
// The stream server acts as a man-in-the-middle for all streams that cross the // The stream server acts as a man-in-the-middle for all streams that cross the
...@@ -288,14 +380,17 @@ actor_system::actor_system(actor_system_config& cfg) ...@@ -288,14 +380,17 @@ actor_system::actor_system(actor_system_config& cfg)
if (mod) if (mod)
mod->init(cfg); mod->init(cfg);
groups_.init(cfg); groups_.init(cfg);
// spawn config and spawn servers (lazily to not access the scheduler yet) // spawn config, spawn, and peer servers
// (lazily to not access the scheduler yet)
static constexpr auto Flags = hidden + lazy_init; static constexpr auto Flags = hidden + lazy_init;
spawn_serv(actor_cast<strong_actor_ptr>(spawn<Flags>(spawn_serv_impl))); spawn_serv(actor_cast<strong_actor_ptr>(spawn<Flags>(spawn_serv_impl)));
config_serv(actor_cast<strong_actor_ptr>(spawn<Flags>(config_serv_impl))); config_serv(actor_cast<strong_actor_ptr>(spawn<Flags>(config_serv_impl)));
peer_serv(actor_cast<strong_actor_ptr>(spawn<Flags>(peer_serv_impl)));
// fire up remaining modules // fire up remaining modules
registry_.start(); registry_.start();
registry_.put(atom("SpawnServ"), spawn_serv()); registry_.put(atom("SpawnServ"), spawn_serv());
registry_.put(atom("ConfigServ"), config_serv()); registry_.put(atom("ConfigServ"), config_serv());
registry_.put(atom("PeerServ"), peer_serv());
for (auto& mod : modules_) for (auto& mod : modules_)
if (mod) if (mod)
mod->start(); mod->start();
...@@ -316,6 +411,7 @@ actor_system::~actor_system() { ...@@ -316,6 +411,7 @@ actor_system::~actor_system() {
} }
registry_.erase(atom("SpawnServ")); registry_.erase(atom("SpawnServ"));
registry_.erase(atom("ConfigServ")); registry_.erase(atom("ConfigServ"));
registry_.erase(atom("PeerServ"));
registry_.erase(atom("StreamServ")); registry_.erase(atom("StreamServ"));
// group module is the first one, relies on MM // group module is the first one, relies on MM
groups_.stop(); groups_.stop();
......
...@@ -124,7 +124,8 @@ inline bool operator!=(const header& lhs, const header& rhs) { ...@@ -124,7 +124,8 @@ inline bool operator!=(const header& lhs, const header& rhs) {
/// Checks whether given header contains a handshake. /// Checks whether given header contains a handshake.
inline bool is_handshake(const header& hdr) { inline bool is_handshake(const header& hdr) {
return hdr.operation == message_type::server_handshake return hdr.operation == message_type::server_handshake
|| hdr.operation == message_type::client_handshake; || hdr.operation == message_type::client_handshake
|| hdr.operation == message_type::acknowledge_handshake;
} }
/// Checks wheter given header contains a heartbeat. /// Checks wheter given header contains a heartbeat.
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include "caf/error.hpp" #include "caf/error.hpp"
#include "caf/variant.hpp" #include "caf/variant.hpp"
#include "caf/defaults.hpp"
#include "caf/actor_system_config.hpp" #include "caf/actor_system_config.hpp"
#include "caf/binary_deserializer.hpp" #include "caf/binary_deserializer.hpp"
...@@ -44,11 +45,12 @@ namespace basp { ...@@ -44,11 +45,12 @@ namespace basp {
/// Describes a protocol instance managing multiple connections. /// Describes a protocol instance managing multiple connections.
class instance { class instance {
public: public:
using endpoint_handle = routing_table::endpoint_handle;
/// Provides a callback-based interface for certain BASP events. /// Provides a callback-based interface for certain BASP events.
class callee { class callee {
protected: protected:
using buffer_type = std::vector<char>; using buffer_type = std::vector<char>;
using endpoint_handle = variant<connection_handle, datagram_handle>; using endpoint_handle = instance::endpoint_handle;
public: public:
explicit callee(actor_system& sys, proxy_registry::backend& backend); explicit callee(actor_system& sys, proxy_registry::backend& backend);
...@@ -82,14 +84,11 @@ public: ...@@ -82,14 +84,11 @@ public:
std::vector<strong_actor_ptr>& forwarding_stack, std::vector<strong_actor_ptr>& forwarding_stack,
message& msg) = 0; message& msg) = 0;
/// Called whenever BASP learns the ID of a remote node /// Called whenever BASP learns the ID of a remote node.
/// to which it does not have a direct connection. virtual void learned_new_node(const node_id& nid) = 0;
virtual void learned_new_node_directly(const node_id& nid,
bool was_known_indirectly) = 0;
/// Called whenever BASP learns the ID of a remote node /// Get contact information for `nid` and establish communication.
/// to which it does not have a direct connection. virtual void establish_communication(const node_id& nid) = 0;
virtual void learned_new_node_indirectly(const node_id& nid) = 0;
/// Called if a heartbeat was received from `nid` /// Called if a heartbeat was received from `nid`
virtual void handle_heartbeat(const node_id& nid) = 0; virtual void handle_heartbeat(const node_id& nid) = 0;
...@@ -130,6 +129,16 @@ public: ...@@ -130,6 +129,16 @@ public:
/// Drop pending messages with sequence number `seq`. /// Drop pending messages with sequence number `seq`.
virtual void drop_pending(endpoint_context& ep, sequence_type seq) = 0; virtual void drop_pending(endpoint_context& ep, sequence_type seq) = 0;
/// Send messages that were buffered while connectivity establishment
/// was pending using `hdl`.
virtual void send_buffered_messages(execution_unit* ctx, node_id nid,
connection_handle hdl) = 0;
/// Send messages that were buffered while connectivity establishment
/// was pending using `hdl`.
virtual void send_buffered_messages(execution_unit* ctx, node_id nid,
datagram_handle hdl) = 0;
/// Returns a reference to the current sent buffer, dispatching the call /// Returns a reference to the current sent buffer, dispatching the call
/// based on the type contained in `hdl`. /// based on the type contained in `hdl`.
virtual buffer_type& get_buffer(endpoint_handle hdl) = 0; virtual buffer_type& get_buffer(endpoint_handle hdl) = 0;
...@@ -142,6 +151,11 @@ public: ...@@ -142,6 +151,11 @@ public:
/// Returns a reference to the sent buffer. /// Returns a reference to the sent buffer.
virtual buffer_type& get_buffer(connection_handle hdl) = 0; virtual buffer_type& get_buffer(connection_handle hdl) = 0;
/// Returns a reference to a buffer to be sent to node with `nid`.
/// If communication with the node is esstablished, it picks the first
/// available handle, otherwise a buffer for a pending message is returned.
virtual buffer_type& get_buffer(node_id nid) = 0;
/// Returns the buffer accessed through a call to `get_buffer` when /// Returns the buffer accessed through a call to `get_buffer` when
/// passing a datagram handle and removes it from the callee. /// passing a datagram handle and removes it from the callee.
virtual buffer_type pop_datagram_buffer(datagram_handle hdl) = 0; virtual buffer_type pop_datagram_buffer(datagram_handle hdl) = 0;
...@@ -181,15 +195,12 @@ public: ...@@ -181,15 +195,12 @@ public:
/// Sends heartbeat messages to all valid nodes those are directly connected. /// Sends heartbeat messages to all valid nodes those are directly connected.
void handle_heartbeat(execution_unit* ctx); void handle_heartbeat(execution_unit* ctx);
/// Returns a route to `target` or `none` on error. /// Flushes the underlying buffer of `hdl`.
optional<routing_table::route> lookup(const node_id& target); void flush(endpoint_handle hdl);
/// Flushes the underlying buffer of `path`. /// Sends a BASP message and implicitly flushes the output buffer of `hdl`.
void flush(const routing_table::route& path);
/// Sends a BASP message and implicitly flushes the output buffer of `r`.
/// This function will update `hdr.payload_len` if a payload was written. /// This function will update `hdr.payload_len` if a payload was written.
void write(execution_unit* ctx, const routing_table::route& r, void write(execution_unit* ctx, endpoint_handle hdl,
header& hdr, payload_writer* writer = nullptr); header& hdr, payload_writer* writer = nullptr);
/// Adds a new actor to the map of published actors. /// Adds a new actor to the map of published actors.
...@@ -254,18 +265,23 @@ public: ...@@ -254,18 +265,23 @@ public:
uint16_t sequence_number = 0); uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`. /// Writes the client handshake to `buf`.
static void write_client_handshake(execution_unit* ctx, void write_client_handshake(execution_unit* ctx,
buffer_type& buf, buffer_type& buf,
const node_id& remote_side, const node_id& remote_side,
const node_id& this_node, const node_id& this_node,
const std::string& app_identifier, const std::string& app_identifier,
uint16_t sequence_number = 0); uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`. /// Writes the client handshake to `buf`.
void write_client_handshake(execution_unit* ctx, void write_client_handshake(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side, buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0); uint16_t sequence_number = 0);
/// Writes the acknowledge handshake to finish a UDP handshake to `buf`.
void write_acknowledge_handshake(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0);
/// Writes an `announce_proxy` to `buf`. /// Writes an `announce_proxy` to `buf`.
void write_announce_proxy(execution_unit* ctx, buffer_type& buf, void write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid, const node_id& dest_node, actor_id aid,
...@@ -299,129 +315,147 @@ public: ...@@ -299,129 +315,147 @@ public:
bool handle(execution_unit* ctx, const Handle& hdl, header& hdr, bool handle(execution_unit* ctx, const Handle& hdl, header& hdr,
std::vector<char>* payload, bool tcp_based, std::vector<char>* payload, bool tcp_based,
optional<endpoint_context&> ep, optional<uint16_t> port) { optional<endpoint_context&> ep, optional<uint16_t> port) {
// function object for checking payload validity // Function object for checking payload validity.
auto payload_valid = [&]() -> bool { auto payload_valid = [&]() -> bool {
return payload != nullptr && payload->size() == hdr.payload_len; return payload != nullptr && payload->size() == hdr.payload_len;
}; };
// handle message to ourselves // Handle message to ourselves.
switch (hdr.operation) { switch (hdr.operation) {
case message_type::server_handshake: { case message_type::server_handshake: {
if (!payload_valid())
return false;
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
actor_id aid = invalid_actor_id; actor_id aid = invalid_actor_id;
std::set<std::string> sigs; std::set<std::string> sigs;
if (!payload_valid()) { basp::routing_table::address_map addrs;
CAF_LOG_ERROR("fail to receive the app identifier"); if (auto err = bd(remote_appid, aid, sigs, addrs)) {
CAF_LOG_ERROR("unable to deserialize payload:"
<< callee_.system().render(err));
return false; return false;
} else {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return false;
auto appid = get_if<std::string>(&callee_.config(),
"middleman.app-identifier");
if ((appid && *appid != remote_appid)
|| (!appid && !remote_appid.empty())) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
e = bd(aid, sigs);
if (e)
return false;
} }
// close self connection after handshake is done if (remote_appid != get_or(callee_.config(), "middleman.app-identifier",
defaults::middleman::app_identifier)) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
// Close self connection after handshake is done.
if (hdr.source_node == this_node_) { if (hdr.source_node == this_node_) {
CAF_LOG_DEBUG("close connection to self immediately"); CAF_LOG_DEBUG("close connection to self immediately");
callee_.finalize_handshake(hdr.source_node, aid, sigs); callee_.finalize_handshake(hdr.source_node, aid, sigs);
return false; return false;
} }
// close this connection if we already have a direct connection // Close this connection if we already established communication.
if (tbl_.lookup_direct(hdr.source_node)) { auto lr = tbl_.lookup(hdr.source_node);
CAF_LOG_DEBUG("close connection since we already have a " // TODO: Anything additional or different for UDP?
"direct connection: " << CAF_ARG(hdr.source_node)); if (lr.hdl) {
CAF_LOG_INFO("close redundant" << CAF_ARG(hdr.source_node));
callee_.finalize_handshake(hdr.source_node, aid, sigs); callee_.finalize_handshake(hdr.source_node, aid, sigs);
return false; return false;
} } else if (!tcp_based && (tbl_.received_client_handshake(hdr.source_node)
// add direct route to this node and remove any indirect entry && this_node() < hdr.source_node)) {
CAF_LOG_DEBUG("new direct connection:" << CAF_ARG(hdr.source_node)); CAF_LOG_INFO("simultaneous handshake, let the other node proceed: "
tbl_.add_direct(hdl, hdr.source_node); << CAF_ARG(hdr.source_node));
auto was_indirect = tbl_.erase_indirect(hdr.source_node); callee_.finalize_handshake(hdr.source_node, aid, sigs);
// write handshake as client in response
auto path = tbl_.lookup(hdr.source_node);
if (!path) {
CAF_LOG_ERROR("no route to host after server handshake");
return false; return false;
} }
// Add this node to our contacts.
CAF_LOG_INFO("new endpoint:" << CAF_ARG(hdr.source_node));
if (lr.known)
tbl_.handle(hdr.source_node, hdl);
else
tbl_.add(hdr.source_node, hdl);
auto peer_server = system().registry().get(atom("PeerServ"));
anon_send(actor_cast<actor>(peer_server), put_atom::value,
to_string(hdr.source_node), make_message(addrs));
// Write handshake as client in response.
if (tcp_based) { if (tcp_based) {
auto ch = get<connection_handle>(path->hdl); write_client_handshake(ctx, callee_.get_buffer(hdl), hdr.source_node);
write_client_handshake(ctx, callee_.get_buffer(ch), } else {
hdr.source_node); auto seq = ep->requires_ordering ? ep->seq_outgoing++ : 0;
write_acknowledge_handshake(ctx, callee_.get_buffer(hdl),
hdr.source_node, seq);
} }
callee_.learned_new_node_directly(hdr.source_node, was_indirect); flush(hdl);
callee_.learned_new_node(hdr.source_node);
callee_.finalize_handshake(hdr.source_node, aid, sigs); callee_.finalize_handshake(hdr.source_node, aid, sigs);
flush(*path); callee_.send_buffered_messages(ctx, hdr.source_node, hdl);
break; break;
} }
case message_type::client_handshake: { case message_type::client_handshake: {
if (!payload_valid()) { if (!payload_valid()) {
CAF_LOG_ERROR("fail to receive the app identifier"); CAF_LOG_ERROR("fail to receive the app identifier");
return false; return false;
} else {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return false;
auto appid = get_if<std::string>(&callee_.config(),
"middleman.app-identifier");
if ((appid && *appid != remote_appid)
|| (!appid && !remote_appid.empty())) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
} }
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
basp::routing_table::address_map addrs;
if (auto err = bd(remote_appid, addrs)) {
CAF_LOG_ERROR("unable to deserialize payload:"
<< callee_.system().render(err));
return false;
}
if (remote_appid != get_or(callee_.config(), "middleman.app-identifier",
defaults::middleman::app_identifier)) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
// Handshakes were only exchanged if `hdl` is set.
auto lr = tbl_.lookup(hdr.source_node);
auto new_node = (this_node() != hdr.source_node && !lr.hdl);
if (tcp_based) { if (tcp_based) {
if (tbl_.lookup_direct(hdr.source_node)) { if (!new_node) {
CAF_LOG_DEBUG("received second client handshake:" CAF_LOG_DEBUG("received second client handshake:"
<< CAF_ARG(hdr.source_node)); << CAF_ARG(hdr.source_node));
break; break;
} }
// add direct route to this node and remove any indirect entry // Add this node to our contacts.
CAF_LOG_DEBUG("new direct connection:" << CAF_ARG(hdr.source_node)); CAF_LOG_DEBUG("new endpoint:" << CAF_ARG(hdr.source_node));
tbl_.add_direct(hdl, hdr.source_node); // Either add a new node or add the handle to a known one.
auto was_indirect = tbl_.erase_indirect(hdr.source_node); if (lr.known)
callee_.learned_new_node_directly(hdr.source_node, was_indirect); tbl_.handle(hdr.source_node, hdl);
else
tbl_.add(hdr.source_node, hdl);
callee_.learned_new_node(hdr.source_node);
callee_.send_buffered_messages(ctx, hdr.source_node, hdl);
} else { } else {
auto new_node = (this_node() != hdr.source_node if (!lr.known)
&& !tbl_.lookup_direct(hdr.source_node)); tbl_.add(hdr.source_node);
if (new_node) { tbl_.received_client_handshake(hdr.source_node, true);
// add direct route to this node and remove any indirect entry uint16_t seq = ep->requires_ordering ? ep->seq_outgoing++ : 0;
CAF_LOG_DEBUG("new direct connection:" << CAF_ARG(hdr.source_node)); write_server_handshake(ctx, callee_.get_buffer(hdl), port, seq);
tbl_.add_direct(hdl, hdr.source_node);
}
uint16_t seq = (ep && ep->requires_ordering) ? ep->seq_outgoing++ : 0;
write_server_handshake(ctx,
callee_.get_buffer(hdl),
port, seq);
callee_.flush(hdl); callee_.flush(hdl);
if (new_node) {
auto was_indirect = tbl_.erase_indirect(hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
}
} }
auto peer_server = system().registry().get(atom("PeerServ"));
anon_send(actor_cast<actor>(peer_server), put_atom::value,
to_string(hdr.source_node), make_message(addrs));
break;
}
case message_type::acknowledge_handshake: {
if (tcp_based) {
CAF_LOG_ERROR("received acknowledge handshake via tcp");
break;
}
auto lr = tbl_.lookup(hdr.source_node);
if (!lr.known) {
CAF_LOG_DEBUG("dropping acknowledge handshake from unknown node");
break;
}
if (lr.hdl) {
CAF_LOG_DEBUG("dropping repeated acknowledge handshake");
// TODO: Or should we just adopt the new handle?
break;
}
CAF_LOG_INFO("new endpoint:" << CAF_ARG(hdr.source_node));
tbl_.handle(hdr.source_node, hdl);
tbl_.received_client_handshake(hdr.source_node, false);
callee_.learned_new_node(hdr.source_node);
callee_.send_buffered_messages(ctx, hdr.source_node, hdl);
break; break;
} }
case message_type::dispatch_message: { case message_type::dispatch_message: {
if (!payload_valid()) if (!payload_valid())
return false; return false;
// in case the sender of this message was received via a third node,
// we assume that that node to offers a route to the original source
auto last_hop = tbl_.lookup_direct(hdl);
if (hdr.source_node != none
&& hdr.source_node != this_node_
&& last_hop != hdr.source_node
&& !tbl_.lookup_direct(hdr.source_node)
&& tbl_.add_indirect(last_hop, hdr.source_node))
callee_.learned_new_node_indirectly(hdr.source_node);
binary_deserializer bd{ctx, *payload}; binary_deserializer bd{ctx, *payload};
auto receiver_name = static_cast<atom_value>(0); auto receiver_name = static_cast<atom_value>(0);
std::vector<strong_actor_ptr> forwarding_stack; std::vector<strong_actor_ptr> forwarding_stack;
......
...@@ -67,6 +67,10 @@ enum class message_type : uint8_t { ...@@ -67,6 +67,10 @@ enum class message_type : uint8_t {
/// ///
/// ![](heartbeat.png) /// ![](heartbeat.png)
heartbeat = 0x05, heartbeat = 0x05,
/// Last message in a BASP handshake for between two endpoints using UDP
/// as a transport protocol.
acknowledge_handshake = 0x06,
}; };
/// @relates message_type /// @relates message_type
......
...@@ -29,6 +29,8 @@ ...@@ -29,6 +29,8 @@
#include "caf/io/basp/buffer_type.hpp" #include "caf/io/basp/buffer_type.hpp"
#include "caf/io/network/interfaces.hpp"
namespace std { namespace std {
template<> template<>
...@@ -49,73 +51,106 @@ namespace basp { ...@@ -49,73 +51,106 @@ namespace basp {
/// Stores routing information for a single broker participating as /// Stores routing information for a single broker participating as
/// BASP peer and provides both direct and indirect paths. /// BASP peer and provides both direct and indirect paths.
// TODO: Rename `routing_table`.
class routing_table { class routing_table {
public: public:
using endpoint_handle = variant<connection_handle, datagram_handle>; using endpoint_handle = variant<connection_handle, datagram_handle>;
using address_endpoint = std::pair<uint16_t, network::address_listing>;
using address_map = std::map<network::protocol::transport,
address_endpoint>;
explicit routing_table(abstract_broker* parent); explicit routing_table(abstract_broker* parent);
virtual ~routing_table(); virtual ~routing_table();
/// Describes a routing path to a node. /// Result for a lookup of a node.
struct route { struct lookup_result {
const node_id& next_hop; /// Tracks whether the node is already known.
endpoint_handle hdl; bool known;
/// Servant handle to communicate with the node -- if already created.
optional<endpoint_handle> hdl;
}; };
/// Describes a function object for erase operations that /// Describes a function object for erase operations that
/// is called for each indirectly lost connection. /// is called for each indirectly lost connection.
using erase_callback = callback<const node_id&>; using erase_callback = callback<const node_id&>;
/// Returns a route to `target` or `none` on error. /// Returns the ID of the peer reachable via `hdl` or
optional<route> lookup(const node_id& target);
/// Returns the ID of the peer connected via `hdl` or
/// `none` if `hdl` is unknown. /// `none` if `hdl` is unknown.
node_id lookup_direct(const endpoint_handle& hdl) const; node_id lookup(const endpoint_handle& hdl) const;
/// Returns the handle offering a direct connection to `nid` or
/// `invalid_connection_handle` if no direct connection to `nid` exists.
optional<endpoint_handle> lookup_direct(const node_id& nid) const;
/// Returns the next hop that would be chosen for `nid` /// Returns the state for communication with `nid` along with a handle
/// or `none` if there's no indirect route to `nid`. /// if communication is established or `none` if `nid` is unknown.
node_id lookup_indirect(const node_id& nid) const; lookup_result lookup(const node_id& nid) const;
/// Adds a new direct route to the table. /// Adds a new endpoint to the table.
/// @pre `hdl != invalid_connection_handle && nid != none` /// @pre `hdl != invalid_connection_handle && nid != none`
void add_direct(const endpoint_handle& hdl, const node_id& nid); void add(const node_id& nid, const endpoint_handle& hdl);
/// Adds a new indirect route to the table. /// Add a new endpoint to the table.
bool add_indirect(const node_id& hop, const node_id& dest); /// @pre `origin != none && nid != none`
void add(const node_id& nid, const node_id& origin);
/// Blacklist the route to `dest` via `hop`. /// Adds a new endpoint to the table that has no attached information.
void blacklist(const node_id& hop, const node_id& dest); /// that propagated information about the node.
/// @pre `nid != none`
void add(const node_id& nid);
/// Removes a direct connection and calls `cb` for any node /// Removes a direct connection and calls `cb` for any node
/// that became unreachable as a result of this operation, /// that became unreachable as a result of this operation,
/// including the node that is assigned as direct path for `hdl`. /// including the node that is assigned as direct path for `hdl`.
void erase_direct(const endpoint_handle& hdl, erase_callback& cb); void erase(const endpoint_handle& hdl, erase_callback& cb);
/// Removes any entry for indirect connection to `dest` and returns /// Queries whether `dest` is reachable directly.
/// `true` if `dest` had an indirect route, otherwise `false`.
bool erase_indirect(const node_id& dest);
/// Queries whether `dest` is reachable.
bool reachable(const node_id& dest); bool reachable(const node_id& dest);
/// Removes all direct and indirect routes to `dest` and calls
/// `cb` for any node that became unreachable as a result of this
/// operation, including `dest`.
/// @returns the number of removed routes (direct and indirect)
size_t erase(const node_id& dest, erase_callback& cb);
/// Returns the parent broker. /// Returns the parent broker.
inline abstract_broker* parent() { inline abstract_broker* parent() {
return parent_; return parent_;
} }
/// Set the forwarding node that first mentioned `hdl`.
bool origin(const node_id& nid, const node_id& origin);
/// Get the forwarding node that first mentioned `hdl`
/// or `none` if the node is unknown.
optional<node_id> origin(const node_id& nid);
/// Set the handle for communication with `nid`.
bool handle(const node_id& nid, const endpoint_handle& hdl);
/// Get the handle for communication with `nid`
/// or `none` if the node is unknown.
optional<endpoint_handle> handle(const node_id& nid);
/// Get the addresses to reach `nid` or `none` if the node is unknown.
optional<const address_map&> addresses(const node_id& nid);
/// Add a port, address pair under a key to the local addresses.
void local_addresses(network::protocol::transport proto,
address_endpoint addrs);
/// Get a reference to an address map for the local node.
const address_map& local_addresses();
// Set the received client handshake flag for `nid`.
bool received_client_handshake(const node_id& nid, bool flag);
// Get the received client handshake flag for `nid`.
bool received_client_handshake(const node_id& nid);
public: public:
/// Entry to bundle information for a remote endpoint.
struct node_info {
/// Handle for the node if communication is established.
optional<endpoint_handle> hdl;
/// The endpoint who told us about the node.
optional<node_id> origin;
/// Track if we received a client handshake to solve simultaneous
/// handshake with UDP.
bool received_client_handshake;
};
template <class Map, class Fallback> template <class Map, class Fallback>
typename Map::mapped_type typename Map::mapped_type
get_opt(const Map& m, const typename Map::key_type& k, Fallback&& x) const { get_opt(const Map& m, const typename Map::key_type& k, Fallback&& x) const {
...@@ -125,16 +160,10 @@ public: ...@@ -125,16 +160,10 @@ public:
return std::forward<Fallback>(x); return std::forward<Fallback>(x);
} }
using node_id_set = std::unordered_set<node_id>;
using indirect_entries = std::unordered_map<node_id, // dest
node_id_set>; // hop
abstract_broker* parent_; abstract_broker* parent_;
std::unordered_map<endpoint_handle, node_id> direct_by_hdl_; std::unordered_map<endpoint_handle, node_id> nid_by_hdl_;
std::unordered_map<node_id, endpoint_handle> direct_by_nid_; std::unordered_map<node_id, node_info> node_information_base_;
indirect_entries indirect_; address_map local_addrs_;
indirect_entries blacklist_;
}; };
/// @} /// @}
......
...@@ -80,14 +80,10 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -80,14 +80,10 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
std::vector<strong_actor_ptr>& stages, message& msg); std::vector<strong_actor_ptr>& stages, message& msg);
// performs bookkeeping such as managing `spawn_servers` // performs bookkeeping such as managing `spawn_servers`
void learned_new_node(const node_id& nid); void learned_new_node(const node_id& nid) override;
// inherited from basp::instance::callee // get contact information for `nid` and establish communication
void learned_new_node_directly(const node_id& nid, void establish_communication(const node_id& nid) override;
bool was_indirectly_before) override;
// inherited from basp::instance::callee
void learned_new_node_indirectly(const node_id& nid) override;
// inherited from basp::instance::callee // inherited from basp::instance::callee
uint16_t next_sequence_number(connection_handle hdl) override; uint16_t next_sequence_number(connection_handle hdl) override;
...@@ -107,6 +103,14 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -107,6 +103,14 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// inherited from basp::instance::callee // inherited from basp::instance::callee
void drop_pending(basp::endpoint_context& ep, uint16_t seq) override; void drop_pending(basp::endpoint_context& ep, uint16_t seq) override;
// inherited from basp::instance::callee
void send_buffered_messages(execution_unit* ctx, node_id nid,
connection_handle hdl) override;
// inherited from basp::instance::callee
void send_buffered_messages(execution_unit* ctx, node_id nid,
datagram_handle hdl) override;
// inherited from basp::instance::callee // inherited from basp::instance::callee
buffer_type& get_buffer(endpoint_handle hdl) override; buffer_type& get_buffer(endpoint_handle hdl) override;
...@@ -116,6 +120,9 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -116,6 +120,9 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// inherited from basp::instance::callee // inherited from basp::instance::callee
buffer_type& get_buffer(connection_handle hdl) override; buffer_type& get_buffer(connection_handle hdl) override;
// inherited from basp::instance::callee
buffer_type& get_buffer(node_id nid) override;
// inherited from basp::instance::callee // inherited from basp::instance::callee
buffer_type pop_datagram_buffer(datagram_handle hdl) override; buffer_type pop_datagram_buffer(datagram_handle hdl) override;
...@@ -180,8 +187,12 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -180,8 +187,12 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// maximum queue size for pending messages of endpoints with ordering // maximum queue size for pending messages of endpoints with ordering
const size_t max_pending_messages; const size_t max_pending_messages;
// buffer messages for nodes while connectivity is established
std::unordered_map<node_id, std::vector<buffer_type>> pending_connectivity;
// timeout for delivery of pending messages of endpoints with ordering // timeout for delivery of pending messages of endpoints with ordering
const std::chrono::milliseconds pending_to = std::chrono::milliseconds(100); const std::chrono::milliseconds pending_timeout
= std::chrono::milliseconds(100);
// returns the node identifier of the underlying BASP instance // returns the node identifier of the underlying BASP instance
const node_id& this_node() const { const node_id& this_node() const {
......
...@@ -46,13 +46,13 @@ struct connection_helper_state { ...@@ -46,13 +46,13 @@ struct connection_helper_state {
static const char* name; static const char* name;
}; };
behavior datagram_connection_broker(broker* self, behavior datagram_connection_broker(broker* self, uint16_t port,
uint16_t port,
network::address_listing addresses, network::address_listing addresses,
actor system_broker); actor system_broker,
basp::instance* instance);
behavior connection_helper(stateful_actor<connection_helper_state>* self, behavior connection_helper(stateful_actor<connection_helper_state>* self,
actor b); actor b, basp::instance* i);
} // namespace io } // namespace io
} // namespace caf } // namespace caf
...@@ -254,7 +254,7 @@ private: ...@@ -254,7 +254,7 @@ private:
std::vector<intrusive_ptr<resumable>> internally_posted_; std::vector<intrusive_ptr<resumable>> internally_posted_;
/// Sequential ids for handles of datagram servants /// Sequential ids for handles of datagram servants
int64_t servant_ids_; std::atomic<int64_t> servant_ids_;
/// Maximum messages per resume run. /// Maximum messages per resume run.
size_t max_throughput_; size_t max_throughput_;
......
...@@ -32,6 +32,7 @@ namespace network { ...@@ -32,6 +32,7 @@ namespace network {
class test_multiplexer : public multiplexer { class test_multiplexer : public multiplexer {
private: private:
struct datagram_endpoint;
struct datagram_data; struct datagram_data;
public: public:
...@@ -96,10 +97,13 @@ public: ...@@ -96,10 +97,13 @@ public:
void provide_acceptor(uint16_t desired_port, accept_handle hdl); void provide_acceptor(uint16_t desired_port, accept_handle hdl);
// Provide a local datagram servant on port with `hdl`.
void provide_datagram_servant(uint16_t desired_port, datagram_handle hdl); void provide_datagram_servant(uint16_t desired_port, datagram_handle hdl);
// Provide a datagram servant for communication a remote endpoint
// with `endpoint_id`.
void provide_datagram_servant(std::string host, uint16_t desired_port, void provide_datagram_servant(std::string host, uint16_t desired_port,
datagram_handle hdl); datagram_handle hdl, intptr_t endpoint_id = 0);
/// Generate an id for a new servant. /// Generate an id for a new servant.
int64_t next_endpoint_id(); int64_t next_endpoint_id();
...@@ -108,10 +112,11 @@ public: ...@@ -108,10 +112,11 @@ public:
using buffer_type = std::vector<char>; using buffer_type = std::vector<char>;
/// Buffers storing bytes for UDP related components. /// Buffers storing bytes for UDP related components.
using endpoint_id_type = std::intptr_t;
using read_buffer_type = network::receive_buffer; using read_buffer_type = network::receive_buffer;
using write_buffer_type = buffer_type; using write_buffer_type = buffer_type;
using read_job_type = std::pair<datagram_handle, read_buffer_type>; using read_job_type = std::pair<endpoint_id_type, read_buffer_type>;
using write_job_type = std::pair<datagram_handle, write_buffer_type>; using write_job_type = std::pair<endpoint_id_type, write_buffer_type>;
using write_job_queue_type = std::deque<write_job_type>; using write_job_queue_type = std::deque<write_job_type>;
using shared_buffer_type = std::shared_ptr<buffer_type>; using shared_buffer_type = std::shared_ptr<buffer_type>;
...@@ -176,8 +181,10 @@ public: ...@@ -176,8 +181,10 @@ public:
datagram_servant_ptr& impl_ptr(datagram_handle hdl); datagram_servant_ptr& impl_ptr(datagram_handle hdl);
/// Returns a map with all servants related to the servant `hdl`. using write_handle_map = std::map<endpoint_id_type, datagram_endpoint>;
std::set<datagram_handle>& servants(datagram_handle hdl);
/// Returns the endpoint for a specific `hdl`.
endpoint_id_type endpoint_id(datagram_handle hdl);
/// Returns `true` if this handle has been closed /// Returns `true` if this handle has been closed
/// for reading, `false` otherwise. /// for reading, `false` otherwise.
...@@ -198,15 +205,27 @@ public: ...@@ -198,15 +205,27 @@ public:
test_multiplexer& peer, std::string host, test_multiplexer& peer, std::string host,
uint16_t port, connection_handle peer_hdl); uint16_t port, connection_handle peer_hdl);
/// Stores `hdl` as a pending endpoint for `src`. void add_pending_endpoint(intptr_t endpoint_id, datagram_handle hdl,
void add_pending_endpoint(datagram_handle src, datagram_handle hdl); shared_job_queue_type write_buffer
= std::make_shared<write_job_queue_type>());
/// Add `hdl` as a pending endpoint to `src` and provide a datagram servant
/// on `peer` that connects the buffers of `hdl` and `peer_hdl`. Calls
/// `add_pending_endpoint(...)` and `peer.provide_endpoint(...)`.
void prepare_endpoints(datagram_handle src, datagram_handle hdl,
test_multiplexer& peer, std::string host,
uint16_t port, datagram_handle peer_hdl);
using pending_connects_map = std::unordered_multimap<accept_handle, using pending_connects_map = std::unordered_multimap<accept_handle,
connection_handle>; connection_handle>;
pending_connects_map& pending_connects(); pending_connects_map& pending_connects();
using pending_endpoints_map = std::unordered_map<int64_t, datagram_handle>; // using pending_endpoints_map = std::unordered_map<int64_t, datagram_handle>;
using pending_endpoints_map = std::map<intptr_t,
std::pair<datagram_handle,
shared_job_queue_type>>;
pending_endpoints_map& pending_endpoints(); pending_endpoints_map& pending_endpoints();
...@@ -219,7 +238,7 @@ public: ...@@ -219,7 +238,7 @@ public:
datagram_handle>; datagram_handle>;
using pending_remote_datagram_endpoints_map using pending_remote_datagram_endpoints_map
= std::map<std::pair<std::string, uint16_t>, datagram_handle>; = std::map<std::pair<std::string, uint16_t>, std::pair<datagram_handle, intptr_t>>;
bool has_pending_scribe(std::string x, uint16_t y); bool has_pending_scribe(std::string x, uint16_t y);
...@@ -237,6 +256,9 @@ public: ...@@ -237,6 +256,9 @@ public:
/// Tries to read data from the external input buffer of `hdl`. /// Tries to read data from the external input buffer of `hdl`.
bool try_read_data(connection_handle hdl); bool try_read_data(connection_handle hdl);
/// Tries to read data from the external input queue of `hdl`.
bool try_read_data(datagram_handle hdl);
/// Poll data on all scribes. /// Poll data on all scribes.
bool read_data(); bool read_data();
...@@ -253,7 +275,7 @@ public: ...@@ -253,7 +275,7 @@ public:
/// Appends `buf` to the virtual network buffer of `hdl` /// Appends `buf` to the virtual network buffer of `hdl`
/// and calls `read_data(hdl)` afterwards. /// and calls `read_data(hdl)` afterwards.
void virtual_send(datagram_handle src, datagram_handle ep, void virtual_send(datagram_handle hdl, endpoint_id_type endpoint,
const buffer_type&); const buffer_type&);
/// Waits until a `runnable` is available and executes it. /// Waits until a `runnable` is available and executes it.
...@@ -323,24 +345,36 @@ private: ...@@ -323,24 +345,36 @@ private:
doorman_data(); doorman_data();
}; };
struct datagram_data { struct datagram_endpoint {
datagram_handle hdl;
shared_job_queue_type vn_buf_ptr; shared_job_queue_type vn_buf_ptr;
shared_job_queue_type wr_buf_ptr; shared_job_queue_type wr_buf_ptr;
write_job_queue_type& vn_buf; write_job_queue_type& vn_buf;
write_job_queue_type& wr_buf; write_job_queue_type& wr_buf;
read_job_type rd_buf;
datagram_endpoint(
datagram_handle hdl = datagram_handle(),
shared_job_queue_type input = std::make_shared<write_job_queue_type>(),
shared_job_queue_type output = std::make_shared<write_job_queue_type>()
);
};
struct datagram_data {
datagram_servant_ptr ptr; datagram_servant_ptr ptr;
datagram_endpoint read_handle;
read_job_type rd_buf;
bool stopped_reading; bool stopped_reading;
bool passive_mode; bool passive_mode;
bool ack_writes; bool ack_writes;
uint16_t port; uint16_t remote_port;
uint16_t local_port; uint16_t local_port;
std::set<datagram_handle> servants; write_handle_map write_handles;
size_t datagram_size; size_t datagram_size;
// Allows creating an entangled scribes where the input of this scribe is // Allows creating an entangled scribes where the input of this scribe is
// the output of another scribe and vice versa. // the output of another scribe and vice versa.
datagram_data( datagram_data(
datagram_handle hdl = datagram_handle{},
shared_job_queue_type input = std::make_shared<write_job_queue_type>(), shared_job_queue_type input = std::make_shared<write_job_queue_type>(),
shared_job_queue_type output = std::make_shared<write_job_queue_type>() shared_job_queue_type output = std::make_shared<write_job_queue_type>()
); );
......
...@@ -211,6 +211,7 @@ void abstract_broker::add_datagram_servant(datagram_servant_ptr ptr) { ...@@ -211,6 +211,7 @@ void abstract_broker::add_datagram_servant(datagram_servant_ptr ptr) {
ptr->set_parent(this); ptr->set_parent(this);
auto hdls = ptr->hdls(); auto hdls = ptr->hdls();
launch_servant(ptr); launch_servant(ptr);
add_hdl_for_datagram_servant(ptr, ptr->hdl());
for (auto& hdl : hdls) for (auto& hdl : hdls)
add_hdl_for_datagram_servant(ptr, hdl); add_hdl_for_datagram_servant(ptr, hdl);
} }
...@@ -275,6 +276,7 @@ void abstract_broker::move_datagram_servant(datagram_servant_ptr ptr) { ...@@ -275,6 +276,7 @@ void abstract_broker::move_datagram_servant(datagram_servant_ptr ptr) {
CAF_ASSERT(ptr != nullptr); CAF_ASSERT(ptr != nullptr);
CAF_ASSERT(ptr->parent() != nullptr && ptr->parent() != this); CAF_ASSERT(ptr->parent() != nullptr && ptr->parent() != this);
ptr->set_parent(this); ptr->set_parent(this);
ptr->add_to_loop();
CAF_ASSERT(ptr->parent() == this); CAF_ASSERT(ptr->parent() == this);
auto hdls = ptr->hdls(); auto hdls = ptr->hdls();
for (auto& hdl : hdls) for (auto& hdl : hdls)
......
...@@ -41,7 +41,7 @@ namespace io { ...@@ -41,7 +41,7 @@ namespace io {
namespace { namespace {
// visitors to access handle variant of the context // Visitors to access handle variant of the context.
struct seq_num_visitor { struct seq_num_visitor {
using result_type = basp::sequence_type; using result_type = basp::sequence_type;
seq_num_visitor(basp_broker_state* ptr) : state(ptr) { } seq_num_visitor(basp_broker_state* ptr) : state(ptr) { }
...@@ -73,7 +73,7 @@ basp_broker_state::basp_broker_state(broker* selfptr) ...@@ -73,7 +73,7 @@ basp_broker_state::basp_broker_state(broker* selfptr)
} }
basp_broker_state::~basp_broker_state() { basp_broker_state::~basp_broker_state() {
// make sure all spawn servers are down // Make sure all spawn servers are down.
for (auto& kvp : spawn_servers) for (auto& kvp : spawn_servers)
anon_send_exit(kvp.second, exit_reason::kill); anon_send_exit(kvp.second, exit_reason::kill);
} }
...@@ -83,25 +83,34 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) { ...@@ -83,25 +83,34 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
CAF_ASSERT(nid != this_node()); CAF_ASSERT(nid != this_node());
if (nid == none || aid == invalid_actor_id) if (nid == none || aid == invalid_actor_id)
return nullptr; return nullptr;
// this member function is being called whenever we deserialize a // This member function is being called whenever we deserialize a
// payload received from a remote node; if a remote node A sends // payload received from a remote node; if a remote node A sends
// us a handle to a third node B, then we assume that A offers a route to B // us a handle to a third node B, then we assume that A can tell us
if (nid != this_context->id // how to contact B.
&& !instance.tbl().lookup_direct(nid) // TODO: This should probably happen somewhere else, but is usually only
&& instance.tbl().add_indirect(this_context->id, nid)) // performed in `finalize_handshake` which is only called on receipt
learned_new_node_indirectly(nid); // of server handshakes.
// we need to tell remote side we are watching this actor now; if (this_context->id == none)
// use a direct route if possible, i.e., when talking to a third node this_context->id = instance.tbl().lookup(this_context->hdl);
auto path = instance.tbl().lookup(nid); auto lr = instance.tbl().lookup(nid);
if (!path) { if (nid != this_context->id && !lr.known) {
// this happens if and only if we don't have a path to `nid` instance.tbl().add(nid, this_context->id);
// and current_context_->hdl has been blacklisted establish_communication(nid);
CAF_LOG_DEBUG("cannot create a proxy instance for an actor " }
"running on a node we don't have a route to"); // We need to tell remote side we are watching this actor now;
// use a direct route if possible, i.e., when talking to a third node.
// TODO: Communication setup might still be in progress.
/*
if (lr.known && !lr.hdl) {
// This happens if and only if we don't have a path to `nid`
// and current_context_->hdl has been blacklisted.
CAF_LOG_INFO("cannot create a proxy instance for an actor "
"running on a node we don't have a route to");
return nullptr; return nullptr;
} }
// create proxy and add functor that will be called if we */
// receive a kill_proxy_instance message // Create proxy and add functor that will be called if we
// receive a kill_proxy_instance message.
auto mm = &system().middleman(); auto mm = &system().middleman();
actor_config cfg; actor_config cfg;
auto res = make_actor<forwarding_actor_proxy, strong_actor_ptr>( auto res = make_actor<forwarding_actor_proxy, strong_actor_ptr>(
...@@ -109,24 +118,31 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) { ...@@ -109,24 +118,31 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
strong_actor_ptr selfptr{self->ctrl()}; strong_actor_ptr selfptr{self->ctrl()};
res->get()->attach_functor([=](const error& rsn) { res->get()->attach_functor([=](const error& rsn) {
mm->backend().post([=] { mm->backend().post([=] {
// using res->id() instead of aid keeps this actor instance alive // Using res->id() instead of aid keeps this actor instance alive
// until the original instance terminates, thus preventing subtle // until the original instance terminates, thus preventing subtle
// bugs with attachables // bugs with attachables.
auto bptr = static_cast<basp_broker*>(selfptr->get()); auto bptr = static_cast<basp_broker*>(selfptr->get());
if (!bptr->getf(abstract_actor::is_terminated_flag)) if (!bptr->getf(abstract_actor::is_terminated_flag))
bptr->state.proxies().erase(nid, res->id(), rsn); bptr->state.proxies().erase(nid, res->id(), rsn);
}); });
}); });
CAF_LOG_DEBUG("successfully created proxy instance, " CAF_LOG_DEBUG("successfully created proxy instance, "
"write announce_proxy_instance:" "write announce_proxy_instance:"
<< CAF_ARG(nid) << CAF_ARG(aid)); << CAF_ARG(nid) << CAF_ARG(aid));
auto& ctx = *this_context; // TODO: Can it happen that things have changed here?
// tell remote side we are monitoring this actor now lr = instance.tbl().lookup(nid);
instance.write_announce_proxy(self->context(), if (lr.hdl) {
get_buffer(this_context->hdl), // Tell remote side we are monitoring this actor now.
nid, aid, instance.write_announce_proxy(self->context(),
ctx.requires_ordering ? ctx.seq_outgoing++ : 0); get_buffer(*lr.hdl),
instance.flush(*path); nid, aid,
visit(seq_num_visitor{this}, *lr.hdl));
flush(*lr.hdl);
} else {
instance.write_announce_proxy(self->context(),
get_buffer(nid),
nid, aid, 0);
}
mm->notify<hook::new_remote_actor>(res); mm->notify<hook::new_remote_actor>(res);
return res; return res;
} }
...@@ -144,10 +160,10 @@ void basp_broker_state::finalize_handshake(const node_id& nid, actor_id aid, ...@@ -144,10 +160,10 @@ void basp_broker_state::finalize_handshake(const node_id& nid, actor_id aid,
if (cb == none) if (cb == none)
return; return;
strong_actor_ptr ptr; strong_actor_ptr ptr;
// aid can be invalid when connecting to the default port of a node // The aid can be invalid when connecting to the default port of a node.
if (aid != invalid_actor_id) { if (aid != invalid_actor_id) {
if (nid == this_node()) { if (nid == this_node()) {
// connected to self // Connected to self.
ptr = actor_cast<strong_actor_ptr>(system().registry().get(aid)); ptr = actor_cast<strong_actor_ptr>(system().registry().get(aid));
CAF_LOG_DEBUG_IF(!ptr, "actor not found:" << CAF_ARG(aid)); CAF_LOG_DEBUG_IF(!ptr, "actor not found:" << CAF_ARG(aid));
} else { } else {
...@@ -171,26 +187,33 @@ void basp_broker_state::purge_state(const node_id& nid) { ...@@ -171,26 +187,33 @@ void basp_broker_state::purge_state(const node_id& nid) {
void basp_broker_state::send_kill_proxy_instance(const node_id& nid, void basp_broker_state::send_kill_proxy_instance(const node_id& nid,
actor_id aid, error rsn) { actor_id aid, error rsn) {
CAF_LOG_TRACE(CAF_ARG(nid) << CAF_ARG(aid) << CAF_ARG(rsn)); CAF_LOG_TRACE(CAF_ARG(nid) << CAF_ARG(aid) << CAF_ARG(rsn));
auto path = instance.tbl().lookup(nid); if (rsn == none)
if (!path) { rsn = exit_reason::unknown;
CAF_LOG_INFO("cannot send exit message for proxy, no route to host:" auto res = instance.tbl().lookup(nid);
<< CAF_ARG(nid)); if (!res.known) {
CAF_LOG_DEBUG("no host to send exit message:" << CAF_ARG(nid));
return; return;
} }
instance.write_kill_proxy(self->context(), if (res.hdl) {
get_buffer(path->hdl), auto hdl = std::move(*res.hdl);
nid, aid, rsn, instance.write_kill_proxy(self->context(),
visit(seq_num_visitor{this}, path->hdl)); get_buffer(hdl),
instance.flush(*path); nid, aid, rsn,
visit(seq_num_visitor{this}, hdl));
flush(hdl);
} else {
instance.write_kill_proxy(self->context(), get_buffer(nid),
nid, aid, rsn, 0);
}
} }
void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) { void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) {
CAF_LOG_TRACE(CAF_ARG(nid) << CAF_ARG(aid)); CAF_LOG_TRACE(CAF_ARG(nid) << CAF_ARG(aid));
// source node has created a proxy for one of our actors // Source node has created a proxy for one of our actors.
auto ptr = system().registry().get(aid); auto ptr = system().registry().get(aid);
if (ptr == nullptr) { if (ptr == nullptr) {
CAF_LOG_DEBUG("kill proxy immediately"); CAF_LOG_DEBUG("kill proxy immediately");
// kill immediately if actor has already terminated // Kill immediately if actor has already terminated.
send_kill_proxy_instance(nid, aid, exit_reason::unknown); send_kill_proxy_instance(nid, aid, exit_reason::unknown);
} else { } else {
auto entry = ptr->address(); auto entry = ptr->address();
...@@ -312,12 +335,12 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -312,12 +335,12 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
} }
auto tmp = system().spawn<hidden>([=](event_based_actor* tself) -> behavior { auto tmp = system().spawn<hidden>([=](event_based_actor* tself) -> behavior {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
// terminate when receiving a down message // Terminate when receiving a down message.
tself->set_down_handler([=](down_msg& dm) { tself->set_down_handler([=](down_msg& dm) {
CAF_LOG_TRACE(CAF_ARG(dm)); CAF_LOG_TRACE(CAF_ARG(dm));
tself->quit(std::move(dm.reason)); tself->quit(std::move(dm.reason));
}); });
// skip messages until we receive the initial ok_atom // Skip messages until we receive the initial ok_atom.
tself->set_default_handler(skip); tself->set_default_handler(skip);
return { return {
[=](ok_atom, const std::string& /* key == "info" */, [=](ok_atom, const std::string& /* key == "info" */,
...@@ -354,76 +377,80 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -354,76 +377,80 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
auto msg = make_message(sys_atom::value, get_atom::value, "info"); auto msg = make_message(sys_atom::value, get_atom::value, "info");
return sink(name_atm, stages, msg); return sink(name_atm, stages, msg);
}); });
auto path = instance.tbl().lookup(nid); auto res = instance.tbl().lookup(nid);
if (!path) { if (!res.known) {
CAF_LOG_ERROR("learned_new_node called, but no route to nid"); CAF_LOG_ERROR("learned_new_node called, but no route to nid");
return; return;
} }
// send message to SpawnServ of remote node // send message to SpawnServ of remote node
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, make_message_id().integer_value(), this_node(), nid, 0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
tmp.id(), invalid_actor_id, 0}; // sequence number only available with connectivity
visit(seq_num_visitor{this}, path->hdl)}; if (res.hdl) {
// writing std::numeric_limits<actor_id>::max() is a hack to get auto hdl = std::move(*res.hdl);
// this send-to-named-actor feature working with older CAF releases hdr.sequence_number = visit(seq_num_visitor{this}, hdl);
instance.write(self->context(), get_buffer(path->hdl), // writing std::numeric_limits<actor_id>::max() is a hack to get
hdr, &writer); // this send-to-named-actor feature working with older CAF releases
instance.flush(*path); instance.write(self->context(), get_buffer(hdl), hdr, &writer);
} flush(hdl);
} else {
void basp_broker_state::learned_new_node_directly(const node_id& nid, instance.write(self->context(), get_buffer(nid), hdr, &writer);
bool was_indirectly_before) { }
CAF_ASSERT(this_context != nullptr);
CAF_LOG_TRACE(CAF_ARG(nid));
if (!was_indirectly_before)
learned_new_node(nid);
} }
void basp_broker_state::learned_new_node_indirectly(const node_id& nid) { void basp_broker_state::establish_communication(const node_id& nid) {
// TODO: Split this by functionality, address query & connecting?
CAF_ASSERT(this_context != nullptr); CAF_ASSERT(this_context != nullptr);
CAF_LOG_TRACE(CAF_ARG(nid)); CAF_LOG_TRACE(CAF_ARG(nid));
learned_new_node(nid);
if (!automatic_connections) if (!automatic_connections)
return; return;
// this member function gets only called once, after adding a new // this member function gets only called once, after adding a new
// indirect connection to the routing table; hence, spawning // indirect connection to the routing table; hence, spawning
// our helper here exactly once and there is no need to track // our helper here exactly once and there is no need to track
// in-flight connection requests // in-flight connection requests
auto path = instance.tbl().lookup(nid); if (instance.tbl().lookup(nid).hdl) {
if (!path) { CAF_LOG_ERROR("establish_communication called with established connection");
CAF_LOG_ERROR("learned_new_node_indirectly called, but no route to nid"); return;
}
auto origin = instance.tbl().origin(nid);
if (!origin) {
CAF_LOG_ERROR("establish_communication called, but no node known "
"to ask for contact information");
return; return;
} }
if (path->next_hop == nid) { auto ehdl = instance.tbl().handle(*origin);
CAF_LOG_ERROR("learned_new_node_indirectly called with direct connection"); if (!ehdl) {
CAF_LOG_ERROR("establish_communication called, but node with contact "
"information is no longer reachable");
return; return;
} }
auto hdl = std::move(*ehdl);
using namespace detail; using namespace detail;
auto try_connect = [&](std::string item) { auto try_connect = [&](std::string item) {
auto tmp = get_or(config(), "middleman.attach-utility-actors", false) auto tmp = get_or(config(), "middleman.attach-utility-actors", false)
? system().spawn<hidden>(connection_helper, self) ? system().spawn<detached + hidden>(connection_helper, self, &instance)
: system().spawn<detached + hidden>(connection_helper, self); : system().spawn<hidden>(connection_helper, self, &instance);
system().registry().put(tmp.id(), actor_cast<strong_actor_ptr>(tmp)); system().registry().put(tmp.id(), actor_cast<strong_actor_ptr>(tmp));
auto writer = make_callback([&item](serializer& sink) -> error { auto writer = make_callback([&item](serializer& sink) -> error {
auto name_atm = atom("ConfigServ"); auto name_atm = atom("PeerServ");
std::vector<actor_id> stages; std::vector<actor_id> stages;
auto msg = make_message(get_atom::value, std::move(item)); auto msg = make_message(get_atom::value, std::move(item));
return sink(name_atm, stages, msg); return sink(name_atm, stages, msg);
}); });
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, make_message_id().integer_value(), this_node(), nid, 0, 0, this_node(), *origin, tmp.id(), invalid_actor_id,
tmp.id(), invalid_actor_id, visit(seq_num_visitor{this}, hdl)};
visit(seq_num_visitor{this}, path->hdl)}; instance.write(self->context(), get_buffer(hdl),
instance.write(self->context(), get_buffer(path->hdl),
hdr, &writer); hdr, &writer);
instance.flush(*path); flush(hdl);
}; };
auto item = to_string(nid);
if (allow_tcp) if (allow_tcp)
try_connect("basp.default-connectivity-tcp"); try_connect(item);
if (allow_udp) if (allow_udp)
try_connect("basp.default-connectivity-udp"); try_connect(item);
} }
void basp_broker_state::set_context(connection_handle hdl) { void basp_broker_state::set_context(connection_handle hdl) {
...@@ -474,7 +501,7 @@ void basp_broker_state::cleanup(connection_handle hdl) { ...@@ -474,7 +501,7 @@ void basp_broker_state::cleanup(connection_handle hdl) {
purge_state(nid); purge_state(nid);
return none; return none;
}); });
instance.tbl().erase_direct(hdl, cb); instance.tbl().erase(hdl, cb);
// Remove the context for `hdl`, making sure clients receive an error in case // Remove the context for `hdl`, making sure clients receive an error in case
// this connection was closed during handshake. // this connection was closed during handshake.
auto i = ctx_tcp.find(hdl); auto i = ctx_tcp.find(hdl);
...@@ -497,7 +524,7 @@ void basp_broker_state::cleanup(datagram_handle hdl) { ...@@ -497,7 +524,7 @@ void basp_broker_state::cleanup(datagram_handle hdl) {
purge_state(nid); purge_state(nid);
return none; return none;
}); });
instance.tbl().erase_direct(hdl, cb); instance.tbl().erase(hdl, cb);
// Remove the context for `hdl`, making sure clients receive an error in case // Remove the context for `hdl`, making sure clients receive an error in case
// this connection was closed during handshake. // this connection was closed during handshake.
auto i = ctx_udp.find(hdl); auto i = ctx_udp.find(hdl);
...@@ -535,7 +562,7 @@ void basp_broker_state::add_pending(execution_unit* ctx, ...@@ -535,7 +562,7 @@ void basp_broker_state::add_pending(execution_unit* ctx,
if (ep.pending.size() >= max_pending_messages) if (ep.pending.size() >= max_pending_messages)
deliver_pending(ctx, ep, true); deliver_pending(ctx, ep, true);
else if (!ep.did_set_timeout) else if (!ep.did_set_timeout)
self->delayed_send(self, pending_to, pending_atom::value, self->delayed_send(self, pending_timeout, pending_atom::value,
get<datagram_handle>(ep.hdl)); get<datagram_handle>(ep.hdl));
} }
...@@ -547,8 +574,9 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx, ...@@ -547,8 +574,9 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx,
std::vector<char>* payload = nullptr; std::vector<char>* payload = nullptr;
auto i = ep.pending.begin(); auto i = ep.pending.begin();
// Force delivery of at least the first messages, if desired. // Force delivery of at least the first messages, if desired.
if (force) if (force) {
ep.seq_incoming = i->first; ep.seq_incoming = i->first;
}
while (i != ep.pending.end() && i->first == ep.seq_incoming) { while (i != ep.pending.end() && i->first == ep.seq_incoming) {
ep.hdr = std::move(i->second.first); ep.hdr = std::move(i->second.first);
payload = &i->second.second; payload = &i->second.second;
...@@ -560,7 +588,7 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx, ...@@ -560,7 +588,7 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx,
} }
// Set a timeout if there are still pending messages. // Set a timeout if there are still pending messages.
if (!ep.pending.empty() && !ep.did_set_timeout) if (!ep.pending.empty() && !ep.did_set_timeout)
self->delayed_send(self, pending_to, pending_atom::value, self->delayed_send(self, pending_timeout, pending_atom::value,
get<datagram_handle>(ep.hdl)); get<datagram_handle>(ep.hdl));
return true; return true;
} }
...@@ -572,6 +600,38 @@ void basp_broker_state::drop_pending(basp::endpoint_context& ep, ...@@ -572,6 +600,38 @@ void basp_broker_state::drop_pending(basp::endpoint_context& ep,
ep.pending.erase(seq); ep.pending.erase(seq);
} }
void basp_broker_state::send_buffered_messages(execution_unit*,
node_id nid,
connection_handle hdl) {
if (pending_connectivity.count(nid) > 0) {
for (auto& msg : pending_connectivity[nid]) {
auto& buf = get_buffer(hdl);
buf.insert(buf.end(), msg.begin(), msg.end());
}
}
flush(hdl);
}
void basp_broker_state::send_buffered_messages(execution_unit* ctx,
node_id nid,
datagram_handle hdl) {
if (pending_connectivity.count(nid) > 0) {
for (auto& msg : pending_connectivity[nid]) {
auto seq_num = next_sequence_number(hdl);
auto seq_size = sizeof(basp::sequence_type);
auto offset = basp::header_size - seq_size;
auto& buf = msg;
stream_serializer<charbuf> out{ctx, buf.data() + offset, seq_size};
auto err = out(seq_num);
if (err)
CAF_LOG_ERROR(CAF_ARG(err));
self->enqueue_datagram(hdl, std::move(buf));
self->flush(hdl);
}
pending_connectivity[nid].clear();
}
}
basp_broker_state::buffer_type& basp_broker_state::buffer_type&
basp_broker_state::get_buffer(endpoint_handle hdl) { basp_broker_state::get_buffer(endpoint_handle hdl) {
if (hdl.is<connection_handle>()) if (hdl.is<connection_handle>())
...@@ -592,6 +652,16 @@ basp_broker_state::get_buffer(connection_handle hdl) { ...@@ -592,6 +652,16 @@ basp_broker_state::get_buffer(connection_handle hdl) {
return self->wr_buf(hdl); return self->wr_buf(hdl);
} }
basp_broker_state::buffer_type&
basp_broker_state::get_buffer(node_id nid) {
auto res = instance.tbl().lookup(nid);
if (res.known && res.hdl)
return get_buffer(*res.hdl);
auto& msgs = pending_connectivity[nid];
msgs.emplace_back();
return msgs.back();
}
basp_broker_state::buffer_type basp_broker_state::buffer_type
basp_broker_state::pop_datagram_buffer(datagram_handle) { basp_broker_state::pop_datagram_buffer(datagram_handle) {
std::vector<char> res; std::vector<char> res;
...@@ -608,8 +678,9 @@ void basp_broker_state::flush(endpoint_handle hdl) { ...@@ -608,8 +678,9 @@ void basp_broker_state::flush(endpoint_handle hdl) {
} }
void basp_broker_state::flush(datagram_handle hdl) { void basp_broker_state::flush(datagram_handle hdl) {
if (!cached_buffers.empty() && !cached_buffers.top().empty()) if (!cached_buffers.empty() && !cached_buffers.top().empty()) {
self->enqueue_datagram(hdl, pop_datagram_buffer(hdl)); self->enqueue_datagram(hdl, pop_datagram_buffer(hdl));
}
self->flush(hdl); self->flush(hdl);
} }
...@@ -634,13 +705,24 @@ behavior basp_broker::make_behavior() { ...@@ -634,13 +705,24 @@ behavior basp_broker::make_behavior() {
state.allow_udp = get_or(config(), "middleman.enable-udp", false); state.allow_udp = get_or(config(), "middleman.enable-udp", false);
if (get_or(config(), "middleman.enable-automatic-connections", false)) { if (get_or(config(), "middleman.enable-automatic-connections", false)) {
CAF_LOG_DEBUG("enable automatic connections"); CAF_LOG_DEBUG("enable automatic connections");
// open a random port and store a record for our peers how to auto addrs = network::interfaces::list_addresses(false);
// connect to this broker directly in the configuration server for (auto& p : addrs) {
auto& vec = p.second;
// Remove link local addresses.
vec.erase(std::remove_if(std::begin(vec), std::end(vec),
[](const std::string& str) {
return str.find("fe80") == 0;
}),
vec.end());
}
// Open a random port and store a record for our peers how to
// connect to this broker directly in the configuration server.
if (state.allow_tcp) { if (state.allow_tcp) {
auto res = add_tcp_doorman(uint16_t{0}); auto res = add_tcp_doorman(uint16_t{0});
if (res) { if (res) {
auto port = res->second; auto port = res->second;
auto addrs = network::interfaces::list_addresses(false); state.instance.tbl().local_addresses(network::protocol::tcp,
{port, addrs});
auto config_server = system().registry().get(atom("ConfigServ")); auto config_server = system().registry().get(atom("ConfigServ"));
send(actor_cast<actor>(config_server), put_atom::value, send(actor_cast<actor>(config_server), put_atom::value,
"basp.default-connectivity-tcp", "basp.default-connectivity-tcp",
...@@ -651,7 +733,8 @@ behavior basp_broker::make_behavior() { ...@@ -651,7 +733,8 @@ behavior basp_broker::make_behavior() {
auto res = add_udp_datagram_servant(uint16_t{0}); auto res = add_udp_datagram_servant(uint16_t{0});
if (res) { if (res) {
auto port = res->second; auto port = res->second;
auto addrs = network::interfaces::list_addresses(false); state.instance.tbl().local_addresses(network::protocol::udp,
{port, addrs});
auto config_server = system().registry().get(atom("ConfigServ")); auto config_server = system().registry().get(atom("ConfigServ"));
send(actor_cast<actor>(config_server), put_atom::value, send(actor_cast<actor>(config_server), put_atom::value,
"basp.default-connectivity-udp", "basp.default-connectivity-udp",
...@@ -667,7 +750,7 @@ behavior basp_broker::make_behavior() { ...@@ -667,7 +750,7 @@ behavior basp_broker::make_behavior() {
send(this, tick_atom::value, heartbeat_interval); send(this, tick_atom::value, heartbeat_interval);
} }
return { return {
// received from underlying broker implementation // Received from underlying broker implementation.
[=](new_data_msg& msg) { [=](new_data_msg& msg) {
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
state.set_context(msg.handle); state.set_context(msg.handle);
...@@ -687,7 +770,7 @@ behavior basp_broker::make_behavior() { ...@@ -687,7 +770,7 @@ behavior basp_broker::make_behavior() {
ctx.cstate = next; ctx.cstate = next;
} }
}, },
// received from auto connect broker for UDP communication // Received from auto connect broker for UDP communication.
[=](new_datagram_msg& msg, datagram_servant_ptr ptr, uint16_t port) { [=](new_datagram_msg& msg, datagram_servant_ptr ptr, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
auto hdl = ptr->hdl(); auto hdl = ptr->hdl();
...@@ -698,11 +781,11 @@ behavior basp_broker::make_behavior() { ...@@ -698,11 +781,11 @@ behavior basp_broker::make_behavior() {
ctx.local_port = local_port(hdl); ctx.local_port = local_port(hdl);
ctx.requires_ordering = true; ctx.requires_ordering = true;
ctx.seq_incoming = 0; ctx.seq_incoming = 0;
ctx.seq_outgoing = 1; // already sent the client handshake ctx.seq_outgoing = 1; // Already sent the client handshake.
// Let's not implement this twice // Let's not implement this twice.
send(this, std::move(msg)); send(this, std::move(msg));
}, },
// received from underlying broker implementation // Received from underlying broker implementation.
[=](new_datagram_msg& msg) { [=](new_datagram_msg& msg) {
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
state.set_context(msg.handle); state.set_context(msg.handle);
...@@ -719,12 +802,12 @@ behavior basp_broker::make_behavior() { ...@@ -719,12 +802,12 @@ behavior basp_broker::make_behavior() {
close(msg.handle); close(msg.handle);
} }
}, },
// received from the underlying broker implementation // Received from the underlying broker implementation.
[=](datagram_sent_msg& msg) { [=](datagram_sent_msg& msg) {
if (state.cached_buffers.size() < state.max_buffers) if (state.cached_buffers.size() < state.max_buffers)
state.cached_buffers.emplace(std::move(msg.buf)); state.cached_buffers.emplace(std::move(msg.buf));
}, },
// received from proxy instances // Received from proxy instances.
[=](forward_atom, strong_actor_ptr& src, [=](forward_atom, strong_actor_ptr& src,
const std::vector<strong_actor_ptr>& fwd_stack, const std::vector<strong_actor_ptr>& fwd_stack,
strong_actor_ptr& dest, message_id mid, const message& msg) { strong_actor_ptr& dest, message_id mid, const message& msg) {
...@@ -744,7 +827,7 @@ behavior basp_broker::make_behavior() { ...@@ -744,7 +827,7 @@ behavior basp_broker::make_behavior() {
srb(src, mid); srb(src, mid);
} }
}, },
// received from some system calls like whereis // Received from some system calls like whereis.
[=](forward_atom, const node_id& dest_node, atom_value dest_name, [=](forward_atom, const node_id& dest_node, atom_value dest_name,
const message& msg) -> result<message> { const message& msg) -> result<message> {
auto cme = current_mailbox_element(); auto cme = current_mailbox_element();
...@@ -757,9 +840,9 @@ behavior basp_broker::make_behavior() { ...@@ -757,9 +840,9 @@ behavior basp_broker::make_behavior() {
<< ", " << CAF_ARG(msg)); << ", " << CAF_ARG(msg));
if (!src) if (!src)
return sec::invalid_argument; return sec::invalid_argument;
auto path = this->state.instance.tbl().lookup(dest_node); auto res = this->state.instance.tbl().lookup(dest_node);
if (!path) { if (!res.known) {
CAF_LOG_ERROR("no route to receiving node"); CAF_LOG_ERROR("host unknown or unreachable");
return sec::no_route_to_receiving_node; return sec::no_route_to_receiving_node;
} }
if (system().node() == src->node()) if (system().node() == src->node())
...@@ -771,14 +854,22 @@ behavior basp_broker::make_behavior() { ...@@ -771,14 +854,22 @@ behavior basp_broker::make_behavior() {
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(), 0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id, dest_node, src->id(), invalid_actor_id,
visit(seq_num_visitor{&state}, path->hdl)}; 0};
state.instance.write(context(), state.get_buffer(path->hdl), if (res.hdl) {
hdr, &writer); auto hdl = std::move(*res.hdl);
state.instance.flush(*path); hdr.sequence_number = visit(seq_num_visitor{&state}, hdl);
state.instance.write(context(), state.get_buffer(hdl),
hdr, &writer);
state.flush(hdl);
} else {
state.instance.write(context(), state.get_buffer(dest_node), hdr,
&writer);
}
return delegated<message>(); return delegated<message>();
}, },
// received from underlying broker implementation // Received from underlying broker implementation.
[=](const new_connection_msg& msg) { [=](const new_connection_msg& msg) {
auto res = state.instance.tbl().lookup(msg.handle);
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
auto& bi = state.instance; auto& bi = state.instance;
bi.write_server_handshake(context(), state.get_buffer(msg.handle), bi.write_server_handshake(context(), state.get_buffer(msg.handle),
...@@ -786,18 +877,18 @@ behavior basp_broker::make_behavior() { ...@@ -786,18 +877,18 @@ behavior basp_broker::make_behavior() {
state.flush(msg.handle); state.flush(msg.handle);
configure_read(msg.handle, receive_policy::exactly(basp::header_size)); configure_read(msg.handle, receive_policy::exactly(basp::header_size));
}, },
// received from underlying broker implementation // Received from underlying broker implementation.
[=](const connection_closed_msg& msg) { [=](const connection_closed_msg& msg) {
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
state.cleanup(msg.handle); state.cleanup(msg.handle);
}, },
// received from underlying broker implementation // Received from underlying broker implementation.
[=](const acceptor_closed_msg& msg) { [=](const acceptor_closed_msg& msg) {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
auto port = local_port(msg.handle); auto port = local_port(msg.handle);
state.instance.remove_published_actor(port); state.instance.remove_published_actor(port);
}, },
// received from middleman actor // Received from middleman actor.
[=](publish_atom, doorman_ptr& ptr, uint16_t port, [=](publish_atom, doorman_ptr& ptr, uint16_t port,
const strong_actor_ptr& whom, std::set<std::string>& sigs) { const strong_actor_ptr& whom, std::set<std::string>& sigs) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port) CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port)
...@@ -808,7 +899,7 @@ behavior basp_broker::make_behavior() { ...@@ -808,7 +899,7 @@ behavior basp_broker::make_behavior() {
system().registry().put(whom->id(), whom); system().registry().put(whom->id(), whom);
state.instance.add_published_actor(port, whom, std::move(sigs)); state.instance.add_published_actor(port, whom, std::move(sigs));
}, },
// received from middleman actor (delegated) // Received from middleman actor (delegated).
[=](connect_atom, scribe_ptr& ptr, uint16_t port) { [=](connect_atom, scribe_ptr& ptr, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port));
CAF_ASSERT(ptr != nullptr); CAF_ASSERT(ptr != nullptr);
...@@ -821,7 +912,7 @@ behavior basp_broker::make_behavior() { ...@@ -821,7 +912,7 @@ behavior basp_broker::make_behavior() {
ctx.cstate = basp::await_header; ctx.cstate = basp::await_header;
ctx.callback = rp; ctx.callback = rp;
ctx.requires_ordering = false; ctx.requires_ordering = false;
// await server handshake // Await server handshake.
configure_read(hdl, receive_policy::exactly(basp::header_size)); configure_read(hdl, receive_policy::exactly(basp::header_size));
}, },
[=](publish_udp_atom, datagram_servant_ptr& ptr, uint16_t port, [=](publish_udp_atom, datagram_servant_ptr& ptr, uint16_t port,
...@@ -834,7 +925,7 @@ behavior basp_broker::make_behavior() { ...@@ -834,7 +925,7 @@ behavior basp_broker::make_behavior() {
system().registry().put(whom->id(), whom); system().registry().put(whom->id(), whom);
state.instance.add_published_actor(port, whom, std::move(sigs)); state.instance.add_published_actor(port, whom, std::move(sigs));
}, },
// received from middleman actor (delegated) // Received from middleman actor (delegated).
[=](contact_atom, datagram_servant_ptr& ptr, uint16_t port) { [=](contact_atom, datagram_servant_ptr& ptr, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port));
auto rp = make_response_promise(); auto rp = make_response_promise();
...@@ -853,11 +944,11 @@ behavior basp_broker::make_behavior() { ...@@ -853,11 +944,11 @@ behavior basp_broker::make_behavior() {
none, ctx.seq_outgoing++); none, ctx.seq_outgoing++);
state.flush(hdl); state.flush(hdl);
}, },
// received from underlying broker implementation // Received from underlying broker implementation.
[=](const datagram_servant_closed_msg& msg) { [=](const datagram_servant_closed_msg& msg) {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
// since all handles share a port, we can take any of them to query for // Since all handles share a port, we can take any of them to query for
// port information // port information.
CAF_ASSERT(msg.handles.size() > 0); CAF_ASSERT(msg.handles.size() > 0);
auto port = local_port(msg.handles.front()); auto port = local_port(msg.handles.front());
state.instance.remove_published_actor(port); state.instance.remove_published_actor(port);
...@@ -893,8 +984,8 @@ behavior basp_broker::make_behavior() { ...@@ -893,8 +984,8 @@ behavior basp_broker::make_behavior() {
[=](close_atom, uint16_t port) -> result<void> { [=](close_atom, uint16_t port) -> result<void> {
if (port == 0) if (port == 0)
return sec::cannot_close_invalid_port; return sec::cannot_close_invalid_port;
// it is well-defined behavior to not have an actor published here, // It is well-defined behavior to not have an actor published here,
// hence the result can be ignored safely // hence the result can be ignored safely.
state.instance.remove_published_actor(port, nullptr); state.instance.remove_published_actor(port, nullptr);
auto res = close(hdl_by_port(port)); auto res = close(hdl_by_port(port));
if (res) if (res)
...@@ -905,10 +996,10 @@ behavior basp_broker::make_behavior() { ...@@ -905,10 +996,10 @@ behavior basp_broker::make_behavior() {
-> std::tuple<node_id, std::string, uint16_t> { -> std::tuple<node_id, std::string, uint16_t> {
std::string addr; std::string addr;
uint16_t port = 0; uint16_t port = 0;
auto hdl = state.instance.tbl().lookup_direct(x); auto res = state.instance.tbl().lookup(x);
if (hdl) { if (res.known && res.hdl) {
addr = visit(addr_visitor{this}, *hdl); addr = visit(addr_visitor{this}, *res.hdl);
port = visit(port_visitor{this}, *hdl); port = visit(port_visitor{this}, *res.hdl);
} }
return std::make_tuple(x, std::move(addr), port); return std::make_tuple(x, std::move(addr), port);
}, },
......
...@@ -34,9 +34,11 @@ auto autoconnect_timeout = std::chrono::minutes(10); ...@@ -34,9 +34,11 @@ auto autoconnect_timeout = std::chrono::minutes(10);
const char* connection_helper_state::name = "connection_helper"; const char* connection_helper_state::name = "connection_helper";
behavior datagram_connection_broker(broker* self, uint16_t port, behavior datagram_connection_broker(broker* self,
uint16_t port,
network::address_listing addresses, network::address_listing addresses,
actor system_broker) { actor system_broker,
basp::instance* instance) {
auto& mx = self->system().middleman().backend(); auto& mx = self->system().middleman().backend();
auto& this_node = self->system().node(); auto& this_node = self->system().node();
auto app_id = get_or(self->config(), "middleman.app-identifier", auto app_id = get_or(self->config(), "middleman.app-identifier",
...@@ -47,81 +49,96 @@ behavior datagram_connection_broker(broker* self, uint16_t port, ...@@ -47,81 +49,96 @@ behavior datagram_connection_broker(broker* self, uint16_t port,
if (eptr) { if (eptr) {
auto hdl = (*eptr)->hdl(); auto hdl = (*eptr)->hdl();
self->add_datagram_servant(std::move(*eptr)); self->add_datagram_servant(std::move(*eptr));
basp::instance::write_client_handshake(self->context(), std::vector<char> buf;
self->wr_buf(hdl), instance->write_client_handshake(self->context(), buf,
none, this_node, none, this_node, app_id);
app_id); self->enqueue_datagram(hdl, std::move(buf));
self->flush(hdl);
} }
} }
} }
// We are not interested in attempts that do not work.
self->set_default_handler(drop);
return { return {
[=](new_datagram_msg& msg) { [=](new_datagram_msg& msg) {
auto hdl = msg.handle; auto hdl = msg.handle;
auto cap = msg.buf.capacity();
self->send(system_broker, std::move(msg), self->take(hdl), port); self->send(system_broker, std::move(msg), self->take(hdl), port);
self->quit(); self->quit();
// The buffer returned to the multiplexer will have capacity zero
// if we don't do this. Handling this case here is cheaper than
// checking before each receive in the multiplexer.
msg.buf.reserve(cap);
}, },
after(autoconnect_timeout) >> [=]() { after(autoconnect_timeout) >> [=]() {
CAF_LOG_TRACE(CAF_ARG("")); CAF_LOG_TRACE(CAF_ARG(""));
// nothing heard in about 10 minutes... just a call it a day, then // Nothing heard in about 10 minutes... just call it a day, then.
CAF_LOG_INFO("aborted direct connection attempt after 10min"); CAF_LOG_INFO("aborted direct connection attempt after 10 min");
self->quit(exit_reason::user_shutdown); self->quit(exit_reason::user_shutdown);
} }
}; };
} }
bool establish_stream_connection(stateful_actor<connection_helper_state>* self,
const actor& system_broker,
uint16_t port,
network::address_listing& addresses) {
auto& mx = self->system().middleman().backend();
for (auto& kvp : addresses) {
for (auto& addr : kvp.second) {
auto hdl = mx.new_tcp_scribe(addr, port);
if (hdl) {
// Gotcha! Send scribe to our BASP broker to initiate handshake etc.
CAF_LOG_INFO("connected directly:" << CAF_ARG(addr));
self->send(system_broker, connect_atom::value, *hdl, port);
return true;
}
}
}
return false;
}
behavior connection_helper(stateful_actor<connection_helper_state>* self, behavior connection_helper(stateful_actor<connection_helper_state>* self,
actor b) { actor system_broker, basp::instance* i) {
CAF_LOG_TRACE(CAF_ARG(b)); CAF_LOG_TRACE(CAF_ARG(system_broker));
self->monitor(b); self->monitor(system_broker);
self->set_down_handler([=](down_msg& dm) { self->set_down_handler([=](down_msg& dm) {
CAF_LOG_TRACE(CAF_ARG(dm)); CAF_LOG_TRACE(CAF_ARG(dm));
self->quit(std::move(dm.reason)); self->quit(std::move(dm.reason));
}); });
return { return {
// this config is send from the remote `ConfigServ` // This config is send from the remote `PeerServ`.
[=](const std::string& item, message& msg) { [=](const std::string& item, message& msg) {
CAF_LOG_TRACE(CAF_ARG(item) << CAF_ARG(msg)); CAF_LOG_TRACE(CAF_ARG(item) << CAF_ARG(msg));
CAF_LOG_DEBUG("received requested config:" << CAF_ARG(msg)); CAF_LOG_DEBUG("received requested config:" << CAF_ARG(msg));
// whatever happens, we are done afterwards if (item.empty() || msg.empty()) {
CAF_LOG_DEBUG("skipping empty info");
return;
}
// Whatever happens, we are done afterwards.
self->quit(); self->quit();
msg.apply({ msg.apply({
[&](uint16_t port, network::address_listing& addresses) { [&](basp::routing_table::address_map& addrs) {
if (item == "basp.default-connectivity-tcp") { if (addrs.count(network::protocol::tcp) > 0) {
auto& mx = self->system().middleman().backend(); auto eps = addrs[network::protocol::tcp];
for (auto& kvp : addresses) { if (!establish_stream_connection(self, system_broker, eps.first,
for (auto& addr : kvp.second) { eps.second))
auto hdl = mx.new_tcp_scribe(addr, port); CAF_LOG_ERROR("could not connect to node ");
if (hdl) { }
// gotcha! send scribe to our BASP broker if (addrs.count(network::protocol::udp) > 0) {
// to initiate handshake etc. auto eps = addrs[network::protocol::udp];
CAF_LOG_INFO("connected directly:" << CAF_ARG(addr)); // Create new broker to try addresses for communication via UDP.
self->send(b, connect_atom::value, *hdl, port); auto b = self->system().middleman().spawn_broker(
return; datagram_connection_broker, eps.first, std::move(eps.second),
} system_broker, i
} );
}
CAF_LOG_INFO("could not connect to node directly");
} else if (item == "basp.default-connectivity-udp") {
auto& sys = self->system();
// create new broker to try addresses for communication via UDP
if (get_or(sys.config(), "middleman.attach-utility-actors", false))
self->system().middleman().spawn_broker<hidden>(
datagram_connection_broker, port, std::move(addresses), b
);
else
self->system().middleman().spawn_broker<detached + hidden>(
datagram_connection_broker, port, std::move(addresses), b
);
} else {
CAF_LOG_INFO("aborted direct connection attempt, unknown item: "
<< CAF_ARG(item));
} }
} }
}); });
}, },
after(autoconnect_timeout) >> [=] { after(autoconnect_timeout) >> [=] {
CAF_LOG_TRACE(CAF_ARG("")); CAF_LOG_TRACE(CAF_ARG(""));
// nothing heard in about 10 minutes... just a call it a day, then // Nothing heard in about 10 minutes... just a call it a day, then.
CAF_LOG_INFO("aborted direct connection attempt after 10min"); CAF_LOG_INFO("aborted direct connection attempt after 10min");
self->quit(exit_reason::user_shutdown); self->quit(exit_reason::user_shutdown);
} }
......
...@@ -733,7 +733,7 @@ default_multiplexer::new_local_udp_endpoint(uint16_t port, const char* in, ...@@ -733,7 +733,7 @@ default_multiplexer::new_local_udp_endpoint(uint16_t port, const char* in,
} }
int64_t default_multiplexer::next_endpoint_id() { int64_t default_multiplexer::next_endpoint_id() {
return servant_ids_++; return servant_ids_.fetch_add(1);
} }
void default_multiplexer::handle_internal_events() { void default_multiplexer::handle_internal_events() {
......
...@@ -73,52 +73,59 @@ bool zero(T val) { ...@@ -73,52 +73,59 @@ bool zero(T val) {
} }
bool server_handshake_valid(const header& hdr) { bool server_handshake_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& zero(hdr.dest_actor) && zero(hdr.dest_actor)
&& !zero(hdr.operation_data); && !zero(hdr.operation_data);
} }
bool client_handshake_valid(const header& hdr) { bool client_handshake_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& hdr.source_node != hdr.dest_node && hdr.source_node != hdr.dest_node
&& zero(hdr.source_actor) && zero(hdr.source_actor)
&& zero(hdr.dest_actor); && zero(hdr.dest_actor);
} }
bool dispatch_message_valid(const header& hdr) { bool dispatch_message_valid(const header& hdr) {
return valid(hdr.dest_node) return valid(hdr.dest_node)
&& (!zero(hdr.dest_actor) || hdr.has(header::named_receiver_flag)) && (!zero(hdr.dest_actor) || hdr.has(header::named_receiver_flag))
&& !zero(hdr.payload_len); && !zero(hdr.payload_len);
} }
bool announce_proxy_instance_valid(const header& hdr) { bool announce_proxy_instance_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& valid(hdr.dest_node) && valid(hdr.dest_node)
&& hdr.source_node != hdr.dest_node && hdr.source_node != hdr.dest_node
&& zero(hdr.source_actor) && zero(hdr.source_actor)
&& !zero(hdr.dest_actor) && !zero(hdr.dest_actor)
&& zero(hdr.payload_len) && zero(hdr.payload_len)
&& zero(hdr.operation_data); && zero(hdr.operation_data);
} }
bool kill_proxy_instance_valid(const header& hdr) { bool kill_proxy_instance_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& valid(hdr.dest_node) && valid(hdr.dest_node)
&& hdr.source_node != hdr.dest_node && hdr.source_node != hdr.dest_node
&& !zero(hdr.source_actor) && !zero(hdr.source_actor)
&& zero(hdr.dest_actor) && zero(hdr.dest_actor)
&& !zero(hdr.payload_len) && !zero(hdr.payload_len)
&& zero(hdr.operation_data); && zero(hdr.operation_data);
} }
bool heartbeat_valid(const header& hdr) { bool heartbeat_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& valid(hdr.dest_node) && valid(hdr.dest_node)
&& hdr.source_node != hdr.dest_node && hdr.source_node != hdr.dest_node
&& zero(hdr.source_actor) && zero(hdr.source_actor)
&& zero(hdr.dest_actor) && zero(hdr.dest_actor)
&& zero(hdr.payload_len) && zero(hdr.payload_len)
&& zero(hdr.operation_data); && zero(hdr.operation_data);
}
bool acknowledge_handshake_valid(const header& hdr) {
return valid(hdr.source_node)
&& hdr.source_node != hdr.dest_node
&& zero(hdr.source_actor)
&& zero(hdr.dest_actor);
} }
} // namespace <anonymous> } // namespace <anonymous>
...@@ -139,6 +146,8 @@ bool valid(const header& hdr) { ...@@ -139,6 +146,8 @@ bool valid(const header& hdr) {
return kill_proxy_instance_valid(hdr); return kill_proxy_instance_valid(hdr);
case message_type::heartbeat: case message_type::heartbeat:
return heartbeat_valid(hdr); return heartbeat_valid(hdr);
case message_type::acknowledge_handshake:
return acknowledge_handshake_valid(hdr);
} }
} }
......
...@@ -59,17 +59,16 @@ instance::instance(abstract_broker* parent, callee& lstnr) ...@@ -59,17 +59,16 @@ instance::instance(abstract_broker* parent, callee& lstnr)
CAF_ASSERT(this_node_ != none); CAF_ASSERT(this_node_ != none);
} }
connection_state instance::handle(execution_unit* ctx, connection_state instance::handle(execution_unit* ctx, new_data_msg& dm,
new_data_msg& dm, header& hdr, header& hdr, bool is_payload) {
bool is_payload) {
CAF_LOG_TRACE(CAF_ARG(dm) << CAF_ARG(is_payload)); CAF_LOG_TRACE(CAF_ARG(dm) << CAF_ARG(is_payload));
// function object providing cleanup code on errors // Function object providing cleanup code on errors.
auto err = [&]() -> connection_state { auto err = [&]() -> connection_state {
auto cb = make_callback([&](const node_id& nid) -> error { auto cb = make_callback([&](const node_id& nid) -> error {
callee_.purge_state(nid); callee_.purge_state(nid);
return none; return none;
}); });
tbl_.erase_direct(dm.handle, cb); tbl_.erase(dm.handle, cb);
return close_connection; return close_connection;
}; };
std::vector<char>* payload = nullptr; std::vector<char>* payload = nullptr;
...@@ -93,35 +92,10 @@ connection_state instance::handle(execution_unit* ctx, ...@@ -93,35 +92,10 @@ connection_state instance::handle(execution_unit* ctx,
} }
} }
CAF_LOG_DEBUG(CAF_ARG(hdr)); CAF_LOG_DEBUG(CAF_ARG(hdr));
// needs forwarding? // Needs forwarding?
if (!is_handshake(hdr) && !is_heartbeat(hdr) && hdr.dest_node != this_node_) { if (!is_handshake(hdr) && !is_heartbeat(hdr) && hdr.dest_node != this_node_) {
CAF_LOG_DEBUG("forward message"); // TODO: Forwarding should no longer happen.
auto path = lookup(hdr.dest_node); return err();
if (path) {
binary_serializer bs{ctx, callee_.get_buffer(path->hdl)};
auto e = bs(hdr);
if (e)
return err();
if (payload != nullptr)
bs.apply_raw(payload->size(), payload->data());
flush(*path);
notify<hook::message_forwarded>(hdr, payload);
} else {
CAF_LOG_INFO("cannot forward message, no route to destination");
if (hdr.source_node != this_node_) {
// TODO: signalize error back to sending node
auto reverse_path = lookup(hdr.source_node);
if (!reverse_path) {
CAF_LOG_WARNING("cannot send error message: no route to source");
} else {
CAF_LOG_WARNING("not implemented yet: signalize forward failure");
}
} else {
CAF_LOG_WARNING("lost packet with probably spoofed source");
}
notify<hook::message_forwarding_failed>(hdr, payload);
}
return await_header;
} }
if (!handle(ctx, dm.handle, hdr, payload, true, none, none)) if (!handle(ctx, dm.handle, hdr, payload, true, none, none))
return err(); return err();
...@@ -131,22 +105,22 @@ connection_state instance::handle(execution_unit* ctx, ...@@ -131,22 +105,22 @@ connection_state instance::handle(execution_unit* ctx,
bool instance::handle(execution_unit* ctx, new_datagram_msg& dm, bool instance::handle(execution_unit* ctx, new_datagram_msg& dm,
endpoint_context& ep) { endpoint_context& ep) {
using itr_t = network::receive_buffer::iterator; using itr_t = network::receive_buffer::iterator;
// function object providing cleanup code on errors // Function object providing cleanup code on errors.
auto err = [&]() -> bool { auto err = [&]() -> bool {
auto cb = make_callback([&](const node_id& nid) -> error { auto cb = make_callback([&](const node_id& nid) -> error {
callee_.purge_state(nid); callee_.purge_state(nid);
return none; return none;
}); });
tbl_.erase_direct(dm.handle, cb); tbl_.erase(dm.handle, cb);
return false; return false;
}; };
// extract payload // Extract payload.
std::vector<char> pl_buf{std::move_iterator<itr_t>(std::begin(dm.buf) + std::vector<char> pl_buf{std::move_iterator<itr_t>(std::begin(dm.buf) +
basp::header_size), basp::header_size),
std::move_iterator<itr_t>(std::end(dm.buf))}; std::move_iterator<itr_t>(std::end(dm.buf))};
// resize header // Resize header.
dm.buf.resize(basp::header_size); dm.buf.resize(basp::header_size);
// extract header // Extract header.
binary_deserializer bd{ctx, dm.buf}; binary_deserializer bd{ctx, dm.buf};
auto e = bd(ep.hdr); auto e = bd(ep.hdr);
if (e || !valid(ep.hdr)) { if (e || !valid(ep.hdr)) {
...@@ -175,36 +149,11 @@ bool instance::handle(execution_unit* ctx, new_datagram_msg& dm, ...@@ -175,36 +149,11 @@ bool instance::handle(execution_unit* ctx, new_datagram_msg& dm,
} }
// This is the expected message. // This is the expected message.
ep.seq_incoming += 1; ep.seq_incoming += 1;
// TODO: add optional reliability here // TODO: Add optional reliability here.
if (!is_handshake(ep.hdr) && !is_heartbeat(ep.hdr) if (!is_handshake(ep.hdr) && !is_heartbeat(ep.hdr)
&& ep.hdr.dest_node != this_node_) { && ep.hdr.dest_node != this_node_) {
CAF_LOG_DEBUG("forward message"); // TODO: Forwarding should no longer happen.
auto path = lookup(ep.hdr.dest_node); return err();
if (path) {
binary_serializer bs{ctx, callee_.get_buffer(path->hdl)};
auto ex = bs(ep.hdr);
if (ex)
return err();
if (payload != nullptr)
bs.apply_raw(payload->size(), payload->data());
flush(*path);
notify<hook::message_forwarded>(ep.hdr, payload);
} else {
CAF_LOG_INFO("cannot forward message, no route to destination");
if (ep.hdr.source_node != this_node_) {
// TODO: signalize error back to sending node
auto reverse_path = lookup(ep.hdr.source_node);
if (!reverse_path) {
CAF_LOG_WARNING("cannot send error message: no route to source");
} else {
CAF_LOG_WARNING("not implemented yet: signalize forward failure");
}
} else {
CAF_LOG_WARNING("lost packet with probably spoofed source");
}
notify<hook::message_forwarding_failed>(ep.hdr, payload);
}
return true;
} }
if (!handle(ctx, dm.handle, ep.hdr, payload, false, ep, ep.local_port)) if (!handle(ctx, dm.handle, ep.hdr, payload, false, ep, ep.local_port))
return err(); return err();
...@@ -216,7 +165,7 @@ bool instance::handle(execution_unit* ctx, new_datagram_msg& dm, ...@@ -216,7 +165,7 @@ bool instance::handle(execution_unit* ctx, new_datagram_msg& dm,
void instance::handle_heartbeat(execution_unit* ctx) { void instance::handle_heartbeat(execution_unit* ctx) {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
for (auto& kvp: tbl_.direct_by_hdl_) { for (auto& kvp: tbl_.nid_by_hdl_) {
CAF_LOG_TRACE(CAF_ARG(kvp.first) << CAF_ARG(kvp.second)); CAF_LOG_TRACE(CAF_ARG(kvp.first) << CAF_ARG(kvp.second));
write_heartbeat(ctx, callee_.get_buffer(kvp.first), write_heartbeat(ctx, callee_.get_buffer(kvp.first),
kvp.second, visit(seq_num_visitor{callee_}, kvp.first)); kvp.second, visit(seq_num_visitor{callee_}, kvp.first));
...@@ -224,20 +173,16 @@ void instance::handle_heartbeat(execution_unit* ctx) { ...@@ -224,20 +173,16 @@ void instance::handle_heartbeat(execution_unit* ctx) {
} }
} }
optional<routing_table::route> instance::lookup(const node_id& target) { void instance::flush(endpoint_handle hdl) {
return tbl_.lookup(target); callee_.flush(hdl);
}
void instance::flush(const routing_table::route& path) {
callee_.flush(path.hdl);
} }
void instance::write(execution_unit* ctx, const routing_table::route& r, void instance::write(execution_unit* ctx, instance::endpoint_handle hdl,
header& hdr, payload_writer* writer) { header& hdr, payload_writer* writer) {
CAF_LOG_TRACE(CAF_ARG(hdr)); CAF_LOG_TRACE(CAF_ARG(hdr));
CAF_ASSERT(hdr.payload_len == 0 || writer != nullptr); CAF_ASSERT(hdr.payload_len == 0 || writer != nullptr);
write(ctx, callee_.get_buffer(r.hdl), hdr, writer); write(ctx, callee_.get_buffer(hdl), hdr, writer);
flush(r); callee_.flush(hdl);
} }
void instance::add_published_actor(uint16_t port, void instance::add_published_actor(uint16_t port,
...@@ -295,7 +240,7 @@ size_t instance::remove_published_actor(const actor_addr& whom, ...@@ -295,7 +240,7 @@ size_t instance::remove_published_actor(const actor_addr& whom,
bool instance::is_greater(sequence_type lhs, sequence_type rhs, bool instance::is_greater(sequence_type lhs, sequence_type rhs,
sequence_type max_distance) { sequence_type max_distance) {
// distance between lhs and rhs is smaller than max_distance. // Distance between lhs and rhs is smaller than max_distance.
return ((lhs > rhs) && (lhs - rhs <= max_distance)) || return ((lhs > rhs) && (lhs - rhs <= max_distance)) ||
((lhs < rhs) && (rhs - lhs > max_distance)); ((lhs < rhs) && (rhs - lhs > max_distance));
} }
...@@ -307,8 +252,8 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, ...@@ -307,8 +252,8 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
CAF_LOG_TRACE(CAF_ARG(sender) << CAF_ARG(receiver) CAF_LOG_TRACE(CAF_ARG(sender) << CAF_ARG(receiver)
<< CAF_ARG(mid) << CAF_ARG(msg)); << CAF_ARG(mid) << CAF_ARG(msg));
CAF_ASSERT(receiver && system().node() != receiver->node()); CAF_ASSERT(receiver && system().node() != receiver->node());
auto path = lookup(receiver->node()); auto res = tbl_.lookup(receiver->node());
if (!path) { if (!res.known) {
notify<hook::message_sending_failed>(sender, receiver, mid, msg); notify<hook::message_sending_failed>(sender, receiver, mid, msg);
return false; return false;
} }
...@@ -319,11 +264,21 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, ...@@ -319,11 +264,21 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(), header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(),
sender ? sender->node() : this_node(), receiver->node(), sender ? sender->node() : this_node(), receiver->node(),
sender ? sender->id() : invalid_actor_id, receiver->id(), sender ? sender->id() : invalid_actor_id, receiver->id(),
visit(seq_num_visitor{callee_}, path->hdl)}; 0};
write(ctx, callee_.get_buffer(path->hdl), hdr, &writer); if (res.hdl) {
flush(*path); auto hdl = std::move(*res.hdl);
notify<hook::message_sent>(sender, path->next_hop, receiver, mid, msg); hdr.sequence_number = visit(seq_num_visitor{callee_}, hdl);
return true; write(ctx, callee_.get_buffer(hdl), hdr, &writer);
callee_.flush(hdl);
notify<hook::message_sent>(sender, receiver->node(), receiver, mid, msg);
return true;
} else {
write(ctx, callee_.get_buffer(receiver->node()), hdr, &writer);
// TODO: Should the hook really be called here, or should we delay this
// until communication is established?
notify<hook::message_sent>(sender, receiver->node(), receiver, mid, msg);
return true;
}
} }
void instance::write(execution_unit* ctx, buffer_type& buf, void instance::write(execution_unit* ctx, buffer_type& buf,
...@@ -332,7 +287,7 @@ void instance::write(execution_unit* ctx, buffer_type& buf, ...@@ -332,7 +287,7 @@ void instance::write(execution_unit* ctx, buffer_type& buf,
error err; error err;
if (pw != nullptr) { if (pw != nullptr) {
auto pos = buf.size(); auto pos = buf.size();
// write payload first (skip first 72 bytes and write header later) // Write payload first (skip first 72 bytes and write header later).
char placeholder[basp::header_size]; char placeholder[basp::header_size];
buf.insert(buf.end(), std::begin(placeholder), std::end(placeholder)); buf.insert(buf.end(), std::begin(placeholder), std::end(placeholder));
binary_serializer bs{ctx, buf}; binary_serializer bs{ctx, buf};
...@@ -371,11 +326,11 @@ void instance::write_server_handshake(execution_unit* ctx, ...@@ -371,11 +326,11 @@ void instance::write_server_handshake(execution_unit* ctx,
return e; return e;
if (pa != nullptr) { if (pa != nullptr) {
auto i = pa->first ? pa->first->id() : invalid_actor_id; auto i = pa->first ? pa->first->id() : invalid_actor_id;
return sink(i, pa->second); return sink(i, pa->second, tbl_.local_addresses());
} }
auto aid = invalid_actor_id; auto aid = invalid_actor_id;
std::set<std::string> tmp; std::set<std::string> tmp;
return sink(aid, tmp); return sink(aid, tmp, tbl_.local_addresses());
}); });
header hdr{message_type::server_handshake, 0, 0, version, header hdr{message_type::server_handshake, 0, 0, version,
this_node_, none, this_node_, none,
...@@ -392,7 +347,8 @@ void instance::write_client_handshake(execution_unit* ctx, ...@@ -392,7 +347,8 @@ void instance::write_client_handshake(execution_unit* ctx,
uint16_t sequence_number) { uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(remote_side)); CAF_LOG_TRACE(CAF_ARG(remote_side));
auto writer = make_callback([&](serializer& sink) -> error { auto writer = make_callback([&](serializer& sink) -> error {
return sink(const_cast<std::string&>(app_identifier)); return sink(const_cast<std::string&>(app_identifier),
tbl_.local_addresses());
}); });
header hdr{message_type::client_handshake, 0, 0, 0, header hdr{message_type::client_handshake, 0, 0, 0,
this_node, remote_side, invalid_actor_id, invalid_actor_id, this_node, remote_side, invalid_actor_id, invalid_actor_id,
...@@ -410,6 +366,17 @@ void instance::write_client_handshake(execution_unit* ctx, ...@@ -410,6 +366,17 @@ void instance::write_client_handshake(execution_unit* ctx,
sequence_number); sequence_number);
} }
void instance::write_acknowledge_handshake(execution_unit* ctx,
buffer_type& buf,
const node_id& remote_side,
uint16_t sequence_number) {
header hdr{message_type::acknowledge_handshake, 0, 0, 0,
this_node_, remote_side, invalid_actor_id, invalid_actor_id,
sequence_number};
write(ctx, buf, hdr);
}
void instance::write_announce_proxy(execution_unit* ctx, buffer_type& buf, void instance::write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid, const node_id& dest_node, actor_id aid,
uint16_t sequence_number) { uint16_t sequence_number) {
......
...@@ -32,7 +32,8 @@ const char* message_type_strings[] = { ...@@ -32,7 +32,8 @@ const char* message_type_strings[] = {
"dispatch_message", "dispatch_message",
"announce_proxy_instance", "announce_proxy_instance",
"kill_proxy_instance", "kill_proxy_instance",
"heartbeat" "heartbeat",
"acknowledge_handshake"
}; };
} // namespace <anonymous> } // namespace <anonymous>
......
...@@ -420,6 +420,8 @@ void middleman::init(actor_system_config& cfg) { ...@@ -420,6 +420,8 @@ void middleman::init(actor_system_config& cfg) {
cfg.add_message_type<network::protocol>("@protocol") cfg.add_message_type<network::protocol>("@protocol")
.add_message_type<network::address_listing>("@address_listing") .add_message_type<network::address_listing>("@address_listing")
.add_message_type<network::receive_buffer>("@receive_buffer") .add_message_type<network::receive_buffer>("@receive_buffer")
.add_message_type<basp::routing_table::address_endpoint>("@address_endpoint")
.add_message_type<basp::routing_table::address_map>("@address_map")
.add_message_type<new_data_msg>("@new_data_msg") .add_message_type<new_data_msg>("@new_data_msg")
.add_message_type<new_connection_msg>("@new_connection_msg") .add_message_type<new_connection_msg>("@new_connection_msg")
.add_message_type<acceptor_closed_msg>("@acceptor_closed_msg") .add_message_type<acceptor_closed_msg>("@acceptor_closed_msg")
......
...@@ -33,123 +33,115 @@ routing_table::~routing_table() { ...@@ -33,123 +33,115 @@ routing_table::~routing_table() {
// nop // nop
} }
optional<routing_table::route> routing_table::lookup(const node_id& target) { node_id routing_table::lookup(const endpoint_handle& hdl) const {
auto hdl = lookup_direct(target); return get_opt(nid_by_hdl_, hdl, none);
if (hdl)
return route{target, *hdl};
// pick first available indirect route
auto i = indirect_.find(target);
if (i != indirect_.end()) {
auto& hops = i->second;
while (!hops.empty()) {
auto& hop = *hops.begin();
hdl = lookup_direct(hop);
if (hdl)
return route{hop, *hdl};
hops.erase(hops.begin());
}
}
return none;
}
node_id routing_table::lookup_direct(const endpoint_handle& hdl) const {
return get_opt(direct_by_hdl_, hdl, none);
} }
optional<routing_table::endpoint_handle> routing_table::lookup_result routing_table::lookup(const node_id& nid) const {
routing_table::lookup_direct(const node_id& nid) const { auto i = node_information_base_.find(nid);
auto i = direct_by_nid_.find(nid); if (i != node_information_base_.end())
if (i != direct_by_nid_.end()) return {true, i->second.hdl};
return i->second; return {false, none};
return none;
} }
node_id routing_table::lookup_indirect(const node_id& nid) const { void routing_table::erase(const endpoint_handle& hdl, erase_callback& cb) {
auto i = indirect_.find(nid); auto i = nid_by_hdl_.find(hdl);
if (i == indirect_.end()) if (i == nid_by_hdl_.end())
return none; return;
if (i->second.empty()) cb(i->second);
return none; parent_->parent().notify<hook::connection_lost>(i->second);
return *i->second.begin(); node_information_base_.erase(i->second);
nid_by_hdl_.erase(i->first);
// TODO: Look through other nodes and remove this one as an origin?
} }
void routing_table::blacklist(const node_id& hop, const node_id& dest) { void routing_table::add(const node_id& nid, const endpoint_handle& hdl) {
blacklist_[dest].emplace(hop); CAF_ASSERT(nid_by_hdl_.count(hdl) == 0);
auto i = indirect_.find(dest); CAF_ASSERT(node_information_base_.count(nid) == 0);
if (i == indirect_.end()) nid_by_hdl_.emplace(hdl, nid);
return; node_information_base_[nid] = node_info{hdl, none, false};
i->second.erase(hop); parent_->parent().notify<hook::new_connection_established>(nid);
if (i->second.empty())
indirect_.erase(i);
} }
void routing_table::erase_direct(const endpoint_handle& hdl, void routing_table::add(const node_id& nid, const node_id& origin) {
erase_callback& cb) { CAF_ASSERT(node_information_base_.count(nid) == 0);
auto i = direct_by_hdl_.find(hdl); node_information_base_[nid] = node_info{none, origin, false};
if (i == direct_by_hdl_.end()) // TODO: Some new related hook?
return; //parent_->parent().notify<hook::new_connection_established>(nid);
cb(i->second); }
parent_->parent().notify<hook::connection_lost>(i->second);
direct_by_nid_.erase(i->second); void routing_table::add(const node_id& nid) {
direct_by_hdl_.erase(i->first); //CAF_ASSERT(hdl_by_nid_.count(nid) == 0);
CAF_ASSERT(node_information_base_.count(nid) == 0);
node_information_base_[nid] = node_info{none, none, false};
// TODO: Some new related hook?
//parent_->parent().notify<hook::new_connection_established>(nid);
} }
bool routing_table::erase_indirect(const node_id& dest) { bool routing_table::reachable(const node_id& dest) {
auto i = indirect_.find(dest); auto i = node_information_base_.find(dest);
if (i == indirect_.end()) if (i != node_information_base_.end())
return i->second.hdl != none;
return false;
}
bool routing_table::origin(const node_id& nid,
const node_id& origin) {
auto i = node_information_base_.find(nid);
if (i == node_information_base_.end())
return false; return false;
if (parent_->parent().has_hook()) i->second.origin = origin;
for (auto& nid : i->second)
parent_->parent().notify<hook::route_lost>(nid, dest);
indirect_.erase(i);
return true; return true;
} }
void routing_table::add_direct(const endpoint_handle& hdl, optional<node_id>
const node_id& nid) { routing_table::origin(const node_id& nid) {
CAF_ASSERT(direct_by_hdl_.count(hdl) == 0); auto i = node_information_base_.find(nid);
CAF_ASSERT(direct_by_nid_.count(nid) == 0); if (i == node_information_base_.end())
direct_by_hdl_.emplace(hdl, nid); return none;
direct_by_nid_.emplace(nid, hdl); return i->second.origin;
parent_->parent().notify<hook::new_connection_established>(nid);
} }
bool routing_table::add_indirect(const node_id& hop, const node_id& dest) { bool routing_table::handle(const node_id& nid,
auto i = blacklist_.find(dest); const routing_table::endpoint_handle& hdl) {
if (i == blacklist_.end() || i->second.count(hop) == 0) { auto i = node_information_base_.find(nid);
auto& hops = indirect_[dest]; if (i == node_information_base_.end())
auto added_first = hops.empty(); return false;
hops.emplace(hop); i->second.hdl = hdl;
parent_->parent().notify<hook::new_route_added>(hop, dest); nid_by_hdl_.emplace(hdl, nid);
return added_first; return true;
}
return false; // blacklisted
} }
bool routing_table::reachable(const node_id& dest) { optional<routing_table::endpoint_handle>
return direct_by_nid_.count(dest) > 0 || indirect_.count(dest) > 0; routing_table::handle(const node_id& nid) {
} auto i = node_information_base_.find(nid);
if (i == node_information_base_.end())
size_t routing_table::erase(const node_id& dest, erase_callback& cb) { return none;
cb(dest); return i->second.hdl;
size_t res = 0; }
auto i = indirect_.find(dest);
if (i != indirect_.end()) { void routing_table::local_addresses(network::protocol::transport key,
res = i->second.size(); routing_table::address_endpoint addrs) {
for (auto& nid : i->second) { local_addrs_[key] = addrs;
cb(nid); }
parent_->parent().notify<hook::route_lost>(nid, dest);
} const routing_table::address_map& routing_table::local_addresses() {
indirect_.erase(i); return local_addrs_;
} }
auto hdl = lookup_direct(dest);
if (hdl) { bool routing_table::received_client_handshake(const node_id& nid, bool flag) {
direct_by_hdl_.erase(*hdl); auto i = node_information_base_.find(nid);
direct_by_nid_.erase(dest); if (i == node_information_base_.end())
parent_->parent().notify<hook::connection_lost>(dest); return false;
++res; i->second.received_client_handshake = flag;
} return true;
return res; }
bool routing_table::received_client_handshake(const node_id& nid) {
auto i = node_information_base_.find(nid);
if (i == node_information_base_.end())
return false;
return i->second.received_client_handshake;
} }
} // namespace basp } // namespace basp
......
...@@ -53,18 +53,27 @@ test_multiplexer::doorman_data::doorman_data() ...@@ -53,18 +53,27 @@ test_multiplexer::doorman_data::doorman_data()
// nop // nop
} }
test_multiplexer::datagram_data::
datagram_data(shared_job_queue_type input, test_multiplexer::datagram_endpoint::datagram_endpoint(datagram_handle hdl,
shared_job_queue_type output) shared_job_queue_type input,
: vn_buf_ptr(std::move(input)), shared_job_queue_type output)
wr_buf_ptr(std::move(output)), : hdl(hdl),
vn_buf(*vn_buf_ptr), vn_buf_ptr(std::move(input)),
wr_buf(*wr_buf_ptr), wr_buf_ptr(std::move(output)),
rd_buf(datagram_handle::from_int(0), receive_buffer_size), vn_buf(*vn_buf_ptr),
wr_buf(*wr_buf_ptr) {
// nop
}
test_multiplexer::datagram_data::datagram_data(datagram_handle hdl,
shared_job_queue_type input,
shared_job_queue_type output)
: read_handle(hdl, std::move(input), std::move(output)),
rd_buf(0, receive_buffer_size),
stopped_reading(false), stopped_reading(false),
passive_mode(false), passive_mode(false),
ack_writes(false), ack_writes(false),
port(0), remote_port(0),
local_port(0), local_port(0),
datagram_size(receive_buffer_size) { datagram_size(receive_buffer_size) {
// nop // nop
...@@ -78,7 +87,7 @@ test_multiplexer::test_multiplexer(actor_system* sys) ...@@ -78,7 +87,7 @@ test_multiplexer::test_multiplexer(actor_system* sys)
} }
test_multiplexer::~test_multiplexer() { test_multiplexer::~test_multiplexer() {
// get rid of extra ref count // Get rid of extra ref count.
for (auto& ptr : resumables_) for (auto& ptr : resumables_)
intrusive_ptr_release(ptr.get()); intrusive_ptr_release(ptr.get());
} }
...@@ -131,7 +140,7 @@ scribe_ptr test_multiplexer::new_scribe(connection_handle hdl) { ...@@ -131,7 +140,7 @@ scribe_ptr test_multiplexer::new_scribe(connection_handle hdl) {
}; };
CAF_LOG_DEBUG(CAF_ARG(hdl)); CAF_LOG_DEBUG(CAF_ARG(hdl));
auto sptr = make_counted<impl>(hdl, this); auto sptr = make_counted<impl>(hdl, this);
{ // lifetime scope of guard { // Lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
impl_ptr(hdl) = sptr; impl_ptr(hdl) = sptr;
} }
...@@ -143,7 +152,7 @@ expected<scribe_ptr> test_multiplexer::new_tcp_scribe(const std::string& host, ...@@ -143,7 +152,7 @@ expected<scribe_ptr> test_multiplexer::new_tcp_scribe(const std::string& host,
uint16_t port) { uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port));
connection_handle hdl; connection_handle hdl;
{ // lifetime scope of guard { // Lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
auto i = scribes_.find(std::make_pair(host, port)); auto i = scribes_.find(std::make_pair(host, port));
if (i != scribes_.end()) { if (i != scribes_.end()) {
...@@ -207,7 +216,7 @@ doorman_ptr test_multiplexer::new_doorman(accept_handle hdl, uint16_t port) { ...@@ -207,7 +216,7 @@ doorman_ptr test_multiplexer::new_doorman(accept_handle hdl, uint16_t port) {
test_multiplexer* mpx_; test_multiplexer* mpx_;
}; };
auto dptr = make_counted<impl>(hdl, this); auto dptr = make_counted<impl>(hdl, this);
{ // lifetime scope of guard { // Lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
auto& ref = doorman_data_[hdl]; auto& ref = doorman_data_[hdl];
ref.ptr = dptr; ref.ptr = dptr;
...@@ -266,24 +275,30 @@ expected<datagram_servant_ptr> ...@@ -266,24 +275,30 @@ expected<datagram_servant_ptr>
test_multiplexer::new_remote_udp_endpoint(const std::string& host, test_multiplexer::new_remote_udp_endpoint(const std::string& host,
uint16_t port) { uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port));
auto key_pair = std::make_pair(host, port);
datagram_handle hdl; datagram_handle hdl;
{ // lifetime scope of guard intptr_t ep;
{ // Lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
auto i = remote_endpoints_.find(std::make_pair(host, port)); auto i = remote_endpoints_.find(key_pair);
if (i != remote_endpoints_.end()) { if (i != remote_endpoints_.end()) {
hdl = i->second; hdl = i->second.first;
ep = i->second.second;
remote_endpoints_.erase(i); remote_endpoints_.erase(i);
} else { } else {
return sec::cannot_connect_to_node; return sec::cannot_connect_to_node;
} }
} }
auto ptr = new_datagram_servant(hdl, port); auto ptr = new_datagram_servant(hdl, port);
// Set state in the struct to enable direct communication? // Set state in the struct to enable direct communication.
{ // lifetime scope of guard { // Lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
auto data = data_for_hdl(hdl); auto data = data_for_hdl(hdl);
data->servants.emplace(hdl); data->write_handles.emplace(ep,
local_port(hdl) = data->local_port; datagram_endpoint(hdl,
data->read_handle.vn_buf_ptr,
data->read_handle.wr_buf_ptr));
data->remote_port = port;
} }
return ptr; return ptr;
} }
...@@ -302,7 +317,7 @@ test_multiplexer::new_local_udp_endpoint(uint16_t desired_port, ...@@ -302,7 +317,7 @@ test_multiplexer::new_local_udp_endpoint(uint16_t desired_port,
port = std::numeric_limits<uint16_t>::max(); port = std::numeric_limits<uint16_t>::max();
while (is_known_port(port)) while (is_known_port(port))
--port; --port;
// Do the same for finding a local dgram handle // Do the same for finding a local datagram handle.
auto y = std::numeric_limits<int64_t>::max(); auto y = std::numeric_limits<int64_t>::max();
while (is_known_handle(datagram_handle::from_int(y))) while (is_known_handle(datagram_handle::from_int(y)))
--y; --y;
...@@ -318,7 +333,9 @@ test_multiplexer::new_local_udp_endpoint(uint16_t desired_port, ...@@ -318,7 +333,9 @@ test_multiplexer::new_local_udp_endpoint(uint16_t desired_port,
} }
} }
} }
return new_datagram_servant(hdl, port); auto tmp = new_datagram_servant(hdl, port);
auto data = data_for_hdl(hdl);
return tmp;
} }
datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
...@@ -332,17 +349,24 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, ...@@ -332,17 +349,24 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
} }
bool new_endpoint(network::receive_buffer& buf) override { bool new_endpoint(network::receive_buffer& buf) override {
datagram_handle dhdl; datagram_handle dhdl;
shared_job_queue_type wr_buf_ptr;
auto data = mpx_->data_for_hdl(hdl());
{ // Try to get a connection handle of a pending connect. { // Try to get a connection handle of a pending connect.
guard_type guard{mpx_->mx_}; guard_type guard{mpx_->mx_};
auto& pe = mpx_->pending_endpoints(); auto& pe = mpx_->pending_endpoints();
auto i = pe.find(hdl().id()); auto i = pe.find(data->rd_buf.first);
if (i == pe.end()) if (i == pe.end())
return false; return false;
dhdl = i->second; dhdl = i->second.first;
wr_buf_ptr = i->second.second;
pe.erase(i); pe.erase(i);
} }
auto data = mpx_->data_for_hdl(hdl()); data->write_handles.emplace(
data->servants.emplace(dhdl); std::make_pair(
data->rd_buf.first,
datagram_endpoint{dhdl, data->read_handle.vn_buf_ptr, wr_buf_ptr}
)
);
mpx_->datagram_data_.emplace(dhdl, data); mpx_->datagram_data_.emplace(dhdl, data);
parent()->add_hdl_for_datagram_servant(this, dhdl); parent()->add_hdl_for_datagram_servant(this, dhdl);
return consume(mpx_, dhdl, buf); return consume(mpx_, dhdl, buf);
...@@ -352,13 +376,11 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, ...@@ -352,13 +376,11 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
} }
std::vector<char>& wr_buf(datagram_handle dh) override { std::vector<char>& wr_buf(datagram_handle dh) override {
auto& buf = mpx_->output_buffer(dh); auto& buf = mpx_->output_buffer(dh);
buf.first = dh;
return buf.second; return buf.second;
} }
void enqueue_datagram(datagram_handle dh, void enqueue_datagram(datagram_handle dh, std::vector<char> buf) override {
std::vector<char> buf) override {
auto& q = mpx_->output_queue(dh); auto& q = mpx_->output_queue(dh);
q.emplace_back(dh, std::move(buf)); q.emplace_back(mpx_->endpoint_id(dh), std::move(buf));
} }
network::receive_buffer& rd_buf() override { network::receive_buffer& rd_buf() override {
auto& buf = mpx_->input_buffer(hdl()); auto& buf = mpx_->input_buffer(hdl());
...@@ -387,8 +409,10 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, ...@@ -387,8 +409,10 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
} }
std::vector<datagram_handle> hdls() const override { std::vector<datagram_handle> hdls() const override {
auto data = mpx_->data_for_hdl(hdl()); auto data = mpx_->data_for_hdl(hdl());
std::vector<datagram_handle> result(data->servants.begin(), std::vector<datagram_handle> result;
data->servants.end()); result.reserve(data->write_handles.size());
for (auto& p : data->write_handles)
result.push_back(p.second.hdl);
return result; return result;
} }
void add_to_loop() override { void add_to_loop() override {
...@@ -402,20 +426,20 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, ...@@ -402,20 +426,20 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
} }
void remove_endpoint(datagram_handle dh) override { void remove_endpoint(datagram_handle dh) override {
auto data = mpx_->data_for_hdl(hdl()); auto data = mpx_->data_for_hdl(hdl());
{ // lifetime scope of guard { // Lifetime scope of guard.
guard_type guard{mpx_->mx_}; guard_type guard{mpx_->mx_};
auto itr = std::find(data->servants.begin(), data->servants.end(), dh); auto endpoint_id = mpx_->endpoint_id(dh);
if (itr != data->servants.end()) data->write_handles.erase(endpoint_id);
data->servants.erase(itr); parent()->erase(dh);
} }
} }
void detach_handles() override { void detach_handles() override {
auto data = mpx_->data_for_hdl(hdl()); auto data = mpx_->data_for_hdl(hdl());
for (auto& p : data->servants) for (auto& p : data->write_handles)
if (p != hdl()) if (p.second.hdl != hdl())
parent()->erase(p); parent()->erase(p.second.hdl);
data->servants.clear(); data->write_handles.clear();
data->servants.emplace(hdl()); data->write_handles.emplace(mpx_->endpoint_id(hdl()), hdl());
} }
private: private:
test_multiplexer* mpx_; test_multiplexer* mpx_;
...@@ -423,11 +447,10 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl, ...@@ -423,11 +447,10 @@ datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
auto dptr = make_counted<impl>(hdl, this); auto dptr = make_counted<impl>(hdl, this);
CAF_LOG_INFO("new datagram servant" << hdl); CAF_LOG_INFO("new datagram servant" << hdl);
auto data = data_for_hdl(hdl); auto data = data_for_hdl(hdl);
{ // lifetime scope of guard { // lifetime scope of guard.
guard_type guard{mx_}; guard_type guard{mx_};
data->ptr = dptr; data->ptr = dptr;
data->port = port; data->remote_port = port;
data->servants.emplace(hdl);
} }
return dptr; return dptr;
} }
...@@ -448,7 +471,7 @@ bool test_multiplexer::is_known_port(uint16_t x) const { ...@@ -448,7 +471,7 @@ bool test_multiplexer::is_known_port(uint16_t x) const {
return x == y.second.port; return x == y.second.port;
}; };
auto pred2 = [&](const datagram_data_map::value_type& y) { auto pred2 = [&](const datagram_data_map::value_type& y) {
return x == y.second->port; return x == y.second->remote_port;
}; };
return (doormen_.count(x) + local_endpoints_.count(x)) > 0 return (doormen_.count(x) + local_endpoints_.count(x)) > 0
|| std::any_of(doorman_data_.begin(), doorman_data_.end(), pred1) || std::any_of(doorman_data_.begin(), doorman_data_.end(), pred1)
...@@ -468,7 +491,7 @@ bool test_multiplexer::is_known_handle(datagram_handle x) const { ...@@ -468,7 +491,7 @@ bool test_multiplexer::is_known_handle(datagram_handle x) const {
return x == y.second; return x == y.second;
}; };
auto pred2 = [&](const pending_remote_datagram_endpoints_map::value_type& y) { auto pred2 = [&](const pending_remote_datagram_endpoints_map::value_type& y) {
return x == y.second; return x == y.second.first;
}; };
return datagram_data_.count(x) > 0 return datagram_data_.count(x) > 0
|| std::any_of(local_endpoints_.begin(), local_endpoints_.end(), pred1) || std::any_of(local_endpoints_.begin(), local_endpoints_.end(), pred1)
...@@ -476,7 +499,7 @@ bool test_multiplexer::is_known_handle(datagram_handle x) const { ...@@ -476,7 +499,7 @@ bool test_multiplexer::is_known_handle(datagram_handle x) const {
} }
auto test_multiplexer::make_supervisor() -> supervisor_ptr { auto test_multiplexer::make_supervisor() -> supervisor_ptr {
// not needed // Not needed.
return nullptr; return nullptr;
} }
...@@ -519,15 +542,17 @@ void test_multiplexer::provide_datagram_servant(uint16_t desired_port, ...@@ -519,15 +542,17 @@ void test_multiplexer::provide_datagram_servant(uint16_t desired_port,
void test_multiplexer::provide_datagram_servant(std::string host, void test_multiplexer::provide_datagram_servant(std::string host,
uint16_t desired_port, uint16_t desired_port,
datagram_handle hdl) { datagram_handle hdl,
intptr_t endpoint_id) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(desired_port) << CAF_ARG(hdl)); CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(desired_port) << CAF_ARG(hdl));
guard_type guard{mx_}; guard_type guard{mx_};
remote_endpoints_.emplace(std::make_pair(std::move(host), desired_port), hdl); if (endpoint_id == 0)
endpoint_id = static_cast<intptr_t>(hdl.id());
remote_endpoints_.emplace(std::make_pair(std::move(host), desired_port),
std::make_pair(std::move(hdl), endpoint_id));
} }
/// The external input buffer should be filled by
/// the test program.
test_multiplexer::buffer_type& test_multiplexer::buffer_type&
test_multiplexer::virtual_network_buffer(connection_handle hdl) { test_multiplexer::virtual_network_buffer(connection_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
...@@ -537,7 +562,7 @@ test_multiplexer::virtual_network_buffer(connection_handle hdl) { ...@@ -537,7 +562,7 @@ test_multiplexer::virtual_network_buffer(connection_handle hdl) {
test_multiplexer::write_job_queue_type& test_multiplexer::write_job_queue_type&
test_multiplexer::virtual_network_buffer(datagram_handle hdl) { test_multiplexer::virtual_network_buffer(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->vn_buf; return data_for_hdl(hdl)->read_handle.vn_buf;
} }
test_multiplexer::buffer_type& test_multiplexer::buffer_type&
...@@ -555,15 +580,32 @@ test_multiplexer::input_buffer(connection_handle hdl) { ...@@ -555,15 +580,32 @@ test_multiplexer::input_buffer(connection_handle hdl) {
test_multiplexer::write_job_type& test_multiplexer::write_job_type&
test_multiplexer::output_buffer(datagram_handle hdl) { test_multiplexer::output_buffer(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
auto& buf = data_for_hdl(hdl)->wr_buf; auto data = data_for_hdl(hdl);
auto itr = std::find_if(std::begin(data->write_handles),
std::end(data->write_handles),
[&](write_handle_map::value_type& p) {
return p.second.hdl == hdl;
});
if (itr == std::end(data->write_handles))
CAF_RAISE_ERROR("write buffer for unknown endpoint");
auto& buf = itr->second.wr_buf;
buf.emplace_back(); buf.emplace_back();
buf.back().first = itr->first;
return buf.back(); return buf.back();
} }
test_multiplexer::write_job_queue_type& test_multiplexer::write_job_queue_type&
test_multiplexer::output_queue(datagram_handle hdl) { test_multiplexer::output_queue(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->wr_buf; auto data = data_for_hdl(hdl);
auto itr = std::find_if(std::begin(data->write_handles),
std::end(data->write_handles),
[&](write_handle_map::value_type& p) {
return p.second.hdl == hdl;
});
if (itr == std::end(data->write_handles))
CAF_RAISE_ERROR("write queue for unknown endpoint");
return itr->second.wr_buf;
} }
test_multiplexer::read_job_type& test_multiplexer::read_job_type&
...@@ -616,7 +658,7 @@ uint16_t& test_multiplexer::port(accept_handle hdl) { ...@@ -616,7 +658,7 @@ uint16_t& test_multiplexer::port(accept_handle hdl) {
} }
uint16_t& test_multiplexer::port(datagram_handle hdl) { uint16_t& test_multiplexer::port(datagram_handle hdl) {
return data_for_hdl(hdl)->port; return data_for_hdl(hdl)->remote_port;
} }
uint16_t& test_multiplexer::local_port(datagram_handle hdl) { uint16_t& test_multiplexer::local_port(datagram_handle hdl) {
...@@ -627,8 +669,9 @@ datagram_servant_ptr& test_multiplexer::impl_ptr(datagram_handle hdl) { ...@@ -627,8 +669,9 @@ datagram_servant_ptr& test_multiplexer::impl_ptr(datagram_handle hdl) {
return data_for_hdl(hdl)->ptr; return data_for_hdl(hdl)->ptr;
} }
std::set<datagram_handle>& test_multiplexer::servants(datagram_handle hdl) { test_multiplexer::endpoint_id_type
return data_for_hdl(hdl)->servants; test_multiplexer::endpoint_id(datagram_handle hdl) {
return reinterpret_cast<intptr_t>(data_for_hdl(hdl).get());
} }
bool& test_multiplexer::stopped_reading(accept_handle hdl) { bool& test_multiplexer::stopped_reading(accept_handle hdl) {
...@@ -656,8 +699,8 @@ test_multiplexer::data_for_hdl(datagram_handle hdl) { ...@@ -656,8 +699,8 @@ test_multiplexer::data_for_hdl(datagram_handle hdl) {
auto itr = datagram_data_.find(hdl); auto itr = datagram_data_.find(hdl);
if (itr != datagram_data_.end()) if (itr != datagram_data_.end())
return itr->second; return itr->second;
// if it does not exist, create a new entry // If it does not exist, create a new entry.
datagram_data_.emplace(hdl, std::make_shared<datagram_data>()); datagram_data_.emplace(hdl, std::make_shared<datagram_data>(hdl));
return datagram_data_[hdl]; return datagram_data_[hdl];
} }
...@@ -689,10 +732,34 @@ void test_multiplexer::prepare_connection(accept_handle src, ...@@ -689,10 +732,34 @@ void test_multiplexer::prepare_connection(accept_handle src,
peer.provide_scribe(std::move(host), port, peer_hdl); peer.provide_scribe(std::move(host), port, peer_hdl);
} }
void test_multiplexer::add_pending_endpoint(datagram_handle src, void test_multiplexer::add_pending_endpoint(intptr_t endpoint_id, datagram_handle hdl,
datagram_handle hdl) { shared_job_queue_type write_buffer) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
pending_endpoints_.emplace(endpoint_id, std::make_pair(hdl, write_buffer));
}
void test_multiplexer::prepare_endpoints(datagram_handle src,
datagram_handle hdl,
test_multiplexer& peer,
std::string host, uint16_t port,
datagram_handle peer_hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
pending_endpoints_.emplace(src.id(), hdl); CAF_ASSERT(this != &peer);
CAF_LOG_TRACE(CAF_ARG(src) << CAF_ARG(hdl) << CAF_ARG(host) << CAF_ARG(port)
<< CAF_ARG(peer_hdl));
if (local_endpoints_.count(port) == 0)
provide_datagram_servant(port, src);
auto data = data_for_hdl(src);
auto src_id = reinterpret_cast<intptr_t>(data.get());
auto dd = std::make_shared<datagram_data>(peer_hdl, data->read_handle.wr_buf_ptr,
data->read_handle.vn_buf_ptr);
auto res = peer.datagram_data_.emplace(peer_hdl, dd);
if (!res.second)
CAF_RAISE_ERROR("prepare_endpoints: peer handle already in use");
auto peer_id = reinterpret_cast<intptr_t>(res.first->second.get());
peer.provide_datagram_servant(std::move(host), port, peer_hdl, src_id);
add_pending_endpoint(peer_id, hdl,
dd->read_handle.vn_buf_ptr);
} }
test_multiplexer::pending_connects_map& test_multiplexer::pending_connects() { test_multiplexer::pending_connects_map& test_multiplexer::pending_connects() {
...@@ -751,13 +818,21 @@ bool test_multiplexer::try_accept_connection() { ...@@ -751,13 +818,21 @@ bool test_multiplexer::try_accept_connection() {
bool test_multiplexer::try_read_data() { bool test_multiplexer::try_read_data() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
// scribe_data might change while we traverse it // scribe_data might change while we traverse it.
std::vector<connection_handle> xs; std::vector<connection_handle> chs;
xs.reserve(scribe_data_.size()); chs.reserve(scribe_data_.size());
for (auto& kvp : scribe_data_) for (auto& kvp : scribe_data_)
xs.emplace_back(kvp.first); chs.emplace_back(kvp.first);
for (auto x : xs) for (auto ch : chs)
if (try_read_data(x)) if (try_read_data(ch))
return true;
// datagram_data might change while we traverse it and multiple handles may
// share a read handle.
std::set<datagram_handle> dhs;
for (auto& kvp : datagram_data_)
dhs.insert(kvp.second->read_handle.hdl);
for (auto dh : dhs)
if (try_read_data(dh))
return true; return true;
return false; return false;
} }
...@@ -810,18 +885,61 @@ bool test_multiplexer::try_read_data(connection_handle hdl) { ...@@ -810,18 +885,61 @@ bool test_multiplexer::try_read_data(connection_handle hdl) {
return false; return false;
} }
bool test_multiplexer::try_read_data(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(hdl));
flush_runnables();
if (passive_mode(hdl))
return false;
auto ditr = datagram_data_.find(hdl);
if (ditr == datagram_data_.end() || ditr->second->ptr == nullptr
|| ditr->second->ptr->parent() == nullptr
|| !ditr->second->ptr->parent()->getf(abstract_actor::is_initialized_flag))
return false;
auto& data = ditr->second;
auto& rh = data->read_handle;
if (rh.vn_buf.empty() || rh.vn_buf.front().second.empty())
return false;
// Since we can't swap std::vector and caf::io::network::receive_buffer
// just copy over the data. This is for testing and not performance critical.
auto& from = rh.vn_buf.front();
auto& to = data->rd_buf;
to.first = from.first;
CAF_ASSERT(to.second.capacity() >= from.second.size());
to.second.resize(from.second.size());
std::copy(from.second.begin(), from.second.end(), to.second.begin());
rh.vn_buf.pop_front();
auto sitr = data->write_handles.find(data->rd_buf.first);
if (sitr == data->write_handles.end()) {
if (!data->ptr->new_endpoint(data->rd_buf.second))
passive_mode(hdl) = true;
} else {
if (!data->ptr->consume(this, sitr->second.hdl, data->rd_buf.second))
passive_mode(hdl) = true;
}
return true;
}
bool test_multiplexer::read_data() { bool test_multiplexer::read_data() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
// scribe_data might change while we traverse it // scribe_data might change while we traverse it.
std::vector<connection_handle> xs; std::vector<connection_handle> chs;
xs.reserve(scribe_data_.size()); chs.reserve(scribe_data_.size());
for (auto& kvp : scribe_data_) for (auto& kvp : scribe_data_)
xs.emplace_back(kvp.first); chs.emplace_back(kvp.first);
long hits = 0; long hits = 0;
for (auto x : xs) for (auto ch : chs)
if (scribe_data_.count(x) > 0) if (scribe_data_.count(ch) > 0)
if (read_data(x)) if (read_data(ch))
++hits;
// datagram_data might change while we traverse it.
std::set<datagram_handle> dhs;
for (auto& kvp : datagram_data_)
dhs.insert(kvp.second->read_handle.hdl);
for (auto dh : dhs)
if (datagram_data_.count(dh) > 0)
if (read_data(dh))
++hits; ++hits;
return hits > 0; return hits > 0;
} }
...@@ -837,7 +955,7 @@ bool test_multiplexer::read_data(connection_handle hdl) { ...@@ -837,7 +955,7 @@ bool test_multiplexer::read_data(connection_handle hdl) {
|| !sd.ptr->parent()->getf(abstract_actor::is_initialized_flag)) { || !sd.ptr->parent()->getf(abstract_actor::is_initialized_flag)) {
return false; return false;
} }
// count how many data packets we could dispatch // Count how many data packets we could dispatch.
long hits = 0; long hits = 0;
for (;;) { for (;;) {
switch (sd.recv_conf.first) { switch (sd.recv_conf.first) {
...@@ -889,34 +1007,8 @@ bool test_multiplexer::read_data(connection_handle hdl) { ...@@ -889,34 +1007,8 @@ bool test_multiplexer::read_data(connection_handle hdl) {
bool test_multiplexer::read_data(datagram_handle hdl) { bool test_multiplexer::read_data(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(hdl)); CAF_LOG_TRACE(CAF_ARG(hdl));
flush_runnables(); // Not really a distinction for udp.
if (passive_mode(hdl)) return try_read_data(hdl);
return false;
auto ditr = datagram_data_.find(hdl);
if (ditr == datagram_data_.end() || ditr->second->ptr->parent() == nullptr
|| !ditr->second->ptr->parent()->getf(abstract_actor::is_initialized_flag))
return false;
auto& data = ditr->second;
if (data->vn_buf.back().second.empty())
return false;
// Since we can't swap std::vector and caf::io::network::receive_buffer
// just copy over the data. This is for testing and not performance critical.
auto& from = data->vn_buf.front();
auto& to = data->rd_buf;
to.first = from.first;
CAF_ASSERT(to.second.capacity() > from.second.size());
to.second.resize(from.second.size());
std::copy(from.second.begin(), from.second.end(), to.second.begin());
data->vn_buf.pop_front();
auto sitr = datagram_data_.find(data->rd_buf.first);
if (sitr == datagram_data_.end()) {
if (!data->ptr->new_endpoint(data->rd_buf.second))
passive_mode(hdl) = true;
} else {
if (!data->ptr->consume(this, data->rd_buf.first, data->rd_buf.second))
passive_mode(hdl) = true;
}
return true;
} }
void test_multiplexer::virtual_send(connection_handle hdl, void test_multiplexer::virtual_send(connection_handle hdl,
...@@ -928,20 +1020,20 @@ void test_multiplexer::virtual_send(connection_handle hdl, ...@@ -928,20 +1020,20 @@ void test_multiplexer::virtual_send(connection_handle hdl,
read_data(hdl); read_data(hdl);
} }
void test_multiplexer::virtual_send(datagram_handle dst, datagram_handle ep, void test_multiplexer::virtual_send(datagram_handle hdl, endpoint_id_type ep,
const buffer_type& buf) { const buffer_type& buf) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(dst) << CAF_ARG(ep)); CAF_LOG_TRACE(CAF_ARG(hdl) << CAF_ARG(ep));
auto& vb = virtual_network_buffer(dst); auto& vb = virtual_network_buffer(hdl);
vb.emplace_back(ep, buf); vb.emplace_back(ep, buf);
read_data(dst); read_data(hdl);
} }
void test_multiplexer::exec_runnable() { void test_multiplexer::exec_runnable() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
resumable_ptr ptr; resumable_ptr ptr;
{ // critical section { // Critical section.
guard_type guard{mx_}; guard_type guard{mx_};
while (resumables_.empty()) while (resumables_.empty())
cv_.wait(guard); cv_.wait(guard);
...@@ -955,7 +1047,7 @@ bool test_multiplexer::try_exec_runnable() { ...@@ -955,7 +1047,7 @@ bool test_multiplexer::try_exec_runnable() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
resumable_ptr ptr; resumable_ptr ptr;
{ // critical section { // Critical section.
guard_type guard{mx_}; guard_type guard{mx_};
if (resumables_.empty()) if (resumables_.empty())
return false; return false;
...@@ -969,16 +1061,16 @@ bool test_multiplexer::try_exec_runnable() { ...@@ -969,16 +1061,16 @@ bool test_multiplexer::try_exec_runnable() {
void test_multiplexer::flush_runnables() { void test_multiplexer::flush_runnables() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
// execute runnables in bursts, pick a small size to // Execute runnables in bursts, pick a small size to
// minimize time in the critical section // minimize time in the critical section.
constexpr size_t max_runnable_count = 8; constexpr size_t max_runnable_count = 8;
std::vector<resumable_ptr> runnables; std::vector<resumable_ptr> runnables;
runnables.reserve(max_runnable_count); runnables.reserve(max_runnable_count);
// runnables can create new runnables, so we need to double-check // Runnables can create new runnables, so we need to double-check
// that `runnables_` is empty after each burst // that `runnables_` is empty after each burst.
do { do {
runnables.clear(); runnables.clear();
{ // critical section { // Critical section.
guard_type guard{mx_}; guard_type guard{mx_};
while (!resumables_.empty() && runnables.size() < max_runnable_count) { while (!resumables_.empty() && runnables.size() < max_runnable_count) {
runnables.emplace_back(std::move(resumables_.front())); runnables.emplace_back(std::move(resumables_.front()));
...@@ -1033,7 +1125,7 @@ void test_multiplexer::exec(resumable_ptr& ptr) { ...@@ -1033,7 +1125,7 @@ void test_multiplexer::exec(resumable_ptr& ptr) {
intrusive_ptr_release(ptr.get()); intrusive_ptr_release(ptr.get());
break; break;
default: default:
; // ignored ; // Ignored.
} }
} }
......
...@@ -25,6 +25,8 @@ ...@@ -25,6 +25,8 @@
#include <thread> #include <thread>
#include <vector> #include <vector>
#include "caf/test/io_dsl.hpp"
#include "caf/all.hpp" #include "caf/all.hpp"
#include "caf/io/all.hpp" #include "caf/io/all.hpp"
...@@ -39,6 +41,14 @@ using std::string; ...@@ -39,6 +41,14 @@ using std::string;
using ping_atom = atom_constant<atom("ping")>; using ping_atom = atom_constant<atom("ping")>;
using pong_atom = atom_constant<atom("pong")>; using pong_atom = atom_constant<atom("pong")>;
using set_atom = atom_constant<atom("set")>;
using begin_atom = atom_constant<atom("begin")>;
using middle_atom = atom_constant<atom("middle")>;
using end_atom = atom_constant<atom("end")>;
using msg_atom = atom_constant<atom("msg")>;
using done_atom = atom_constant<atom("shutdown")>;
/* /*
This test checks whether automatic connections work as expected This test checks whether automatic connections work as expected
...@@ -65,223 +75,501 @@ using pong_atom = atom_constant<atom("pong")>; ...@@ -65,223 +75,501 @@ using pong_atom = atom_constant<atom("pong")>;
*/ */
/* namespace {
std::thread run_prog(const char* arg, uint16_t port, bool use_asio) {
return detail::run_sub_unit_test(invalid_actor, constexpr uint16_t port_earth = 12340;
test::engine::path(), constexpr uint16_t port_mars = 12341;
test::engine::max_runtime(), constexpr uint16_t port_jupiter = 12342;
CAF_XSTR(CAF_SUITE),
use_asio, // Used for the tests with the test backend.
{"--port=" + std::to_string(port), arg}); class config : public actor_system_config {
} public:
config(bool use_tcp = true) {
load<caf::io::middleman, io::network::test_multiplexer>();
set("scheduler.policy", caf::atom("testing"));
set("middleman.detach-utility-actors", false);
set("middleman.enable-automatic-connections", true);
set("middleman.enable-tcp", use_tcp);
set("middleman.enable-udp", !use_tcp);
}
};
class config_udp : public config {
public:
config_udp() : config(false) {
// nop
}
};
// Used for the tests with the default multiplexer backend.
class simple_config : public actor_system_config {
public:
simple_config(bool use_tcp = true) {
load<caf::io::middleman>();
set("middleman.enable-automatic-connections", true);
set("middleman.enable-tcp", use_tcp);
set("middleman.enable-udp", !use_tcp);
}
};
class fixture {
public:
fixture(bool use_tcp = true)
: cfg_earth(use_tcp),
cfg_mars(use_tcp),
cfg_jupiter(use_tcp),
earth(cfg_earth),
mars(cfg_mars),
jupiter(cfg_jupiter) {
CAF_MESSAGE("Earth : " << to_string(earth.node()));
CAF_MESSAGE("Mars : " << to_string(mars.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.node()));
}
// we run the same code on all three nodes, a simple ping-pong client simple_config cfg_earth;
struct testee_state { simple_config cfg_mars;
std::set<actor> buddies; simple_config cfg_jupiter;
uint16_t port = 0; actor_system earth{cfg_earth};
const char* name = "testee"; actor_system mars{cfg_mars};
actor_system jupiter{cfg_jupiter};
}; };
behavior testee(stateful_actor<testee_state>* self) { class fixture_udp : public fixture {
public:
fixture_udp() : fixture(false) {
// nop
}
};
struct cache {
actor tmp;
};
behavior test_actor(stateful_actor<cache>* self, std::string location,
bool quit_directly) {
return { return {
[self](ping_atom, actor buddy, bool please_broadcast) -> message { [=](set_atom, actor val) {
if (please_broadcast) self->state.tmp = val;
for (auto& x : self->state.buddies) },
if (x != buddy) [=](begin_atom) {
send_as(buddy, x, ping_atom::value, buddy, false); CAF_REQUIRE(self->state.tmp);
self->state.buddies.emplace(std::move(buddy)); CAF_MESSAGE("starting messaging on " << location);
return make_message(pong_atom::value, self); self->send(self->state.tmp, middle_atom::value, self);
}, },
[self](pong_atom, actor buddy) { [=](middle_atom, actor start) {
self->state.buddies.emplace(std::move(buddy)); CAF_REQUIRE(self->state.tmp);
CAF_MESSAGE("forwaring message on " << location);
self->send(self->state.tmp, end_atom::value, start, self);
}, },
[self](put_atom, uint16_t new_port) { [=](end_atom, actor start, actor middle) {
self->state.port = new_port; CAF_MESSAGE("message arrived on " << location);
if (quit_directly) {
CAF_MESSAGE("telling other nodes to quit from " << location);
self->send(start, done_atom::value);
self->send(middle, done_atom::value);
self->send(self, done_atom::value);
} else {
CAF_MESSAGE("telling intermediate node to quit from " << location);
self->state.tmp = start;
self->send(middle, done_atom::value);
}
}, },
[self](get_atom) { [=](msg_atom) {
return self->state.port; CAF_REQUIRE(self->state.tmp);
CAF_MESSAGE("telling tmp actor to quit from " << location);
self->send(self->state.tmp, done_atom::value);
self->send(self, done_atom::value);
},
[=](done_atom) {
CAF_MESSAGE("actor on " << location << " is quitting");
self->quit();
} }
}; };
} }
void run_earth(bool use_asio, bool as_server, uint16_t pub_port) { } // namespace <anonymous>
scoped_actor self{system};
struct captain : hook {
public:
captain(actor parent) : parent_(std::move(parent)) {
// nop
}
void new_connection_established_cb(const node_id& node) override { CAF_TEST_FIXTURE_SCOPE(autoconn_tcp_simple_test, fixture)
anon_send(parent_, put_atom::value, node);
call_next<hook::new_connection_established>(node);
}
void new_remote_actor_cb(const actor_addr& addr) override {
anon_send(parent_, put_atom::value, addr);
call_next<hook::new_remote_actor>(addr);
}
void connection_lost_cb(const node_id& dest) override { CAF_TEST(build_triangle_simple_tcp) {
anon_send(parent_, delete_atom::value, dest); CAF_MESSAGE("setting up Earth");
} auto on_earth = earth.spawn(test_actor, "Earth", true);
auto earth_port = earth.middleman().publish(on_earth, 0);
CAF_REQUIRE(earth_port);
CAF_MESSAGE("Earth reachable via " << *earth_port);
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.middleman().remote_actor("localhost", *earth_port);
CAF_REQUIRE(from_earth);
auto on_mars = mars.spawn(test_actor, "Mars", true);
anon_send(on_mars, set_atom::value, *from_earth);
auto mars_port = mars.middleman().publish(on_mars, 0);
CAF_REQUIRE(mars_port);
CAF_MESSAGE("Mars reachable via " << *mars_port);
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.middleman().remote_actor("localhost", *mars_port);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, *from_mars);
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
jupiter.await_all_actors_done();
mars.await_all_actors_done();
earth.await_all_actors_done();
}
private: CAF_TEST(break_triangle_simple_tcp) {
actor parent_; actor on_earth;
}; actor on_jupiter;
middleman::instance()->add_hook<captain>(self); {
auto aut = system.spawn(testee); simple_config conf;
auto port = publish(aut, pub_port); actor_system mars(conf);
CAF_MESSAGE("published testee at port " << port); // Earth.
std::thread mars_process; CAF_MESSAGE("setting up Earth");
std::thread jupiter_process; on_earth = earth.spawn(test_actor, "Earth", false);
// launch process for Mars auto earth_port = earth.middleman().publish(on_earth, 0);
if (!as_server) { CAF_REQUIRE(earth_port);
CAF_MESSAGE("launch process for Mars"); CAF_MESSAGE("Earth reachable via " << *earth_port);
mars_process = run_prog("--mars", port, use_asio); // Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.middleman().remote_actor("localhost", *earth_port);
CAF_REQUIRE(from_earth);
auto on_mars = mars.spawn(test_actor, "Mars", false);
anon_send(on_mars, set_atom::value, *from_earth);
auto mars_port = mars.middleman().publish(on_mars, 0);
CAF_REQUIRE(mars_port);
CAF_MESSAGE("Mars reachable via " << *mars_port);
// Jupiter.
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.middleman().remote_actor("localhost", *mars_port);
CAF_REQUIRE(from_mars);
on_jupiter = jupiter.spawn(test_actor, "Jupiter", false);
anon_send(on_jupiter, set_atom::value, *from_mars);
// Trigger the connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
mars.await_all_actors_done();
// Leaving the scope will shutdown Mars.
} }
CAF_MESSAGE("wait for Mars to connect"); // Let the remaining nodes communicate.
node_id mars; anon_send(on_earth, msg_atom::value);
self->receive( jupiter.await_all_actors_done();
[&](put_atom, const node_id& nid) { earth.await_all_actors_done();
mars = nid; }
CAF_MESSAGE(CAF_ARG(mars));
} CAF_TEST_FIXTURE_SCOPE_END()
);
actor_addr mars_addr; CAF_TEST_FIXTURE_SCOPE(autoconn_udp_simple_test, fixture_udp)
uint16_t mars_port;
self->receive_while([&] { return mars_addr == invalid_actor_addr; })( CAF_TEST_DISABLED(build_triangle_simple_udp) {
[&](put_atom, const actor_addr& addr) { CAF_MESSAGE("setting up Earth");
auto hdl = actor_cast<actor>(addr); auto on_earth = earth.spawn(test_actor, "Earth", true);
self->request(hdl, sys_atom::value, get_atom::value, "info").then( auto earth_port = earth.middleman().publish_udp(on_earth, 0);
[&](ok_atom, const string&, const actor_addr&, const string& name) { CAF_REQUIRE(earth_port);
if (name != "testee") CAF_MESSAGE("Earth reachable via " << *earth_port);
return; CAF_MESSAGE("setting up Mars");
mars_addr = addr; auto from_earth = mars.middleman().remote_actor_udp("localhost", *earth_port);
CAF_MESSAGE(CAF_ARG(mars_addr)); CAF_REQUIRE(from_earth);
self->request(actor_cast<actor>(mars_addr), get_atom::value).then( auto on_mars = mars.spawn(test_actor, "Mars", true);
[&](uint16_t mp) { anon_send(on_mars, set_atom::value, *from_earth);
CAF_MESSAGE("mars published its actor at port " << mp); auto mars_port = mars.middleman().publish_udp(on_mars, 0);
mars_port = mp; CAF_REQUIRE(mars_port);
} CAF_MESSAGE("Mars reachable via " << *mars_port);
); CAF_MESSAGE("setting up Jupiter");
} auto from_mars = jupiter.middleman().remote_actor_udp("localhost", *mars_port);
); CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, *from_mars);
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
jupiter.await_all_actors_done();
mars.await_all_actors_done();
earth.await_all_actors_done();
}
CAF_TEST_DISABLED(break_triangle_simple_udp) {
actor on_earth;
actor on_jupiter;
{
// Use UDP instead of TCP.
simple_config conf(false);
actor_system mars(conf);
// Earth.
CAF_MESSAGE("setting up Earth");
on_earth = earth.spawn(test_actor, "Earth", false);
auto earth_port = earth.middleman().publish_udp(on_earth, 0);
CAF_REQUIRE(earth_port);
CAF_MESSAGE("Earth reachable via " << *earth_port);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.middleman().remote_actor_udp("localhost", *earth_port);
if (!from_earth) {
CAF_MESSAGE("Failed to contact earth: " << mars.render(from_earth.error()));
} }
); CAF_REQUIRE(from_earth);
// launch process for Jupiter auto on_mars = mars.spawn(test_actor, "Mars", false);
if (!as_server) { anon_send(on_mars, set_atom::value, *from_earth);
CAF_MESSAGE("launch process for Jupiter"); auto mars_port = mars.middleman().publish_udp(on_mars, 0);
jupiter_process = run_prog("--jupiter", mars_port, use_asio); CAF_REQUIRE(mars_port);
CAF_MESSAGE("Mars reachable via " << *mars_port);
// Jupiter.
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.middleman().remote_actor_udp("localhost", *mars_port);
CAF_REQUIRE(from_mars);
on_jupiter = jupiter.spawn(test_actor, "Jupiter", false);
anon_send(on_jupiter, set_atom::value, *from_mars);
// Trigger the connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
mars.await_all_actors_done();
// Leaving the scope will shutdown Mars.
} }
CAF_MESSAGE("wait for Jupiter to connect"); // Let the remaining nodes communicate.
self->receive( anon_send(on_earth, msg_atom::value);
[](put_atom, const node_id& jupiter) { jupiter.await_all_actors_done();
CAF_MESSAGE(CAF_ARG(jupiter)); earth.await_all_actors_done();
}
);
actor_addr jupiter_addr;
self->receive_while([&] { return jupiter_addr == invalid_actor_addr; })(
[&](put_atom, const actor_addr& addr) {
auto hdl = actor_cast<actor>(addr);
self->request(hdl, sys_atom::value, get_atom::value, "info").then(
[&](ok_atom, const string&, const actor_addr&, const string& name) {
if (name != "testee")
return;
jupiter_addr = addr;
CAF_MESSAGE(CAF_ARG(jupiter_addr));
}
);
}
);
CAF_MESSAGE("shutdown Mars");
anon_send_exit(mars_addr, exit_reason::kill);
if (mars_process.joinable())
mars_process.join();
self->receive(
[&](delete_atom, const node_id& nid) {
CAF_CHECK(nid == mars);
}
);
CAF_MESSAGE("check whether we still can talk to Jupiter");
self->send(aut, ping_atom::value, self, true);
std::set<actor_addr> found;
int i = 0;
self->receive_for(i, 2)(
[&](pong_atom, const actor&) {
found.emplace(self->current_sender());
}
);
std::set<actor_addr> expected{aut.address(), jupiter_addr};
CAF_CHECK(found == expected);
CAF_MESSAGE("shutdown Jupiter");
anon_send_exit(jupiter_addr, exit_reason::kill);
if (jupiter_process.joinable())
jupiter_process.join();
anon_send_exit(aut, exit_reason::kill);
} }
void run_mars(uint16_t port_to_earth, uint16_t pub_port) { CAF_TEST_FIXTURE_SCOPE_END()
auto aut = system.spawn(testee);
auto port = publish(aut, pub_port); CAF_TEST_FIXTURE_SCOPE(autoconn_tcp_test, belt_fixture<>)
anon_send(aut, put_atom::value, port);
CAF_MESSAGE("published testee at port " << port); CAF_TEST_DISABLED(build_triangle_tcp) {
auto earth = remote_actor("localhost", port_to_earth); CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
send_as(aut, earth, ping_atom::value, aut, false); CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", true);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
prepare_connection(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish dummy on earth");
earth.publish(on_earth, port_earth);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.remote_actor("earth", port_earth);
CAF_REQUIRE(from_earth);
auto on_mars = mars.sys.spawn(test_actor, "Mars", true);
anon_send(on_mars, set_atom::value, from_earth);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
prepare_connection(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish dummy on mars");
mars.publish(on_mars, port_mars);
// Jupiter
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.remote_actor("mars", port_mars);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, from_mars);
exec_all();
// This handle will be created by the test multiplexer for the automatically
// opened socket when automatic connections are enabled.
auto hdl_jupiter = accept_handle::from_int(std::numeric_limits<int64_t>::max());
// Prepare automatic connection between Jupiter and Earth.
prepare_connection(jupiter, earth, "jupiter", port_jupiter, hdl_jupiter);
// Add the address information for this test to the config server on Mars.
auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
network::address_listing interfaces{
{network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::tcp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
} }
void run_jupiter(uint16_t port_to_mars) { /*
auto aut = system.spawn(testee);
auto mars = remote_actor("localhost", port_to_mars); CAF_TEST(break_triangle_tcp) {
send_as(aut, mars, ping_atom::value, aut, true); CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", false);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
prepare_connection(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish dummy on earth");
earth.publish(on_earth, port_earth);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.remote_actor("earth", port_earth);
CAF_REQUIRE(from_earth);
auto on_mars = mars.sys.spawn(test_actor, "Mars", false);
anon_send(on_mars, set_atom::value, from_earth);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
prepare_connection(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish dummy on mars");
mars.publish(on_mars, port_mars);
// Jupiter.
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.remote_actor("mars", port_mars);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", false);
anon_send(on_jupiter, set_atom::value, from_mars);
exec_all();
// This handle will be created by the test multiplexer for the automatically
// opened socket when automatic connections are enabled.
auto hdl_jupiter = accept_handle::from_int(std::numeric_limits<int64_t>::max());
// Prepare automatic connection between Jupiter and Earth.
prepare_connection(jupiter, earth, "jupiter", port_jupiter, hdl_jupiter);
// Add the address information for this test to the config server on Mars.
auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
network::address_listing interfaces{
{network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::tcp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup between the edge nodes.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
// Shutdown the basp broker of the intermediate node.
anon_send_exit(mars.basp, exit_reason::kill);
exec_all();
// Let the remaining nodes communicate.
anon_send(on_earth, msg_atom::value);
exec_all();
} }
*/
CAF_TEST(triangle_setup) { CAF_TEST_FIXTURE_SCOPE_END()
// this unit test is temporarily disabled until problems
// with OBS are sorted out or new actor_system API is in place CAF_TEST_FIXTURE_SCOPE(autoconn_udp_test, belt_fixture_t<config_udp>)
CAF_TEST(build_triangle_udp) {
CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", true);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
prepare_endpoints(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish_udp dummy on earth");
earth.publish_udp(on_earth, port_earth);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.remote_actor_udp("earth", port_earth);
CAF_REQUIRE(from_earth);
auto on_mars = mars.sys.spawn(test_actor, "Mars", true);
anon_send(on_mars, set_atom::value, from_earth);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
datagram_handle mj, jm;
prepare_endpoints(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish_udp dummy on mars");
mars.publish_udp(on_mars, port_mars);
// Jupiter
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.remote_actor_udp("mars", port_mars);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, from_mars);
exec_all();
// This handle will be created by the test multiplexer for the automatically
// opened socket when automatic connections are enabled.
auto hdl_jup = datagram_handle::from_int(std::numeric_limits<int64_t>::max());
// Prepare automatic connection between Jupiter and Earth.
datagram_handle je, ej;
prepare_endpoints(jupiter, earth, "jupiter", port_jupiter, hdl_jup);
// Add the address information for this test to the config server on Mars.
auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
network::address_listing interfaces{
{network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::udp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
} }
/* CAF_TEST(break_triangle_udp) {
CAF_TEST(triangle_setup) { CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
uint16_t port = 0; CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
uint16_t publish_port = 0; CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
auto argv = test::engine::argv(); // Earth.
auto argc = test::engine::argc(); CAF_MESSAGE("setting up Earth");
auto r = message_builder(argv, argv + argc).extract_opts({ auto on_earth = earth.sys.spawn(test_actor, "Earth", false);
{"port,p", "port of remote side (when running mars or jupiter)", port}, CAF_MESSAGE("run initialization code");
{"mars", "run mars"}, exec_all();
{"jupiter", "run jupiter"}, CAF_MESSAGE("prepare endpoints");
{"use-asio", "use ASIO network backend (if available)"}, prepare_endpoints(earth, mars, "earth", port_earth);
{"server,s", "run in server mode (don't run clients)", publish_port} CAF_MESSAGE("publish_udp dummy on earth");
}); earth.publish_udp(on_earth, port_earth);
// check arguments // Mars.
bool is_mars = r.opts.count("mars") > 0; CAF_MESSAGE("setting up Mars");
bool is_jupiter = r.opts.count("jupiter") > 0; auto from_earth = mars.remote_actor_udp("earth", port_earth);
bool has_port = r.opts.count("port") > 0; CAF_REQUIRE(from_earth);
if (((is_mars || is_jupiter) && !has_port) || (is_mars && is_jupiter)) { auto on_mars = mars.sys.spawn(test_actor, "Mars", false);
CAF_ERROR("need a port when running Mars or Jupiter and cannot " anon_send(on_mars, set_atom::value, from_earth);
"both at the same time"); CAF_MESSAGE("run initialization code");
return; exec_all();
} CAF_MESSAGE("prepare endpoints");
// enable automatic connections prepare_endpoints(mars, jupiter, "mars", port_mars);
anon_send(whereis(atom("ConfigServ")), put_atom::value, CAF_MESSAGE("publish_udp dummy on mars");
"middleman.enable-automatic-connections", make_message(true)); mars.publish_udp(on_mars, port_mars);
auto use_asio = r.opts.count("use-asio") > 0; // Jupiter.
# ifdef CAF_USE_ASIO CAF_MESSAGE("setting up Jupiter");
if (use_asio) { auto from_mars = jupiter.remote_actor_udp("mars", port_mars);
CAF_MESSAGE("enable ASIO backend"); CAF_REQUIRE(from_mars);
set_middleman<network::asio_multiplexer>(); auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", false);
} anon_send(on_jupiter, set_atom::value, from_mars);
# endif // CAF_USE_ASIO exec_all();
auto as_server = r.opts.count("server") > 0; // This handle will be created by the test multiplexer for the automatically
if (is_mars) // opened socket when automatic connections are enabled.
run_mars(port, publish_port); auto hdl_jup = datagram_handle::from_int(std::numeric_limits<int64_t>::max());
else if (is_jupiter) // Prepare automatic connection between Jupiter and Earth.
run_jupiter(port); prepare_endpoints(jupiter, earth, "jupiter", port_jupiter, hdl_jup);
else // Add the address information for this test to the config server on Mars.
run_earth(use_asio, as_server, publish_port); auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
await_all_actors_done(); network::address_listing interfaces{
shutdown(); {network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::udp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup between the edge nodes.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
// Shutdown the basp broker of the intermediate node.
anon_send_exit(mars.basp, exit_reason::kill);
exec_all();
// Let the remaining nodes communicate.
anon_send(on_earth, msg_atom::value);
exec_all();
} }
*/ */
CAF_TEST_FIXTURE_SCOPE_END()
...@@ -263,7 +263,8 @@ public: ...@@ -263,7 +263,8 @@ public:
void connect_node(node& n, void connect_node(node& n,
optional<accept_handle> ax = none, optional<accept_handle> ax = none,
actor_id published_actor_id = invalid_actor_id, actor_id published_actor_id = invalid_actor_id,
const set<string>& published_actor_ifs = std::set<std::string>{}) { const set<string>& published_actor_ifs = std::set<std::string>{},
const basp::routing_table::address_map& local_addrs_map = {}) {
auto src = ax ? *ax : ahdl_; auto src = ax ? *ax : ahdl_;
CAF_MESSAGE("connect remote node " << n.name CAF_MESSAGE("connect remote node " << n.name
<< ", connection ID = " << n.connection.id() << ", connection ID = " << n.connection.id()
...@@ -276,30 +277,31 @@ public: ...@@ -276,30 +277,31 @@ public:
mock(hdl, mock(hdl,
{basp::message_type::client_handshake, 0, 0, 0, {basp::message_type::client_handshake, 0, 0, 0,
n.id, this_node(), n.id, this_node(),
invalid_actor_id, invalid_actor_id}, std::string{}) invalid_actor_id, invalid_actor_id}, std::string{},
basp::routing_table::address_map{})
.receive(hdl, .receive(hdl,
basp::message_type::server_handshake, no_flags, basp::message_type::server_handshake, no_flags,
any_vals, basp::version, this_node(), node_id{none}, any_vals, basp::version, this_node(), node_id{none},
published_actor_id, invalid_actor_id, std::string{}, published_actor_id, invalid_actor_id, std::string{},
published_actor_id, published_actor_id,
published_actor_ifs) published_actor_ifs,
local_addrs_map)
// upon receiving our client handshake, BASP will check // upon receiving our client handshake, BASP will check
// whether there is a SpawnServ actor on this node // whether there is a SpawnServ actor on this node
.receive(hdl, .receive(hdl,
basp::message_type::dispatch_message, basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, basp::header::named_receiver_flag, any_vals,
default_operation_data, no_operation_data,
this_node(), n.id, this_node(), n.id,
any_vals, invalid_actor_id, any_vals, invalid_actor_id,
spawn_serv_atom, spawn_serv_atom,
std::vector<actor_addr>{}, std::vector<actor_addr>{},
make_message(sys_atom::value, get_atom::value, "info")); make_message(sys_atom::value, get_atom::value, "info"));
// test whether basp instance correctly updates the // test whether basp instance correctly updates the
// routing table upon receiving client handshakes // routing table upon receiving client handshakes
auto path = tbl().lookup(n.id); auto res = tbl().lookup(n.id);
CAF_REQUIRE(path); CAF_REQUIRE(res.hdl);
CAF_CHECK_EQUAL(path->hdl, n.connection); CAF_CHECK_EQUAL(*res.hdl, n.connection);
CAF_CHECK_EQUAL(path->next_hop, n.id);
} }
std::pair<basp::header, buffer> read_from_out_buf(connection_handle hdl) { std::pair<basp::header, buffer> read_from_out_buf(connection_handle hdl) {
...@@ -440,12 +442,16 @@ public: ...@@ -440,12 +442,16 @@ public:
scheduler_type& sched; scheduler_type& sched;
middleman_actor mma; middleman_actor mma;
actor_system_config phobos_cfg;
actor_system phobos;
scoped_actor on_phobos;
autoconn_enabled_fixture() autoconn_enabled_fixture()
: fixture(true), : fixture(true),
sched(dynamic_cast<scheduler_type&>(sys.scheduler())), sched(dynamic_cast<scheduler_type&>(sys.scheduler())),
mma(sys.middleman().actor_handle()) { mma(sys.middleman().actor_handle()),
phobos(phobos_cfg.load<caf::io::middleman>()),
on_phobos(phobos) {
// nop // nop
} }
...@@ -495,7 +501,8 @@ CAF_TEST(non_empty_server_handshake) { ...@@ -495,7 +501,8 @@ CAF_TEST(non_empty_server_handshake) {
basp::version, this_node(), none, basp::version, this_node(), none,
self()->id(), invalid_actor_id}; self()->id(), invalid_actor_id};
to_buf(expected_buf, expected, nullptr, std::string{}, to_buf(expected_buf, expected, nullptr, std::string{},
self()->id(), set<string>{"caf::replies_to<@u16>::with<@u16>"}); self()->id(), set<string>{"caf::replies_to<@u16>::with<@u16>"},
basp::routing_table::address_map{});
CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf)); CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf));
} }
...@@ -555,24 +562,6 @@ CAF_TEST(client_handshake_and_dispatch) { ...@@ -555,24 +562,6 @@ CAF_TEST(client_handshake_and_dispatch) {
); );
} }
CAF_TEST(message_forwarding) {
// connect two remote nodes
connect_node(jupiter());
connect_node(mars());
auto msg = make_message(1, 2, 3);
// send a message from node 0 to node 1, forwarded by this node
mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, default_operation_data,
jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id()},
msg)
.receive(mars().connection,
basp::message_type::dispatch_message, no_flags, any_vals,
default_operation_data, jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id(),
msg);
}
CAF_TEST(publish_and_connect) { CAF_TEST(publish_and_connect) {
auto ax = accept_handle::from_int(4242); auto ax = accept_handle::from_int(4242);
mpx()->provide_acceptor(4242, ax); mpx()->provide_acceptor(4242, ax);
...@@ -604,23 +593,25 @@ CAF_TEST(remote_actor_and_send) { ...@@ -604,23 +593,25 @@ CAF_TEST(remote_actor_and_send) {
jupiter().dummy_actor->id(), invalid_actor_id}, jupiter().dummy_actor->id(), invalid_actor_id},
std::string{}, std::string{},
jupiter().dummy_actor->id(), jupiter().dummy_actor->id(),
uint32_t{0}) uint32_t{0},
basp::routing_table::address_map{})
.receive(jupiter().connection, .receive(jupiter().connection,
basp::message_type::client_handshake, no_flags, 1u, basp::message_type::client_handshake, no_flags, 2u,
no_operation_data, this_node(), jupiter().id, no_operation_data, this_node(), jupiter().id,
invalid_actor_id, invalid_actor_id, std::string{}) invalid_actor_id, invalid_actor_id, std::string{},
basp::routing_table::address_map{})
.receive(jupiter().connection, .receive(jupiter().connection,
basp::message_type::dispatch_message, basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, basp::header::named_receiver_flag, any_vals,
default_operation_data, this_node(), jupiter().id, no_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id, any_vals, invalid_actor_id,
spawn_serv_atom, spawn_serv_atom,
std::vector<actor_id>{}, std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info")) make_message(sys_atom::value, get_atom::value, "info"))
.receive(jupiter().connection, .receive(jupiter().connection,
basp::message_type::announce_proxy, no_flags, no_payload, basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id, no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id()); invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("BASP broker should've send the proxy"); CAF_MESSAGE("BASP broker should've send the proxy");
f.receive( f.receive(
[&](node_id nid, strong_actor_ptr res, std::set<std::string> ifs) { [&](node_id nid, strong_actor_ptr res, std::set<std::string> ifs) {
...@@ -708,169 +699,27 @@ CAF_TEST(actor_serialize_and_deserialize) { ...@@ -708,169 +699,27 @@ CAF_TEST(actor_serialize_and_deserialize) {
msg); msg);
} }
CAF_TEST(indirect_connections) {
// this node receives a message from jupiter via mars and responds via mars
// and any ad-hoc automatic connection requests are ignored
CAF_MESSAGE("self: " << to_string(self()->address()));
CAF_MESSAGE("publish self at port 4242");
auto ax = accept_handle::from_int(4242);
mpx()->provide_acceptor(4242, ax);
sys.middleman().publish(self(), 4242);
mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to Mars");
connect_node(mars(), ax, self()->id());
CAF_MESSAGE("actor from Jupiter sends a message to us via Mars");
auto mx = mock(mars().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id()},
std::vector<actor_id>{},
make_message("hello from jupiter!"));
CAF_MESSAGE("expect ('sys', 'get', \"info\") from Earth to Jupiter at Mars");
// this asks Jupiter if it has a 'SpawnServ'
mx.receive(mars().connection,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
default_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"));
CAF_MESSAGE("expect announce_proxy message at Mars from Earth to Jupiter");
mx.receive(mars().connection,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("receive message from jupiter");
self()->receive(
[](const std::string& str) -> std::string {
CAF_CHECK_EQUAL(str, "hello from jupiter!");
return "hello from earth!";
}
);
mpx()->exec_runnable(); // process forwarded message in basp_broker
mock()
.receive(mars().connection,
basp::message_type::dispatch_message, no_flags, any_vals,
default_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
}
CAF_TEST_FIXTURE_SCOPE_END() CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(basp_tests_with_autoconn, autoconn_enabled_fixture) CAF_TEST_FIXTURE_SCOPE(basp_tests_with_autoconn, autoconn_enabled_fixture)
CAF_TEST(automatic_connection) { CAF_TEST(address_handshake) {
// this tells our BASP broker to enable the automatic connection feature // Test whether basp instance correctly sends a server handshake
//anon_send(aut(), ok_atom::value, // when there's no actor published and automatic connections are enabled.
// "middleman.enable-automatic-connections", make_message(true)); buffer buf;
//mpx()->exec_runnable(); // process publish message in basp_broker instance().write_server_handshake(mpx(), buf, none);
// jupiter [remote hdl 0] -> mars [remote hdl 1] -> earth [this_node] auto addrs = instance().tbl().local_addresses();
// (this node receives a message from jupiter via mars and responds via mars, CAF_CHECK(!addrs.empty());
// but then also establishes a connection to jupiter directly) CAF_CHECK(addrs.count(network::protocol::tcp) > 0 &&
auto check_node_in_tbl = [&](node& n) { !addrs[network::protocol::tcp].second.empty());
io::id_visitor id_vis; CAF_CHECK(addrs.count(network::protocol::udp) == 0);
auto hdl = tbl().lookup_direct(n.id); buffer expected_buf;
CAF_REQUIRE(hdl); basp::header expected{basp::message_type::server_handshake, 0, 0,
CAF_CHECK_EQUAL(visit(id_vis, *hdl), n.connection.id()); basp::version, this_node(), none,
}; invalid_actor_id, invalid_actor_id};
mpx()->provide_scribe("jupiter", 8080, jupiter().connection); to_buf(expected_buf, expected, nullptr, std::string{},
CAF_CHECK(mpx()->has_pending_scribe("jupiter", 8080)); invalid_actor_id, set<string>{}, addrs);
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf));
auto ax = accept_handle::from_int(4242);
mpx()->provide_acceptor(4242, ax);
publish(self(), 4242);
mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to mars");
connect_node(mars(), ax, self()->id());
//CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id());
check_node_in_tbl(mars());
CAF_MESSAGE("simulate that an actor from jupiter "
"sends a message to us via mars");
mock(mars().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id()},
std::vector<actor_id>{},
make_message("hello from jupiter!"))
.receive(mars().connection,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
default_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id, spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"))
.receive(mars().connection,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
default_operation_data, this_node(), jupiter().id,
any_vals, // actor ID of an actor spawned by the BASP broker
invalid_actor_id,
config_serv_atom,
std::vector<actor_id>{},
make_message(get_atom::value, "basp.default-connectivity-tcp"))
.receive(mars().connection,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u);
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), mars().id);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
auto connection_helper_actor = sys.latest_actor_id();
CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u);
// create a dummy config server and respond to the name lookup
CAF_MESSAGE("receive ConfigServ of jupiter");
network::address_listing res;
res[network::protocol::ipv4].emplace_back("jupiter");
mock(mars().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
this_node(), this_node(),
invalid_actor_id, connection_helper_actor},
std::vector<actor_id>{},
make_message("basp.default-connectivity-tcp",
make_message(uint16_t{8080}, std::move(res))));
// our connection helper should now connect to jupiter and
// send the scribe handle over to the BASP broker
while (mpx()->has_pending_scribe("jupiter", 8080)) {
sched.run();
mpx()->flush_runnables();
}
CAF_REQUIRE(mpx()->output_buffer(mars().connection).empty());
// send handshake from jupiter
mock(jupiter().connection,
{basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id},
std::string{},
jupiter().dummy_actor->id(),
uint32_t{0})
.receive(jupiter().connection,
basp::message_type::client_handshake, no_flags, 1u,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, invalid_actor_id, std::string{});
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), none);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
check_node_in_tbl(jupiter());
check_node_in_tbl(mars());
CAF_MESSAGE("receive message from jupiter");
self()->receive(
[](const std::string& str) -> std::string {
CAF_CHECK_EQUAL(str, "hello from jupiter!");
return "hello from earth!";
}
);
mpx()->exec_runnable(); // process forwarded message in basp_broker
CAF_MESSAGE("response message must take direct route now");
mock()
.receive(jupiter().connection,
basp::message_type::dispatch_message, no_flags, any_vals,
default_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u);
} }
CAF_TEST_FIXTURE_SCOPE_END() CAF_TEST_FIXTURE_SCOPE_END()
...@@ -76,7 +76,7 @@ constexpr uint64_t no_operation_data = 0; ...@@ -76,7 +76,7 @@ constexpr uint64_t no_operation_data = 0;
constexpr auto basp_atom = caf::atom("BASP"); constexpr auto basp_atom = caf::atom("BASP");
constexpr auto spawn_serv_atom = caf::atom("SpawnServ"); constexpr auto spawn_serv_atom = caf::atom("SpawnServ");
constexpr auto config_serv_atom = caf::atom("ConfigServ"); constexpr auto peer_serv_atom = caf::atom("PeerServ");
} // namespace <anonymous> } // namespace <anonymous>
...@@ -210,6 +210,10 @@ public: ...@@ -210,6 +210,10 @@ public:
return dhdl_; return dhdl_;
} }
intptr_t default_sender() {
return default_sender_;
}
// implementation of the Binary Actor System Protocol // implementation of the Binary Actor System Protocol
basp::instance& instance() { basp::instance& instance() {
return aut()->state.instance; return aut()->state.instance;
...@@ -274,9 +278,11 @@ public: ...@@ -274,9 +278,11 @@ public:
void establish_communication(node& n, void establish_communication(node& n,
optional<datagram_handle> dx = none, optional<datagram_handle> dx = none,
optional<intptr_t> endpoint_id = none,
actor_id published_actor_id = invalid_actor_id, actor_id published_actor_id = invalid_actor_id,
const set<string>& published_actor_ifs const set<string>& published_actor_ifs
= std::set<std::string>{}) { = set<std::string>{},
const basp::routing_table::address_map& am = {}) {
auto src = dx ? *dx : dhdl_; auto src = dx ? *dx : dhdl_;
CAF_MESSAGE("establish communication on node " << n.name CAF_MESSAGE("establish communication on node " << n.name
<< ", delegated servant ID = " << n.endpoint.id() << ", delegated servant ID = " << n.endpoint.id()
...@@ -284,20 +290,26 @@ public: ...@@ -284,20 +290,26 @@ public:
// send the client handshake and receive the server handshake // send the client handshake and receive the server handshake
// and a dispatch_message as answers // and a dispatch_message as answers
auto hdl = n.endpoint; auto hdl = n.endpoint;
mpx_->add_pending_endpoint(src, hdl); auto ep = endpoint_id ? *endpoint_id : default_sender_;
CAF_MESSAGE("Send client handshake"); mpx_->add_pending_endpoint(ep, hdl);
mock(src, hdl, CAF_MESSAGE("send client handshake");
mock(src, ep,
{basp::message_type::client_handshake, 0, 0, 0, {basp::message_type::client_handshake, 0, 0, 0,
n.id, this_node(), n.id, this_node(), invalid_actor_id, invalid_actor_id},
invalid_actor_id, invalid_actor_id}, std::string{}) std::string{}, basp::routing_table::address_map{})
// upon receiving the client handshake, the server should answer // upon receiving the client handshake, the server should answer
// with the server handshake and send the dispatch_message blow // with the server handshake and send the dispatch_message blow
.receive(hdl, .receive(hdl,
basp::message_type::server_handshake, no_flags, basp::message_type::server_handshake, no_flags,
any_vals, basp::version, this_node(), node_id{none}, any_vals, basp::version, this_node(), node_id{none},
published_actor_id, invalid_actor_id, std::string{}, published_actor_id, invalid_actor_id, std::string{},
published_actor_id, published_actor_ifs) published_actor_id, published_actor_ifs, am);
// upon receiving our client handshake, BASP will check // UDP uses a three-way handshake, so let's answer with the final message.
mock(src, ep,
{basp::message_type::acknowledge_handshake, 0,0,0,
n.id, this_node(), invalid_actor_id, invalid_actor_id,
1})
// upon receiving our acknowledge handshake, BASP will check
// whether there is a SpawnServ actor on this node // whether there is a SpawnServ actor on this node
.receive(hdl, .receive(hdl,
basp::message_type::dispatch_message, basp::message_type::dispatch_message,
...@@ -310,10 +322,9 @@ public: ...@@ -310,10 +322,9 @@ public:
make_message(sys_atom::value, get_atom::value, "info")); make_message(sys_atom::value, get_atom::value, "info"));
// test whether basp instance correctly updates the // test whether basp instance correctly updates the
// routing table upon receiving client handshakes // routing table upon receiving client handshakes
auto path = tbl().lookup(n.id); auto res = tbl().lookup(n.id);
CAF_REQUIRE(path); CAF_REQUIRE(res.hdl);
CAF_CHECK_EQUAL(path->hdl, n.endpoint); CAF_CHECK_EQUAL(*res.hdl, n.endpoint);
CAF_CHECK_EQUAL(path->next_hop, n.id);
} }
std::pair<basp::header, buffer> read_from_out_buf(datagram_handle hdl) { std::pair<basp::header, buffer> read_from_out_buf(datagram_handle hdl) {
...@@ -377,11 +388,8 @@ public: ...@@ -377,11 +388,8 @@ public:
while (oq.empty()) while (oq.empty())
this_->mpx()->exec_runnable(); this_->mpx()->exec_runnable();
CAF_MESSAGE("output queue has " << oq.size() << " messages"); CAF_MESSAGE("output queue has " << oq.size() << " messages");
auto dh = oq.front().first;
buffer& ob = oq.front().second; buffer& ob = oq.front().second;
CAF_MESSAGE("next datagram has " << ob.size() CAF_REQUIRE_EQUAL(this_->mpx()->endpoint_id(hdl), oq.front().first);
<< " bytes, servant ID = " << dh.id());
CAF_CHECK_EQUAL(dh.id(), hdl.id());
basp::header hdr; basp::header hdr;
{ // lifetime scope of source { // lifetime scope of source
binary_deserializer source{this_->mpx(), ob}; binary_deserializer source{this_->mpx(), ob};
...@@ -412,39 +420,41 @@ public: ...@@ -412,39 +420,41 @@ public:
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, datagram_handle ep, mock_t& enqueue_back(datagram_handle hdl, intptr_t sender_id,
basp::header hdr, const Ts&... xs) { basp::header hdr, const Ts&... xs) {
buffer buf; buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...); this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation) CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload to back of queue"); << " bytes payload to back of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_back(ep, buf); this_->mpx()->virtual_network_buffer(hdl).emplace_back(sender_id, buf);
return *this; return *this;
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, basp::header hdr, mock_t& enqueue_back(datagram_handle hdl, basp::header hdr,
Ts&&... xs) { Ts&&... xs) {
return enqueue_back(hdl, hdl, hdr, std::forward<Ts>(xs)...); return enqueue_back(hdl, this_->default_sender(), hdr,
std::forward<Ts>(xs)...);
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, datagram_handle ep, mock_t& enqueue_front(datagram_handle hdl, intptr_t sender_id,
basp::header hdr, const Ts&... xs) { basp::header hdr, const Ts&... xs) {
buffer buf; buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...); this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation) CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload to front of queue"); << " bytes payload to front of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_front(ep, buf); this_->mpx()->virtual_network_buffer(hdl).emplace_front(sender_id, buf);
return *this; return *this;
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, basp::header hdr, mock_t& enqueue_front(datagram_handle hdl, basp::header hdr,
Ts&&... xs) { Ts&&... xs) {
return enqueue_front(hdl, hdl, hdr, std::forward<Ts>(xs)...); return enqueue_front(hdl, this_->default_sender(), hdr,
std::forward<Ts>(xs)...);
} }
mock_t& deliver(datagram_handle hdl, size_t num_messages = 1) { mock_t& deliver(datagram_handle hdl, size_t num_messages = 1) {
...@@ -466,19 +476,19 @@ public: ...@@ -466,19 +476,19 @@ public:
CAF_MESSAGE("virtually send " << to_string(hdr.operation) CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload"); << " bytes payload");
mpx()->virtual_send(hdl, hdl, buf); mpx()->virtual_send(hdl, default_sender_, buf);
return {this}; return {this};
} }
template <class... Ts> template <class... Ts>
mock_t mock(datagram_handle hdl, datagram_handle ep, basp::header hdr, mock_t mock(datagram_handle hdl, intptr_t sender_id, basp::header hdr,
const Ts&... xs) { const Ts&... xs) {
buffer buf; buffer buf;
to_buf(buf, hdr, nullptr, xs...); to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("virtually send " << to_string(hdr.operation) CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload"); << " bytes payload");
mpx()->virtual_send(hdl, ep, buf); mpx()->virtual_send(hdl, sender_id, buf);
return {this}; return {this};
} }
...@@ -492,6 +502,7 @@ public: ...@@ -492,6 +502,7 @@ public:
private: private:
basp_broker* aut_; basp_broker* aut_;
datagram_handle dhdl_; datagram_handle dhdl_;
intptr_t default_sender_ = static_cast<intptr_t>(0xdeadbeef);
network::test_multiplexer* mpx_; network::test_multiplexer* mpx_;
node_id this_node_; node_id this_node_;
unique_ptr<scoped_actor> self_; unique_ptr<scoped_actor> self_;
...@@ -575,15 +586,15 @@ CAF_TEST_DISABLED(empty_server_handshake_udp) { ...@@ -575,15 +586,15 @@ CAF_TEST_DISABLED(empty_server_handshake_udp) {
CAF_CHECK_EQUAL(to_string(hdr), to_string(expected)); CAF_CHECK_EQUAL(to_string(hdr), to_string(expected));
} }
CAF_TEST_DISABLED(empty_client_handshake_udp) { CAF_TEST_DISABLED(empty_acknowledge_handshake_udp) {
// test whether basp instance correctly sends a // test whether a basp instance correctly sends an
// client handshake when there's no actor published // acknowldge handshake
buffer buf; buffer buf;
instance().write_client_handshake(mpx(), buf, none); instance().write_acknowledge_handshake(mpx(), buf, none);
basp::header hdr; basp::header hdr;
buffer payload; buffer payload;
std::tie(hdr, payload) = from_buf(buf); std::tie(hdr, payload) = from_buf(buf);
basp::header expected{basp::message_type::client_handshake, 0, basp::header expected{basp::message_type::acknowledge_handshake, 0,
static_cast<uint32_t>(payload.size()), 0, static_cast<uint32_t>(payload.size()), 0,
this_node(), none, this_node(), none,
invalid_actor_id, invalid_actor_id, invalid_actor_id, invalid_actor_id,
...@@ -607,7 +618,8 @@ CAF_TEST_DISABLED(non_empty_server_handshake_udp) { ...@@ -607,7 +618,8 @@ CAF_TEST_DISABLED(non_empty_server_handshake_udp) {
basp::version, this_node(), none, basp::version, this_node(), none,
self()->id(), invalid_actor_id, 0}; self()->id(), invalid_actor_id, 0};
to_buf(expected_buf, expected, nullptr, std::string{}, to_buf(expected_buf, expected, nullptr, std::string{},
self()->id(), set<string>{"caf::replies_to<@u16>::with<@u16>"}); self()->id(), set<string>{"caf::replies_to<@u16>::with<@u16>"},
basp::routing_table::address_map{});
CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf)); CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf));
} }
...@@ -636,10 +648,10 @@ CAF_TEST_DISABLED(client_handshake_and_dispatch_udp) { ...@@ -636,10 +648,10 @@ CAF_TEST_DISABLED(client_handshake_and_dispatch_udp) {
establish_communication(jupiter()); establish_communication(jupiter());
CAF_MESSAGE("send dispatch message"); CAF_MESSAGE("send dispatch message");
// send a message via `dispatch` from node 0 on endpoint 1 // send a message via `dispatch` from node 0 on endpoint 1
mock(endpoint_handle(), jupiter().endpoint, mock(jupiter().endpoint, default_sender(),
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(), jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(),
1}, // increment sequence number 2}, // increment sequence number
std::vector<actor_addr>{}, std::vector<actor_addr>{},
make_message(1, 2, 3)) make_message(1, 2, 3))
.receive(jupiter().endpoint, .receive(jupiter().endpoint,
...@@ -669,41 +681,19 @@ CAF_TEST_DISABLED(client_handshake_and_dispatch_udp) { ...@@ -669,41 +681,19 @@ CAF_TEST_DISABLED(client_handshake_and_dispatch_udp) {
); );
} }
CAF_TEST_DISABLED(message_forwarding_udp) {
// connect two remote nodes
CAF_MESSAGE("establish communication with Jupiter");
establish_communication(jupiter());
CAF_MESSAGE("establish communication with Mars");
establish_communication(mars());
CAF_MESSAGE("send dispatch message to ... ");
auto msg = make_message(1, 2, 3);
// send a message from node 0 to node 1, forwarded by this node
mock(endpoint_handle(), jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id(),
1}, // increment sequence number
msg)
.receive(mars().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id(),
msg);
}
CAF_TEST_DISABLED(publish_and_connect_udp) { CAF_TEST_DISABLED(publish_and_connect_udp) {
auto dx = datagram_handle::from_int(4242); auto dx = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx); mpx()->provide_datagram_servant(4242, dx);
auto res = sys.middleman().publish_udp(self(), 4242); auto res = sys.middleman().publish_udp(self(), 4242);
CAF_REQUIRE(res == 4242); CAF_REQUIRE(res == 4242);
mpx()->flush_runnables(); // process publish message in basp_broker mpx()->flush_runnables(); // process publish message in basp_broker
establish_communication(jupiter(), dx, self()->id()); establish_communication(jupiter(), dx, default_sender(), self()->id());
} }
CAF_TEST_DISABLED(remote_actor_and_send_udp) { CAF_TEST_DISABLED(remote_actor_and_send_udp) {
constexpr const char* lo = "localhost"; constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint); mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint, default_sender());
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242)); CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle(); auto mm1 = sys.middleman().actor_handle();
actor result; actor result;
...@@ -712,23 +702,30 @@ CAF_TEST_DISABLED(remote_actor_and_send_udp) { ...@@ -712,23 +702,30 @@ CAF_TEST_DISABLED(remote_actor_and_send_udp) {
// wait until BASP broker has received and processed the connect message // wait until BASP broker has received and processed the connect message
while (!aut()->valid(jupiter().endpoint)) while (!aut()->valid(jupiter().endpoint))
mpx()->exec_runnable(); mpx()->exec_runnable();
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242)); CAF_REQUIRE(!mpx()->has_pending_remote_endpoint(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor // build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement"); CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors(); auto na = registry()->named_actors();
mock() mock()
.receive(jupiter().endpoint, .receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u, basp::message_type::client_handshake, no_flags, 2u,
no_operation_data, this_node(), node_id(), no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{}); invalid_actor_id, invalid_actor_id, std::string{},
mock(jupiter().endpoint, basp::routing_table::address_map{});
mock(jupiter().endpoint, default_sender(),
{basp::message_type::server_handshake, 0, 0, basp::version, {basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none, jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id, jupiter().dummy_actor->id(), invalid_actor_id,
0}, // sequence number, first message 0}, // sequence number, first message
std::string{}, std::string{},
jupiter().dummy_actor->id(), jupiter().dummy_actor->id(),
uint32_t{0}) uint32_t{0},
basp::routing_table::address_map{})
.receive(jupiter().endpoint,
basp::message_type::acknowledge_handshake,
no_flags, no_payload, no_operation_data,
this_node(), jupiter().id,
invalid_actor_id, invalid_actor_id)
.receive(jupiter().endpoint, .receive(jupiter().endpoint,
basp::message_type::dispatch_message, basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, basp::header::named_receiver_flag, any_vals,
...@@ -811,11 +808,11 @@ CAF_TEST_DISABLED(actor_serialize_and_deserialize_udp) { ...@@ -811,11 +808,11 @@ CAF_TEST_DISABLED(actor_serialize_and_deserialize_udp) {
registry()->put(testee->id(), actor_cast<strong_actor_ptr>(testee)); registry()->put(testee->id(), actor_cast<strong_actor_ptr>(testee));
CAF_MESSAGE("send message via BASP (from proxy)"); CAF_MESSAGE("send message via BASP (from proxy)");
auto msg = make_message(actor_cast<actor_addr>(prx)); auto msg = make_message(actor_cast<actor_addr>(prx));
mock(endpoint_handle(), jupiter().endpoint, mock(jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
prx->node(), this_node(), prx->node(), this_node(),
prx->id(), testee->id(), prx->id(), testee->id(),
1}, // sequence number, first message after handshake 2}, // sequence number, previous messages: client and ack handshake
std::vector<actor_id>{}, std::vector<actor_id>{},
msg); msg);
// testee must've responded (process forwarded message in BASP broker) // testee must've responded (process forwarded message in BASP broker)
...@@ -830,60 +827,8 @@ CAF_TEST_DISABLED(actor_serialize_and_deserialize_udp) { ...@@ -830,60 +827,8 @@ CAF_TEST_DISABLED(actor_serialize_and_deserialize_udp) {
std::vector<actor_id>{}, msg); std::vector<actor_id>{}, msg);
} }
CAF_TEST_DISABLED(indirect_connections_udp) {
// this node receives a message from jupiter via mars and responds via mars
// and any ad-hoc automatic connection requests are ignored
CAF_MESSAGE("self: " << to_string(self()->address()));
auto dx = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx);
sys.middleman().publish_udp(self(), 4242);
mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to Mars");
establish_communication(mars(), dx, self()->id());
CAF_MESSAGE("actor from Jupiter sends a message to us via Mars");
auto mx = mock(dx, mars().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id(),
1}, // sequence number
std::vector<actor_id>{},
make_message("hello from jupiter!"));
CAF_MESSAGE("expect ('sys', 'get', \"info\") from Earth to Jupiter at Mars");
// this asks Jupiter if it has a 'SpawnServ'
mx.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"));
CAF_MESSAGE("expect announce_proxy message at Mars from Earth to Jupiter");
mx.receive(mars().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("receive message from jupiter");
self()->receive(
[](const std::string& str) -> std::string {
CAF_CHECK_EQUAL(str, "hello from jupiter!");
return "hello from earth!";
}
);
mpx()->exec_runnable(); // process forwarded message in basp_broker
mock()
.receive(mars().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
}
CAF_TEST_FIXTURE_SCOPE_END() CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(basp_udp_tests_with_manual_timer, manual_timer_fixture) CAF_TEST_FIXTURE_SCOPE(basp_udp_tests_with_manual_timer, manual_timer_fixture)
CAF_TEST_DISABLED(out_of_order_delivery_udp) { CAF_TEST_DISABLED(out_of_order_delivery_udp) {
...@@ -891,7 +836,7 @@ CAF_TEST_DISABLED(out_of_order_delivery_udp) { ...@@ -891,7 +836,7 @@ CAF_TEST_DISABLED(out_of_order_delivery_udp) {
// timeouts that deliver pending message. // timeouts that deliver pending message.
constexpr const char* lo = "localhost"; constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint); mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint, default_sender());
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242)); CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle(); auto mm1 = sys.middleman().actor_handle();
actor result; actor result;
...@@ -902,23 +847,28 @@ CAF_TEST_DISABLED(out_of_order_delivery_udp) { ...@@ -902,23 +847,28 @@ CAF_TEST_DISABLED(out_of_order_delivery_udp) {
sched.run(); sched.run();
mpx()->exec_runnable(); mpx()->exec_runnable();
} }
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242)); CAF_REQUIRE(!mpx()->has_pending_remote_endpoint(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor // build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement"); CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors(); auto na = registry()->named_actors();
mock() mock()
.receive(jupiter().endpoint, .receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u, basp::message_type::client_handshake, no_flags, 2u,
no_operation_data, this_node(), node_id(), no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{}); invalid_actor_id, invalid_actor_id, std::string{},
mock(jupiter().endpoint, jupiter().endpoint, basp::routing_table::address_map{});
mock(jupiter().endpoint, default_sender(),
{basp::message_type::server_handshake, 0, 0, basp::version, {basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none, jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id, jupiter().dummy_actor->id(), invalid_actor_id,
0}, // sequence number, first message 0}, // sequence number, first message
std::string{}, std::string{},
jupiter().dummy_actor->id(), jupiter().dummy_actor->id(),
uint32_t{0}) uint32_t{0},
basp::routing_table::address_map{})
.receive(jupiter().endpoint, basp::message_type::acknowledge_handshake,
no_flags, no_payload, no_operation_data,
this_node(), jupiter().id, invalid_actor_id, invalid_actor_id)
.receive(jupiter().endpoint, .receive(jupiter().endpoint,
basp::message_type::dispatch_message, basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, basp::header::named_receiver_flag, any_vals,
...@@ -1024,124 +974,54 @@ CAF_TEST_FIXTURE_SCOPE_END() ...@@ -1024,124 +974,54 @@ CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(basp_udp_tests_with_autoconn, autoconn_enabled_fixture) CAF_TEST_FIXTURE_SCOPE(basp_udp_tests_with_autoconn, autoconn_enabled_fixture)
CAF_TEST_DISABLED(automatic_connection_udp) { CAF_TEST_DISABLED(address_handshake) {
// this tells our BASP broker to enable the automatic connection feature // Test whether basp instance correctly sends a server handshake
//anon_send(aut(), ok_atom::value, // when there's no actor published and automatic connections are enabled.
// "middleman.enable-automatic-connections", make_message(true)); buffer buf;
//mpx()->exec_runnable(); // process publish message in basp_broker instance().write_server_handshake(mpx(), buf, none);
// jupiter [remote hdl 0] -> mars [remote hdl 1] -> earth [this_node] auto addrs = instance().tbl().local_addresses();
// (this node receives a message from jupiter via mars and responds via mars, CAF_CHECK(!addrs.empty());
// but then also establishes a connection to jupiter directly) CAF_CHECK(addrs.count(network::protocol::udp) > 0 &&
auto check_node_in_tbl = [&](node& n) { !addrs[network::protocol::udp].second.empty());
io::id_visitor id_vis; CAF_CHECK(addrs.count(network::protocol::tcp) == 0);
auto hdl = tbl().lookup_direct(n.id); buffer expected_buf;
CAF_REQUIRE(hdl); basp::header expected{basp::message_type::server_handshake, 0, 0,
CAF_CHECK_EQUAL(visit(id_vis, *hdl), n.endpoint.id()); basp::version, this_node(), none,
}; invalid_actor_id, invalid_actor_id};
to_buf(expected_buf, expected, nullptr, std::string{},
invalid_actor_id, set<string>{}, addrs);
CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf));
}
CAF_TEST_DISABLED(read_address_after_handshake) {
mpx()->provide_datagram_servant("jupiter", 8080, jupiter().endpoint); mpx()->provide_datagram_servant("jupiter", 8080, jupiter().endpoint);
CAF_CHECK(mpx()->has_pending_remote_endpoint("jupiter", 8080)); CAF_CHECK(mpx()->has_pending_remote_endpoint("jupiter", 8080));
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
auto dx = datagram_handle::from_int(4242); auto dh = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx); mpx()->provide_datagram_servant(4242, dh);
publish(self(), 4242, true); publish(self(), 4242, true);
mpx()->flush_runnables(); // process publish message in basp_broker mpx()->flush_runnables();
CAF_MESSAGE("connect to mars"); CAF_MESSAGE("contacting mars");
establish_communication(mars(), dx, self()->id()); auto& addrs = instance().tbl().local_addresses();
//CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id()); establish_communication(mars(), dh, default_sender(), self()->id(), std::set<string>{}, addrs);
check_node_in_tbl(mars()); CAF_MESSAGE("Look for mars address information in our config server");
CAF_MESSAGE("simulate that an actor from jupiter " auto config_server = sys.registry().get(peer_serv_atom);
"sends a message to us via mars"); self()->send(actor_cast<actor>(config_server), get_atom::value,
mock(dx, mars().endpoint, to_string(mars().id));
{basp::message_type::dispatch_message, 0, 0, 0, sched.run();
jupiter().id, this_node(), mpx()->flush_runnables(); // process get request and send answer
jupiter().dummy_actor->id(), self()->id(),
1}, // sequence number
std::vector<actor_id>{},
make_message("hello from jupiter!"))
.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, no_operation_data,
this_node(), jupiter().id, any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"))
// if we do not disable TCP, we would receive a second message here asking
// for the tcp related addresses
.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, // actor ID of an actor spawned by the BASP broker
invalid_actor_id,
config_serv_atom,
std::vector<actor_id>{},
make_message(get_atom::value, "basp.default-connectivity-udp"))
.receive(mars().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), mars().id);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
auto connection_helper_actor = sys.latest_actor_id();
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
// create a dummy config server and respond to the name lookup
CAF_MESSAGE("receive ConfigServ of jupiter");
network::address_listing res;
res[network::protocol::ipv4].emplace_back("jupiter");
mock(dx, mars().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
this_node(), this_node(),
invalid_actor_id, connection_helper_actor,
2}, // sequence number
std::vector<actor_id>{},
make_message("basp.default-connectivity-udp",
make_message(uint16_t{8080}, std::move(res))));
// our connection helper should now connect to jupiter and
// send the scribe handle over to the BASP broker
while (mpx()->has_pending_remote_endpoint("jupiter", 8080)) {
sched.run();
mpx()->flush_runnables();
}
CAF_REQUIRE(mpx()->output_queue(mars().endpoint).empty());
CAF_MESSAGE("Let's do the handshake.");
mock()
.receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u,
no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{});
CAF_MESSAGE("Received client handshake.");
// send handshake from jupiter
mock(jupiter().endpoint, jupiter().endpoint,
{basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id},
std::string{},
jupiter().dummy_actor->id(),
uint32_t{0});
mpx()->exec_runnable(); // Receive message from connection helper
mpx()->exec_runnable(); // Process BASP server handshake
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), none);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
check_node_in_tbl(jupiter());
check_node_in_tbl(mars());
CAF_MESSAGE("receive message from jupiter");
self()->receive( self()->receive(
[](const std::string& str) -> std::string { [&](const std::string& item, message& msg) {
CAF_CHECK_EQUAL(str, "hello from jupiter!"); // Check that we got an entry under the name of our peer.
return "hello from earth!"; CAF_REQUIRE_EQUAL(item, to_string(mars().id));
msg.apply(
[&](basp::routing_table::address_map& addrs) {
// The addresses of our dummy node, thus empty.
CAF_CHECK(addrs.empty());
}
);
} }
); );
mpx()->exec_runnable(); // process forwarded message in basp_broker
CAF_MESSAGE("response message must take direct route now");
mock()
.receive(jupiter().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
} }
CAF_TEST_FIXTURE_SCOPE_END() CAF_TEST_FIXTURE_SCOPE_END()
...@@ -39,6 +39,8 @@ class config : public actor_system_config { ...@@ -39,6 +39,8 @@ class config : public actor_system_config {
public: public:
config() { config() {
load<io::middleman>(); load<io::middleman>();
set("middleman.enable-tcp", true);
set("middleman.enable-udp", false);
add_message_type<std::vector<int>>("std::vector<int>"); add_message_type<std::vector<int>>("std::vector<int>");
if (auto err = parse(test::engine::argc(), test::engine::argv())) if (auto err = parse(test::engine::argc(), test::engine::argv()))
CAF_FAIL("failed to parse config: " << to_string(err)); CAF_FAIL("failed to parse config: " << to_string(err));
...@@ -65,7 +67,11 @@ struct fixture { ...@@ -65,7 +67,11 @@ struct fixture {
} }
}; };
behavior make_pong_behavior() { behavior make_pong_behavior(event_based_actor* self) {
self->set_exit_handler([=](exit_msg& m) {
CAF_MESSAGE("Pong received exit message.");
self->quit(m.reason);
});
return { return {
[](int val) -> int { [](int val) -> int {
++val; ++val;
......
...@@ -39,6 +39,7 @@ class config : public actor_system_config { ...@@ -39,6 +39,7 @@ class config : public actor_system_config {
public: public:
config() { config() {
load<io::middleman>(); load<io::middleman>();
set("middleman.enable-tcp", false);
set("middleman.enable-udp", true); set("middleman.enable-udp", true);
add_message_type<std::vector<int>>("std::vector<int>"); add_message_type<std::vector<int>>("std::vector<int>");
if (auto err = parse(test::engine::argc(), test::engine::argv())) if (auto err = parse(test::engine::argc(), test::engine::argv()))
...@@ -66,7 +67,11 @@ struct fixture { ...@@ -66,7 +67,11 @@ struct fixture {
} }
}; };
behavior make_pong_behavior() { behavior make_pong_behavior(event_based_actor* self) {
self->set_exit_handler([=](exit_msg& m) {
CAF_MESSAGE("Pong received exit message.");
self->quit(m.reason);
});
return { return {
[](int val) -> int { [](int val) -> int {
++val; ++val;
...@@ -204,8 +209,7 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) { ...@@ -204,8 +209,7 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) {
} }
}; };
}); });
auto port = server_sys.middleman().publish_udp(mirror, 0); auto port = unbox(server_sys.middleman().publish_udp(mirror, 0));
CAF_REQUIRE(port);
CAF_MESSAGE("server running on port " << port); CAF_MESSAGE("server running on port " << port);
auto client_fun = [](event_based_actor* self) -> behavior { auto client_fun = [](event_based_actor* self) -> behavior {
return { return {
...@@ -225,7 +229,8 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) { ...@@ -225,7 +229,8 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) {
actor_system client_sys{client_cfg}; actor_system client_sys{client_cfg};
auto client = client_sys.spawn(client_fun); auto client = client_sys.spawn(client_fun);
// Acquire remote actor from the server. // Acquire remote actor from the server.
auto client_srv = client_sys.middleman().remote_actor_udp("localhost", *port); // acquire remote actor from the server
auto client_srv = client_sys.middleman().remote_actor_udp("localhost", port);
CAF_REQUIRE(client_srv); CAF_REQUIRE(client_srv);
// Setup other clients. // Setup other clients.
for (int i = 0; i < 5; ++i) { for (int i = 0; i < 5; ++i) {
...@@ -234,7 +239,7 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) { ...@@ -234,7 +239,7 @@ CAF_TEST_DISABLED(multiple_endpoints_udp) {
CAF_MESSAGE("creating new client"); CAF_MESSAGE("creating new client");
auto other = other_sys.spawn(client_fun); auto other = other_sys.spawn(client_fun);
// Acquire remote actor from the new server. // Acquire remote actor from the new server.
auto other_srv = other_sys.middleman().remote_actor_udp("localhost", *port); auto other_srv = other_sys.middleman().remote_actor_udp("localhost", port);
CAF_REQUIRE(other_srv); CAF_REQUIRE(other_srv);
// Establish communication and exit. // Establish communication and exit.
CAF_MESSAGE("client contacts server and exits"); CAF_MESSAGE("client contacts server and exits");
......
...@@ -82,6 +82,17 @@ public: ...@@ -82,6 +82,17 @@ public:
return *res; return *res;
} }
/// Convenience function for calling `mm.publish` and requiring a valid
/// result.
template <class Handle>
uint16_t publish_udp(Handle whom, uint16_t port, const char* in = nullptr,
bool reuse = false) {
this->sched.inline_next_enqueue();
auto res = mm.publish_udp(whom, port, in, reuse);
CAF_REQUIRE(res);
return *res;
}
/// Convenience function for calling `mm.remote_actor` and requiring a valid /// Convenience function for calling `mm.remote_actor` and requiring a valid
/// result. /// result.
template <class Handle = caf::actor> template <class Handle = caf::actor>
...@@ -93,6 +104,17 @@ public: ...@@ -93,6 +104,17 @@ public:
return *res; return *res;
} }
/// Convenience function for calling `mm.remote_actor` and requiring a valid
/// result.
template <class Handle = caf::actor>
Handle remote_actor_udp(std::string host, uint16_t port) {
this->sched.inline_next_enqueue();
this->sched.after_next_enqueue(run_all_nodes);
auto res = mm.remote_actor_udp<Handle>(std::move(host), port);
CAF_REQUIRE(res);
return *res;
}
// -- member variables ------------------------------------------------------- // -- member variables -------------------------------------------------------
/// Reference to the node's middleman. /// Reference to the node's middleman.
...@@ -152,6 +174,8 @@ public: ...@@ -152,6 +174,8 @@ public:
using accept_handle = caf::io::accept_handle; using accept_handle = caf::io::accept_handle;
using datagram_handle = caf::io::datagram_handle;
test_network_fixture_base(planets_vector xs) : planets_(std::move(xs)) { test_network_fixture_base(planets_vector xs) : planets_(std::move(xs)) {
// nop // nop
} }
...@@ -166,6 +190,11 @@ public: ...@@ -166,6 +190,11 @@ public:
return connection_handle::from_int(++hdl_id_); return connection_handle::from_int(++hdl_id_);
} }
/// Returns a unique datagram handle.
datagram_handle next_datagram_handle() {
return datagram_handle::from_int(++hdl_id_);
}
/// Prepare a connection from `client` (calls `remote_actor`) to `server` /// Prepare a connection from `client` (calls `remote_actor`) to `server`
/// (calls `publish`). /// (calls `publish`).
/// @returns randomly picked connection handles for the server and the client. /// @returns randomly picked connection handles for the server and the client.
...@@ -190,6 +219,31 @@ public: ...@@ -190,6 +219,31 @@ public:
next_accept_handle()); next_accept_handle());
} }
/// Prepares comminication between `client` (calls `remote_actor_udp`) and
/// `server` (calls `publish_udp`).
/// @returns randomly picked datagram handles for the server and the client.
std::pair<datagram_handle, datagram_handle>
prepare_endpoints(PlanetType& server, PlanetType& client,
std::string host, uint16_t port,
datagram_handle server_endpoint) {
auto server_hdl = next_datagram_handle();
auto client_hdl = next_datagram_handle();
server.mpx.prepare_endpoints(server_endpoint, server_hdl, client.mpx,
std::move(host), port, client_hdl);
return std::make_pair(server_hdl, client_hdl);
}
/// Prepares comminication between `client` (calls `remote_actor_udp`) and
/// `server` (calls `publish_udp`).
/// @returns randomly picked datagram handles for the server and the client.
std::pair<datagram_handle, datagram_handle>
prepare_endpoints(PlanetType& server, PlanetType& client,
std::string host, uint16_t port) {
return prepare_endpoints(server, client, std::move(host), port,
next_datagram_handle());
}
// Convenience function for transmitting all "network" traffic (no new // Convenience function for transmitting all "network" traffic (no new
// connections are accepted). // connections are accepted).
void network_traffic() { void network_traffic() {
...@@ -245,12 +299,17 @@ public: ...@@ -245,12 +299,17 @@ public:
} }
}; };
struct mm_enabled_config : caf::actor_system_config {
mm_enabled_config() {
load<caf::io::middleman>();
}
};
/// A simple fixture that includes three nodes (`earth`, `mars`, and `jupiter`) /// A simple fixture that includes three nodes (`earth`, `mars`, and `jupiter`)
/// that can connect to each other. /// that can connect to each other.
template <class BaseFixture = template <class BaseFixture = test_coordinator_fixture<mm_enabled_config>>
test_coordinator_fixture<caf::actor_system_config>>
class belt_fixture class belt_fixture
: public test_network_fixture_base<test_node_fixture<BaseFixture>> { : public test_network_fixture_base<test_node_fixture<BaseFixture>> {
public: public:
using planet_type = test_node_fixture<BaseFixture>; using planet_type = test_node_fixture<BaseFixture>;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment