Commit 2eed3d57 authored by Joseph Noir's avatar Joseph Noir

Rework UDP endpoint handling in test multiplexer

parent 63cb12d6
...@@ -32,6 +32,7 @@ namespace network { ...@@ -32,6 +32,7 @@ namespace network {
class test_multiplexer : public multiplexer { class test_multiplexer : public multiplexer {
private: private:
struct datagram_endpoint;
struct datagram_data; struct datagram_data;
public: public:
...@@ -96,10 +97,13 @@ public: ...@@ -96,10 +97,13 @@ public:
void provide_acceptor(uint16_t desired_port, accept_handle hdl); void provide_acceptor(uint16_t desired_port, accept_handle hdl);
// Provide a local datagram servant on port with `hdl`.
void provide_datagram_servant(uint16_t desired_port, datagram_handle hdl); void provide_datagram_servant(uint16_t desired_port, datagram_handle hdl);
// Provide a datagram servant for communication a remote endpoint
// with `endpoint_id`.
void provide_datagram_servant(std::string host, uint16_t desired_port, void provide_datagram_servant(std::string host, uint16_t desired_port,
datagram_handle hdl); datagram_handle hdl, intptr_t endpoint_id = 0);
/// Generate an id for a new servant. /// Generate an id for a new servant.
int64_t next_endpoint_id(); int64_t next_endpoint_id();
...@@ -108,10 +112,11 @@ public: ...@@ -108,10 +112,11 @@ public:
using buffer_type = std::vector<char>; using buffer_type = std::vector<char>;
/// Buffers storing bytes for UDP related components. /// Buffers storing bytes for UDP related components.
using endpoint_id_type = std::intptr_t;
using read_buffer_type = network::receive_buffer; using read_buffer_type = network::receive_buffer;
using write_buffer_type = buffer_type; using write_buffer_type = buffer_type;
using read_job_type = std::pair<datagram_handle, read_buffer_type>; using read_job_type = std::pair<endpoint_id_type, read_buffer_type>;
using write_job_type = std::pair<datagram_handle, write_buffer_type>; using write_job_type = std::pair<endpoint_id_type, write_buffer_type>;
using write_job_queue_type = std::deque<write_job_type>; using write_job_queue_type = std::deque<write_job_type>;
using shared_buffer_type = std::shared_ptr<buffer_type>; using shared_buffer_type = std::shared_ptr<buffer_type>;
...@@ -176,8 +181,10 @@ public: ...@@ -176,8 +181,10 @@ public:
datagram_servant_ptr& impl_ptr(datagram_handle hdl); datagram_servant_ptr& impl_ptr(datagram_handle hdl);
/// Returns a map with all servants related to the servant `hdl`. using write_handle_map = std::map<endpoint_id_type, datagram_endpoint>;
std::set<datagram_handle>& servants(datagram_handle hdl);
/// Returns the endpoint for a specific `hdl`.
endpoint_id_type endpoint_id(datagram_handle hdl);
/// Returns `true` if this handle has been closed /// Returns `true` if this handle has been closed
/// for reading, `false` otherwise. /// for reading, `false` otherwise.
...@@ -198,15 +205,27 @@ public: ...@@ -198,15 +205,27 @@ public:
test_multiplexer& peer, std::string host, test_multiplexer& peer, std::string host,
uint16_t port, connection_handle peer_hdl); uint16_t port, connection_handle peer_hdl);
/// Stores `hdl` as a pending endpoint for `src`. void add_pending_endpoint(intptr_t endpoint_id, datagram_handle hdl,
void add_pending_endpoint(datagram_handle src, datagram_handle hdl); shared_job_queue_type write_buffer
= std::make_shared<write_job_queue_type>());
/// Add `hdl` as a pending endpoint to `src` and provide a datagram servant
/// on `peer` that connects the buffers of `hdl` and `peer_hdl`. Calls
/// `add_pending_endpoint(...)` and `peer.provide_endpoint(...)`.
void prepare_endpoints(datagram_handle src, datagram_handle hdl,
test_multiplexer& peer, std::string host,
uint16_t port, datagram_handle peer_hdl);
using pending_connects_map = std::unordered_multimap<accept_handle, using pending_connects_map = std::unordered_multimap<accept_handle,
connection_handle>; connection_handle>;
pending_connects_map& pending_connects(); pending_connects_map& pending_connects();
using pending_endpoints_map = std::unordered_map<int64_t, datagram_handle>; // using pending_endpoints_map = std::unordered_map<int64_t, datagram_handle>;
using pending_endpoints_map = std::map<intptr_t,
std::pair<datagram_handle,
shared_job_queue_type>>;
pending_endpoints_map& pending_endpoints(); pending_endpoints_map& pending_endpoints();
...@@ -219,7 +238,7 @@ public: ...@@ -219,7 +238,7 @@ public:
datagram_handle>; datagram_handle>;
using pending_remote_datagram_endpoints_map using pending_remote_datagram_endpoints_map
= std::map<std::pair<std::string, uint16_t>, datagram_handle>; = std::map<std::pair<std::string, uint16_t>, std::pair<datagram_handle, intptr_t>>;
bool has_pending_scribe(std::string x, uint16_t y); bool has_pending_scribe(std::string x, uint16_t y);
...@@ -237,6 +256,9 @@ public: ...@@ -237,6 +256,9 @@ public:
/// Tries to read data from the external input buffer of `hdl`. /// Tries to read data from the external input buffer of `hdl`.
bool try_read_data(connection_handle hdl); bool try_read_data(connection_handle hdl);
/// Tries to read data from the external input queue of `hdl`.
bool try_read_data(datagram_handle hdl);
/// Poll data on all scribes. /// Poll data on all scribes.
bool read_data(); bool read_data();
...@@ -253,13 +275,9 @@ public: ...@@ -253,13 +275,9 @@ public:
/// Appends `buf` to the virtual network buffer of `hdl` /// Appends `buf` to the virtual network buffer of `hdl`
/// and calls `read_data(hdl)` afterwards. /// and calls `read_data(hdl)` afterwards.
void virtual_send(datagram_handle src, datagram_handle ep, void virtual_send(datagram_handle hdl, endpoint_id_type endpoint,
const buffer_type&); const buffer_type&);
/// Detaches a scribe identified by `hdl` and optianlly makes it send a
/// disconnect message to its broker.
void detach(connection_handle hdl, bool invoke_detach_message = false);
/// Waits until a `runnable` is available and executes it. /// Waits until a `runnable` is available and executes it.
void exec_runnable(); void exec_runnable();
...@@ -327,24 +345,36 @@ private: ...@@ -327,24 +345,36 @@ private:
doorman_data(); doorman_data();
}; };
struct datagram_data { struct datagram_endpoint {
datagram_handle hdl;
shared_job_queue_type vn_buf_ptr; shared_job_queue_type vn_buf_ptr;
shared_job_queue_type wr_buf_ptr; shared_job_queue_type wr_buf_ptr;
write_job_queue_type& vn_buf; write_job_queue_type& vn_buf;
write_job_queue_type& wr_buf; write_job_queue_type& wr_buf;
read_job_type rd_buf;
datagram_endpoint(
datagram_handle hdl = datagram_handle(),
shared_job_queue_type input = std::make_shared<write_job_queue_type>(),
shared_job_queue_type output = std::make_shared<write_job_queue_type>()
);
};
struct datagram_data {
datagram_servant_ptr ptr; datagram_servant_ptr ptr;
datagram_endpoint read_handle;
read_job_type rd_buf;
bool stopped_reading; bool stopped_reading;
bool passive_mode; bool passive_mode;
bool ack_writes; bool ack_writes;
uint16_t port; uint16_t remote_port;
uint16_t local_port; uint16_t local_port;
std::set<datagram_handle> servants; write_handle_map write_handles;
size_t datagram_size; size_t datagram_size;
// Allows creating an entangled scribes where the input of this scribe is // Allows creating an entangled scribes where the input of this scribe is
// the output of another scribe and vice versa. // the output of another scribe and vice versa.
datagram_data( datagram_data(
datagram_handle hdl = datagram_handle{},
shared_job_queue_type input = std::make_shared<write_job_queue_type>(), shared_job_queue_type input = std::make_shared<write_job_queue_type>(),
shared_job_queue_type output = std::make_shared<write_job_queue_type>() shared_job_queue_type output = std::make_shared<write_job_queue_type>()
); );
......
This diff is collapsed.
...@@ -209,6 +209,10 @@ public: ...@@ -209,6 +209,10 @@ public:
return dhdl_; return dhdl_;
} }
intptr_t default_sender() {
return default_sender_;
}
// implementation of the Binary Actor System Protocol // implementation of the Binary Actor System Protocol
basp::instance& instance() { basp::instance& instance() {
return aut()->state.instance; return aut()->state.instance;
...@@ -273,9 +277,10 @@ public: ...@@ -273,9 +277,10 @@ public:
void establish_communication(node& n, void establish_communication(node& n,
optional<datagram_handle> dx = none, optional<datagram_handle> dx = none,
optional<intptr_t> endpoint_id = none,
actor_id published_actor_id = invalid_actor_id, actor_id published_actor_id = invalid_actor_id,
const set<string>& published_actor_ifs const set<string>& published_actor_ifs
= std::set<std::string>{}, = set<std::string>{},
const basp::routing_table::address_map& am = {}) { const basp::routing_table::address_map& am = {}) {
auto src = dx ? *dx : dhdl_; auto src = dx ? *dx : dhdl_;
CAF_MESSAGE("establish communication on node " << n.name CAF_MESSAGE("establish communication on node " << n.name
...@@ -284,9 +289,10 @@ public: ...@@ -284,9 +289,10 @@ public:
// send the client handshake and receive the server handshake // send the client handshake and receive the server handshake
// and a dispatch_message as answers // and a dispatch_message as answers
auto hdl = n.endpoint; auto hdl = n.endpoint;
mpx_->add_pending_endpoint(src, hdl); auto ep = endpoint_id ? *endpoint_id : default_sender_;
CAF_MESSAGE("Send client handshake"); mpx_->add_pending_endpoint(ep, hdl);
mock(src, hdl, CAF_MESSAGE("send client handshake");
mock(src, ep,
{basp::message_type::client_handshake, 0, 0, 0, {basp::message_type::client_handshake, 0, 0, 0,
n.id, this_node(), n.id, this_node(),
invalid_actor_id, invalid_actor_id}, std::string{}, invalid_actor_id, invalid_actor_id}, std::string{},
...@@ -377,11 +383,8 @@ public: ...@@ -377,11 +383,8 @@ public:
while (oq.empty()) while (oq.empty())
this_->mpx()->exec_runnable(); this_->mpx()->exec_runnable();
CAF_MESSAGE("output queue has " << oq.size() << " messages"); CAF_MESSAGE("output queue has " << oq.size() << " messages");
auto dh = oq.front().first;
buffer& ob = oq.front().second; buffer& ob = oq.front().second;
CAF_MESSAGE("next datagram has " << ob.size() CAF_REQUIRE_EQUAL(this_->mpx()->endpoint_id(hdl), oq.front().first);
<< " bytes, servant ID = " << dh.id());
CAF_CHECK_EQUAL(dh.id(), hdl.id());
basp::header hdr; basp::header hdr;
{ // lifetime scope of source { // lifetime scope of source
binary_deserializer source{this_->mpx(), ob}; binary_deserializer source{this_->mpx(), ob};
...@@ -412,39 +415,41 @@ public: ...@@ -412,39 +415,41 @@ public:
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, datagram_handle ep, mock_t& enqueue_back(datagram_handle hdl, intptr_t sender_id,
basp::header hdr, const Ts&... xs) { basp::header hdr, const Ts&... xs) {
buffer buf; buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...); this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation) CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload to back of queue"); << " bytes payload to back of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_back(ep, buf); this_->mpx()->virtual_network_buffer(hdl).emplace_back(sender_id, buf);
return *this; return *this;
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, basp::header hdr, mock_t& enqueue_back(datagram_handle hdl, basp::header hdr,
Ts&&... xs) { Ts&&... xs) {
return enqueue_back(hdl, hdl, hdr, std::forward<Ts>(xs)...); return enqueue_back(hdl, this_->default_sender(), hdr,
std::forward<Ts>(xs)...);
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, datagram_handle ep, mock_t& enqueue_front(datagram_handle hdl, intptr_t sender_id,
basp::header hdr, const Ts&... xs) { basp::header hdr, const Ts&... xs) {
buffer buf; buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...); this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation) CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload to front of queue"); << " bytes payload to front of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_front(ep, buf); this_->mpx()->virtual_network_buffer(hdl).emplace_front(sender_id, buf);
return *this; return *this;
} }
template <class... Ts> template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, basp::header hdr, mock_t& enqueue_front(datagram_handle hdl, basp::header hdr,
Ts&&... xs) { Ts&&... xs) {
return enqueue_front(hdl, hdl, hdr, std::forward<Ts>(xs)...); return enqueue_front(hdl, this_->default_sender(), hdr,
std::forward<Ts>(xs)...);
} }
mock_t& deliver(datagram_handle hdl, size_t num_messages = 1) { mock_t& deliver(datagram_handle hdl, size_t num_messages = 1) {
...@@ -466,19 +471,19 @@ public: ...@@ -466,19 +471,19 @@ public:
CAF_MESSAGE("virtually send " << to_string(hdr.operation) CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload"); << " bytes payload");
mpx()->virtual_send(hdl, hdl, buf); mpx()->virtual_send(hdl, default_sender_, buf);
return {this}; return {this};
} }
template <class... Ts> template <class... Ts>
mock_t mock(datagram_handle hdl, datagram_handle ep, basp::header hdr, mock_t mock(datagram_handle hdl, intptr_t sender_id, basp::header hdr,
const Ts&... xs) { const Ts&... xs) {
buffer buf; buffer buf;
to_buf(buf, hdr, nullptr, xs...); to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("virtually send " << to_string(hdr.operation) CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size) << " with " << (buf.size() - basp::header_size)
<< " bytes payload"); << " bytes payload");
mpx()->virtual_send(hdl, ep, buf); mpx()->virtual_send(hdl, sender_id, buf);
return {this}; return {this};
} }
...@@ -492,6 +497,7 @@ public: ...@@ -492,6 +497,7 @@ public:
private: private:
basp_broker* aut_; basp_broker* aut_;
datagram_handle dhdl_; datagram_handle dhdl_;
intptr_t default_sender_ = static_cast<intptr_t>(0xdeadbeef);
network::test_multiplexer* mpx_; network::test_multiplexer* mpx_;
node_id this_node_; node_id this_node_;
unique_ptr<scoped_actor> self_; unique_ptr<scoped_actor> self_;
...@@ -637,7 +643,7 @@ CAF_TEST(client_handshake_and_dispatch_udp) { ...@@ -637,7 +643,7 @@ CAF_TEST(client_handshake_and_dispatch_udp) {
establish_communication(jupiter()); establish_communication(jupiter());
CAF_MESSAGE("send dispatch message"); CAF_MESSAGE("send dispatch message");
// send a message via `dispatch` from node 0 on endpoint 1 // send a message via `dispatch` from node 0 on endpoint 1
mock(endpoint_handle(), jupiter().endpoint, mock(jupiter().endpoint, default_sender(),
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(), jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(),
1}, // increment sequence number 1}, // increment sequence number
...@@ -676,13 +682,13 @@ CAF_TEST(publish_and_connect_udp) { ...@@ -676,13 +682,13 @@ CAF_TEST(publish_and_connect_udp) {
auto res = sys.middleman().publish_udp(self(), 4242); auto res = sys.middleman().publish_udp(self(), 4242);
CAF_REQUIRE(res == 4242); CAF_REQUIRE(res == 4242);
mpx()->flush_runnables(); // process publish message in basp_broker mpx()->flush_runnables(); // process publish message in basp_broker
establish_communication(jupiter(), dx, self()->id()); establish_communication(jupiter(), dx, default_sender(), self()->id());
} }
CAF_TEST(remote_actor_and_send_udp) { CAF_TEST(remote_actor_and_send_udp) {
constexpr const char* lo = "localhost"; constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint); mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint, default_sender());
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242)); CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle(); auto mm1 = sys.middleman().actor_handle();
actor result; actor result;
...@@ -691,7 +697,7 @@ CAF_TEST(remote_actor_and_send_udp) { ...@@ -691,7 +697,7 @@ CAF_TEST(remote_actor_and_send_udp) {
// wait until BASP broker has received and processed the connect message // wait until BASP broker has received and processed the connect message
while (!aut()->valid(jupiter().endpoint)) while (!aut()->valid(jupiter().endpoint))
mpx()->exec_runnable(); mpx()->exec_runnable();
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242)); CAF_REQUIRE(!mpx()->has_pending_remote_endpoint(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor // build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement"); CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors(); auto na = registry()->named_actors();
...@@ -701,7 +707,7 @@ CAF_TEST(remote_actor_and_send_udp) { ...@@ -701,7 +707,7 @@ CAF_TEST(remote_actor_and_send_udp) {
no_operation_data, this_node(), node_id(), no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{}, invalid_actor_id, invalid_actor_id, std::string{},
basp::routing_table::address_map{}); basp::routing_table::address_map{});
mock(jupiter().endpoint, mock(jupiter().endpoint, default_sender(),
{basp::message_type::server_handshake, 0, 0, basp::version, {basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none, jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id, jupiter().dummy_actor->id(), invalid_actor_id,
...@@ -792,7 +798,7 @@ CAF_TEST(actor_serialize_and_deserialize_udp) { ...@@ -792,7 +798,7 @@ CAF_TEST(actor_serialize_and_deserialize_udp) {
registry()->put(testee->id(), actor_cast<strong_actor_ptr>(testee)); registry()->put(testee->id(), actor_cast<strong_actor_ptr>(testee));
CAF_MESSAGE("send message via BASP (from proxy)"); CAF_MESSAGE("send message via BASP (from proxy)");
auto msg = make_message(actor_cast<actor_addr>(prx)); auto msg = make_message(actor_cast<actor_addr>(prx));
mock(endpoint_handle(), jupiter().endpoint, mock(jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
prx->node(), this_node(), prx->node(), this_node(),
prx->id(), testee->id(), prx->id(), testee->id(),
...@@ -820,7 +826,7 @@ CAF_TEST(out_of_order_delivery_udp) { ...@@ -820,7 +826,7 @@ CAF_TEST(out_of_order_delivery_udp) {
// timeouts that deliver pending message. // timeouts that deliver pending message.
constexpr const char* lo = "localhost"; constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint); mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint, default_sender());
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242)); CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle(); auto mm1 = sys.middleman().actor_handle();
actor result; actor result;
...@@ -831,7 +837,7 @@ CAF_TEST(out_of_order_delivery_udp) { ...@@ -831,7 +837,7 @@ CAF_TEST(out_of_order_delivery_udp) {
sched.run(); sched.run();
mpx()->exec_runnable(); mpx()->exec_runnable();
} }
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242)); CAF_REQUIRE(!mpx()->has_pending_remote_endpoint(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor // build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement"); CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors(); auto na = registry()->named_actors();
...@@ -841,7 +847,7 @@ CAF_TEST(out_of_order_delivery_udp) { ...@@ -841,7 +847,7 @@ CAF_TEST(out_of_order_delivery_udp) {
no_operation_data, this_node(), node_id(), no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{}, invalid_actor_id, invalid_actor_id, std::string{},
basp::routing_table::address_map{}); basp::routing_table::address_map{});
mock(jupiter().endpoint, jupiter().endpoint, mock(jupiter().endpoint, default_sender(),
{basp::message_type::server_handshake, 0, 0, basp::version, {basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none, jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id, jupiter().dummy_actor->id(), invalid_actor_id,
...@@ -984,7 +990,7 @@ CAF_TEST(read_address_after_handshake) { ...@@ -984,7 +990,7 @@ CAF_TEST(read_address_after_handshake) {
mpx()->flush_runnables(); mpx()->flush_runnables();
CAF_MESSAGE("contacting mars"); CAF_MESSAGE("contacting mars");
auto& addrs = instance().tbl().local_addresses(); auto& addrs = instance().tbl().local_addresses();
establish_communication(mars(), dh, self()->id(), std::set<string>{}, addrs); establish_communication(mars(), dh, default_sender(), self()->id(), std::set<string>{}, addrs);
CAF_MESSAGE("Look for mars address information in our config server"); CAF_MESSAGE("Look for mars address information in our config server");
auto config_server = sys.registry().get(peer_serv_atom); auto config_server = sys.registry().get(peer_serv_atom);
self()->send(actor_cast<actor>(config_server), get_atom::value, self()->send(actor_cast<actor>(config_server), get_atom::value,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment