Commit b5c049b6 authored by Joseph Noir's avatar Joseph Noir

Add autoconn test for UDP and fix related bugs

parent 2eed3d57
......@@ -46,8 +46,7 @@ struct connection_helper_state {
static const char* name;
};
behavior datagram_connection_broker(broker* self,
uint16_t port,
behavior datagram_connection_broker(broker* self, uint16_t port,
network::address_listing addresses,
actor system_broker,
basp::instance* instance);
......
......@@ -632,19 +632,19 @@ void basp_broker_state::send_buffered_messages(execution_unit* ctx,
datagram_handle hdl) {
if (pending_connectivity.count(nid) > 0) {
for (auto& msg : pending_connectivity[nid]) {
// TODO: Validate that this actually works.
auto seq_num = next_sequence_number(hdl);
auto seq_size = sizeof(basp::sequence_type);
auto offset = basp::header_size - seq_size;
auto& buf = get_buffer(hdl);
auto& buf = msg;
stream_serializer<charbuf> out{ctx, buf.data() + offset, seq_size};
auto err = out(seq_num);
if (err)
CAF_LOG_ERROR(CAF_ARG(err));
buf.insert(buf.end(), msg.begin(), msg.end());
self->enqueue_datagram(hdl, std::move(buf));
self->flush(hdl);
}
pending_connectivity[nid].clear();
}
flush(hdl);
}
basp_broker_state::buffer_type&
......@@ -720,13 +720,22 @@ behavior basp_broker::make_behavior() {
state.enable_udp = system().config().middleman_enable_udp;
if (system().config().middleman_enable_automatic_connections) {
CAF_LOG_INFO("enable automatic connections");
auto addrs = network::interfaces::list_addresses(false);
for (auto& p : addrs) {
auto& vec = p.second;
// Remove link local addresses.
vec.erase(std::remove_if(std::begin(vec), std::end(vec),
[](const std::string& str) {
return str.find("fe80") == 0;
}),
vec.end());
}
// Open a random port and store a record for our peers how to
// connect to this broker directly in the configuration server.
if (state.enable_tcp) {
auto res = add_tcp_doorman(uint16_t{0});
if (res) {
auto port = res->second;
auto addrs = network::interfaces::list_addresses(false);
state.instance.tbl().local_addresses(network::protocol::tcp,
{port, addrs});
auto config_server = system().registry().get(atom("ConfigServ"));
......@@ -739,7 +748,6 @@ behavior basp_broker::make_behavior() {
auto res = add_udp_datagram_servant(uint16_t{0});
if (res) {
auto port = res->second;
auto addrs = network::interfaces::list_addresses(false);
state.instance.tbl().local_addresses(network::protocol::udp,
{port, addrs});
auto config_server = system().registry().get(atom("ConfigServ"));
......
......@@ -45,13 +45,17 @@ behavior datagram_connection_broker(broker* self, uint16_t port,
if (eptr) {
auto hdl = (*eptr)->hdl();
self->add_datagram_servant(std::move(*eptr));
instance->write_client_handshake(self->context(),
self->wr_buf(hdl),
std::vector<char> buf;
instance->write_client_handshake(self->context(), buf,
none, this_node,
app_id);
self->enqueue_datagram(hdl, std::move(buf));
self->flush(hdl);
}
}
}
// We are not interested in attempts that do not work.
self->set_default_handler(drop);
return {
[=](new_datagram_msg& msg) {
auto hdl = msg.handle;
......@@ -61,7 +65,7 @@ behavior datagram_connection_broker(broker* self, uint16_t port,
after(autoconnect_timeout) >> [=]() {
CAF_LOG_TRACE(CAF_ARG(""));
// Nothing heard in about 10 minutes... just call it a day, then.
CAF_LOG_INFO("aborted direct connection attempt after 10min");
CAF_LOG_INFO("aborted direct connection attempt after 10 min");
self->quit(exit_reason::user_shutdown);
}
};
......@@ -76,8 +80,7 @@ bool establish_stream_connection(stateful_actor<connection_helper_state>* self,
for (auto& addr : kvp.second) {
auto hdl = mx.new_tcp_scribe(addr, port);
if (hdl) {
// Gotcha! Send scribe to our BASP broker
// to initiate handshake etc.
// Gotcha! Send scribe to our BASP broker to initiate handshake etc.
CAF_LOG_INFO("connected directly:" << CAF_ARG(addr));
self->send(b, connect_atom::value, *hdl, port);
return true;
......
......@@ -81,6 +81,7 @@ constexpr uint16_t port_earth = 12340;
constexpr uint16_t port_mars = 12341;
constexpr uint16_t port_jupiter = 12342;
// Used for the tests with the test backend.
class config : public actor_system_config {
public:
config(bool use_tcp = true) {
......@@ -93,6 +94,14 @@ public:
}
};
class config_udp : public config {
public:
config_udp() : config(false) {
// nop
}
};
// Used for the tests with the default multiplexer backend.
class simple_config : public actor_system_config {
public:
simple_config(bool use_tcp = true) {
......@@ -250,7 +259,80 @@ CAF_TEST(break_triangle_simple_tcp) {
CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(autoconn_belt_tcp_test, belt_fixture_t<config>)
CAF_TEST_FIXTURE_SCOPE(autoconn_udp_simple_test, fixture_udp)
CAF_TEST(build_triangle_simple_udp) {
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.spawn(test_actor, "Earth", true);
auto earth_port = earth.middleman().publish_udp(on_earth, 0);
CAF_REQUIRE(earth_port);
CAF_MESSAGE("Earth reachable via " << *earth_port);
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.middleman().remote_actor_udp("localhost", *earth_port);
CAF_REQUIRE(from_earth);
auto on_mars = mars.spawn(test_actor, "Mars", true);
anon_send(on_mars, set_atom::value, *from_earth);
auto mars_port = mars.middleman().publish_udp(on_mars, 0);
CAF_REQUIRE(mars_port);
CAF_MESSAGE("Mars reachable via " << *mars_port);
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.middleman().remote_actor_udp("localhost", *mars_port);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, *from_mars);
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
jupiter.await_all_actors_done();
mars.await_all_actors_done();
earth.await_all_actors_done();
}
CAF_TEST(break_triangle_simple_udp) {
actor on_earth;
actor on_jupiter;
{
// Use UDP instead of TCP.
simple_config conf(false);
actor_system mars(conf);
// Earth.
CAF_MESSAGE("setting up Earth");
on_earth = earth.spawn(test_actor, "Earth", false);
auto earth_port = earth.middleman().publish_udp(on_earth, 0);
CAF_REQUIRE(earth_port);
CAF_MESSAGE("Earth reachable via " << *earth_port);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.middleman().remote_actor_udp("localhost", *earth_port);
if (!from_earth) {
CAF_MESSAGE("Failed to contact earth: " << mars.render(from_earth.error()));
}
CAF_REQUIRE(from_earth);
auto on_mars = mars.spawn(test_actor, "Mars", false);
anon_send(on_mars, set_atom::value, *from_earth);
auto mars_port = mars.middleman().publish_udp(on_mars, 0);
CAF_REQUIRE(mars_port);
CAF_MESSAGE("Mars reachable via " << *mars_port);
// Jupiter.
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.middleman().remote_actor_udp("localhost", *mars_port);
CAF_REQUIRE(from_mars);
on_jupiter = jupiter.spawn(test_actor, "Jupiter", false);
anon_send(on_jupiter, set_atom::value, *from_mars);
// Trigger the connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
mars.await_all_actors_done();
// Leaving the scope will shutdown Mars.
}
// Let the remaining nodes communicate.
anon_send(on_earth, msg_atom::value);
jupiter.await_all_actors_done();
earth.await_all_actors_done();
}
CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(autoconn_tcp_test, belt_fixture_t<config>)
CAF_TEST(build_triangle_tcp) {
CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
......@@ -309,14 +391,13 @@ CAF_TEST(break_triangle_tcp) {
CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
connection_handle em, me, mj, jm;
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", false);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
std::tie(em, me) = prepare_connection(earth, mars, "earth", port_earth);
prepare_connection(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish dummy on earth");
earth.publish(on_earth, port_earth);
// Mars.
......@@ -328,7 +409,7 @@ CAF_TEST(break_triangle_tcp) {
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare connection");
std::tie(mj, jm) = prepare_connection(mars, jupiter, "mars", port_mars);
prepare_connection(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish dummy on mars");
mars.publish(on_mars, port_mars);
// Jupiter.
......@@ -357,13 +438,131 @@ CAF_TEST(break_triangle_tcp) {
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
// Shutdown the connections to the intermediate node.
earth.mpx.detach(em, true);
mars.mpx.detach(me, true);
mars.mpx.detach(mj, true);
jupiter.mpx.detach(jm, true);
// Shutdown the basp broker of the intermediate node.
anon_send_exit(mars.basp, exit_reason::kill);
exec_all();
// Let the remaining nodes communicate.
anon_send(on_earth, msg_atom::value);
exec_all();
}
CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(autoconn_udp_test, belt_fixture_t<config_udp>)
CAF_TEST(build_triangle_udp) {
CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", true);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
prepare_endpoints(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish_udp dummy on earth");
earth.publish_udp(on_earth, port_earth);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.remote_actor_udp("earth", port_earth);
CAF_REQUIRE(from_earth);
auto on_mars = mars.sys.spawn(test_actor, "Mars", true);
anon_send(on_mars, set_atom::value, from_earth);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
datagram_handle mj, jm;
prepare_endpoints(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish_udp dummy on mars");
mars.publish_udp(on_mars, port_mars);
// Jupiter
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.remote_actor_udp("mars", port_mars);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", true);
anon_send(on_jupiter, set_atom::value, from_mars);
exec_all();
// This handle will be created by the test multiplexer for the automatically
// opened socket when automatic connections are enabled.
auto hdl_jup = datagram_handle::from_int(std::numeric_limits<int64_t>::max());
// Prepare automatic connection between Jupiter and Earth.
datagram_handle je, ej;
prepare_endpoints(jupiter, earth, "jupiter", port_jupiter, hdl_jup);
// Add the address information for this test to the config server on Mars.
auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
network::address_listing interfaces{
{network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::udp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
}
CAF_TEST(break_triangle_udp) {
CAF_MESSAGE("Earth : " << to_string(earth.sys.node()));
CAF_MESSAGE("Mars : " << to_string(mars.sys.node()));
CAF_MESSAGE("Jupiter: " << to_string(jupiter.sys.node()));
// Earth.
CAF_MESSAGE("setting up Earth");
auto on_earth = earth.sys.spawn(test_actor, "Earth", false);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
prepare_endpoints(earth, mars, "earth", port_earth);
CAF_MESSAGE("publish_udp dummy on earth");
earth.publish_udp(on_earth, port_earth);
// Mars.
CAF_MESSAGE("setting up Mars");
auto from_earth = mars.remote_actor_udp("earth", port_earth);
CAF_REQUIRE(from_earth);
auto on_mars = mars.sys.spawn(test_actor, "Mars", false);
anon_send(on_mars, set_atom::value, from_earth);
CAF_MESSAGE("run initialization code");
exec_all();
CAF_MESSAGE("prepare endpoints");
prepare_endpoints(mars, jupiter, "mars", port_mars);
CAF_MESSAGE("publish_udp dummy on mars");
mars.publish_udp(on_mars, port_mars);
// Jupiter.
CAF_MESSAGE("setting up Jupiter");
auto from_mars = jupiter.remote_actor_udp("mars", port_mars);
CAF_REQUIRE(from_mars);
auto on_jupiter = jupiter.sys.spawn(test_actor, "Jupiter", false);
anon_send(on_jupiter, set_atom::value, from_mars);
exec_all();
// This handle will be created by the test multiplexer for the automatically
// opened socket when automatic connections are enabled.
auto hdl_jup = datagram_handle::from_int(std::numeric_limits<int64_t>::max());
// Prepare automatic connection between Jupiter and Earth.
prepare_endpoints(jupiter, earth, "jupiter", port_jupiter, hdl_jup);
// Add the address information for this test to the config server on Mars.
auto mars_config_server = mars.sys.registry().get(atom("PeerServ"));
network::address_listing interfaces{
{network::protocol::ipv4, std::vector<std::string>{"jupiter"}}
};
basp::routing_table::address_map addrs{
{network::protocol::udp, {port_jupiter, interfaces}}
};
anon_send(actor_cast<actor>(mars_config_server), put_atom::value,
to_string(jupiter.sys.node()), make_message(addrs));
// Trigger the automatic connection setup between the edge nodes.
CAF_MESSAGE("forwarding an actor from Jupiter to Earth via Mars");
anon_send(on_jupiter, begin_atom::value);
exec_all();
// Shutdown its basp broker.
// Shutdown the basp broker of the intermediate node.
anon_send_exit(mars.basp, exit_reason::kill);
exec_all();
// Let the remaining nodes communicate.
......
......@@ -68,6 +68,17 @@ public:
return *res;
}
/// Convenience function for calling `mm.publish` and requiring a valid
/// result.
template <class Handle>
uint16_t publish_udp(Handle whom, uint16_t port, const char* in = nullptr,
bool reuse = false) {
this->sched.inline_next_enqueue();
auto res = mm.publish_udp(whom, port, in, reuse);
CAF_REQUIRE(res);
return *res;
}
/// Convenience function for calling `mm.remote_actor` and requiring a valid
/// result.
template <class Handle = caf::actor>
......@@ -79,6 +90,17 @@ public:
return *res;
}
/// Convenience function for calling `mm.remote_actor` and requiring a valid
/// result.
template <class Handle = caf::actor>
Handle remote_actor_udp(std::string host, uint16_t port) {
this->sched.inline_next_enqueue();
this->sched.after_next_enqueue(exec_all_nodes);
auto res = mm.remote_actor_udp<Handle>(std::move(host), port);
CAF_REQUIRE(res);
return *res;
}
private:
caf::io::basp_broker* get_basp_broker() {
auto hdl = mm.named_broker<caf::io::basp_broker>(caf::atom("BASP"));
......@@ -126,6 +148,8 @@ public:
using accept_handle = caf::io::accept_handle;
using datagram_handle = caf::io::datagram_handle;
fake_network_fixture_base(planets_vector xs) : planets_(std::move(xs)) {
// nop
}
......@@ -140,6 +164,11 @@ public:
return connection_handle::from_int(++hdl_id_);
}
/// Returns a unique datagram handle.
datagram_handle next_datagram_handle() {
return datagram_handle::from_int(++hdl_id_);
}
/// Prepare a connection from `client` (calls `remote_actor`) to `server`
/// (calls `publish`).
/// @returns randomly picked connection handles for the server and the client.
......@@ -164,6 +193,31 @@ public:
next_accept_handle());
}
/// Prepares comminication between `client` (calls `remote_actor_udp`) and
/// `server` (calls `publish_udp`).
/// @returns randomly picked datagram handles for the server and the client.
std::pair<datagram_handle, datagram_handle>
prepare_endpoints(PlanetType& server, PlanetType& client,
std::string host, uint16_t port,
datagram_handle server_endpoint) {
auto server_hdl = next_datagram_handle();
auto client_hdl = next_datagram_handle();
server.mpx.prepare_endpoints(server_endpoint, server_hdl, client.mpx,
std::move(host), port, client_hdl);
return std::make_pair(server_hdl, client_hdl);
}
/// Prepares comminication between `client` (calls `remote_actor_udp`) and
/// `server` (calls `publish_udp`).
/// @returns randomly picked datagram handles for the server and the client.
std::pair<datagram_handle, datagram_handle>
prepare_endpoints(PlanetType& server, PlanetType& client,
std::string host, uint16_t port) {
return prepare_endpoints(server, client, std::move(host), port,
next_datagram_handle());
}
// Convenience function for transmitting all "network" traffic (no new
// connections are accepted).
void network_traffic() {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment