Commit fe9b3ed2 authored by Joseph Noir's avatar Joseph Noir

Add sequence numbers for ordering to basp messages

Work for messages in right order, next up: add out of order delivery
to test multiplexer and ensure buffering and delviery for unordered
messages.
parent 6374186c
......@@ -104,6 +104,12 @@ public:
return namespace_.system();
}
// return the next outgoing sequence number for a connection
virtual uint16_t next_sequence_number(connection_handle hdl) = 0;
// return the next outgoing sequence number for an endpoint
virtual uint16_t next_sequence_number(dgram_scribe_handle hdl) = 0;
protected:
proxy_registry namespace_;
};
......@@ -197,25 +203,30 @@ public:
/// actor published at `port` to `buf`. If `port == none` or
/// if no actor is published at this port then a standard handshake is
/// written (e.g. used when establishing direct connections on-the-fly).
void write_server_handshake(execution_unit* ctx,
buffer_type& buf, optional<uint16_t> port);
void write_server_handshake(execution_unit* ctx, buffer_type& buf,
optional<uint16_t> port,
uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`.
void write_client_handshake(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side);
void write_client_handshake(execution_unit* ctx, buffer_type& buf,
const node_id& remote_side,
uint16_t sequence_number = 0);
/// Writes an `announce_proxy` to `buf`.
void write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid);
const node_id& dest_node, actor_id aid,
uint16_t sequence_number = 0);
/// Writes a `kill_proxy` to `buf`.
void write_kill_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid,
const error& fail_state);
const error& fail_state,
uint16_t sequence_number = 0);
/// Writes a `heartbeat` to `buf`.
void write_heartbeat(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side);
buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0);
inline const node_id& this_node() const {
return this_node_;
......@@ -234,9 +245,7 @@ public:
template <class Handle>
bool handle_msg(execution_unit* ctx, const Handle& hdl, header& hdr,
std::vector<char>* payload, bool tcp_based,
optional<uint16_t> port) {
// std::cerr << "[MSG] From " << hdl.id() << " (" << to_string(hdr.operation)
// << ")" << std::endl;
optional<endpoint_context&> ep, optional<uint16_t> port) {
auto payload_valid = [&]() -> bool {
return payload != nullptr && payload->size() == hdr.payload_len;
};
......@@ -286,7 +295,8 @@ public:
return false;
}
if (tcp_based)
write_client_handshake(ctx, apply_visitor(wr_buf_, path->hdl), hdr.source_node);
write_client_handshake(ctx, apply_visitor(wr_buf_, path->hdl),
hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node);
callee_.finalize_handshake(hdr.source_node, aid, sigs);
flush(*path);
......@@ -319,7 +329,8 @@ public:
tbl_.add(hdl, hdr.source_node);
}
if (!tcp_based) {
write_server_handshake(ctx, wr_buf_.ptr->wr_buf(hdl), port);
auto seq = (ep && ep->requires_ordering) ? ep->seq_outgoing++ : 0;
write_server_handshake(ctx, wr_buf_.ptr->wr_buf(hdl), port, seq);
wr_buf_.ptr->flush(hdl);
}
if (!is_known_node) {
......@@ -390,12 +401,22 @@ public:
}
private:
struct sequence_number_visitor {
using result_type = uint16_t;
sequence_number_visitor(instance::callee& c) : cal{c} { }
template <class T>
result_type operator()(const T& hdl) {
return cal.next_sequence_number(hdl);
}
instance::callee& cal;
};
routing_table tbl_;
published_actor_map published_actors_;
node_id this_node_;
callee& callee_;
flush_visitor flush_;
wr_buf_visitor wr_buf_;
sequence_number_visitor seq_num_;
};
/// @}
......
......@@ -98,6 +98,13 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// nop
}
// inherited from basp::instance::listener
uint16_t next_sequence_number(connection_handle hdl) override;
uint16_t next_sequence_number(dgram_scribe_handle hdl) override;
void purge(connection_handle hdl);
void purge(dgram_scribe_handle hdl);
void set_context(connection_handle hdl);
void set_context(dgram_scribe_handle hdl);
......@@ -105,12 +112,24 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
struct purge_visitor {
using result_type = void;
purge_visitor(basp_broker_state* ptr) : state{ptr} { }
result_type operator()(const connection_handle& hdl);
result_type operator()(const dgram_scribe_handle& hdl);
template <class T>
result_type operator()(const T& hdl) {
return state->purge(hdl);
}
basp_broker_state* state;
};
struct sequence_number_visitor {
using result_type = uint16_t;
sequence_number_visitor(basp_broker_state* ptr) : state{ptr} { }
template <class T>
result_type operator()(const T& hdl) {
return state->next_sequence_number(hdl);
}
basp_broker_state* state;
};
purge_visitor purge_state_vis;
wr_buf_visitor wr_buf_vis;
purge_visitor purge_state_vis;
sequence_number_visitor seq_num_vis;
// pointer to ourselves
broker* self;
......
......@@ -48,6 +48,9 @@ struct endpoint_context {
uint16_t remote_port;
// pending operations to be performed after handshake completed
optional<response_promise> callback;
// TODO: introduce some call to ask scribe for such info
// Should be configurable by policies in the future
bool requires_ordering;
uint16_t seq_incoming;
uint16_t seq_outgoing;
std::unordered_map<uint16_t, std::pair<basp::header,std::vector<char>>> pending;
......
......@@ -32,7 +32,7 @@ struct wr_buf_visitor {
result_type operator()(const Handle& hdl) { return ptr->wr_buf(hdl); }
abstract_broker* ptr;
};
struct flush_visitor {
using result_type = void;
flush_visitor(abstract_broker* ptr) : ptr{ptr} { }
......
......@@ -50,8 +50,9 @@ const char* basp_broker_state::name = "basp_broker";
basp_broker_state::basp_broker_state(broker* selfptr)
: basp::instance::callee(selfptr->system(),
static_cast<proxy_registry::backend&>(*this)),
purge_state_vis(this),
wr_buf_vis(selfptr),
purge_state_vis(this),
seq_num_vis(this),
self(selfptr),
instance(selfptr, *this) {
CAF_ASSERT(this_node() != none);
......@@ -108,10 +109,12 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
CAF_LOG_INFO("successfully created proxy instance, "
"write announce_proxy_instance:"
<< CAF_ARG(nid) << CAF_ARG(aid));
auto& ctx = *this_context;
// tell remote side we are monitoring this actor now
instance.write_announce_proxy(self->context(),
apply_visitor(wr_buf_vis, this_context->hdl),
nid, aid);
apply_visitor(wr_buf_vis, ctx.hdl),
nid, aid,
ctx.requires_ordering ? ctx.seq_outgoing++ : 0);
instance.tbl().flush(*path);
mm->notify<hook::new_remote_actor>(res);
return res;
......@@ -175,7 +178,8 @@ void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) {
}
instance.write_kill_proxy(self->context(),
apply_visitor(wr_buf_vis, path->hdl),
nid, aid, rsn);
nid, aid, rsn,
apply_visitor(seq_num_vis, path->hdl));
instance.tbl().flush(*path);
};
auto ptr = actor_cast<strong_actor_ptr>(entry);
......@@ -353,7 +357,8 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
// send message to SpawnServ of remote node
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id};
0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
apply_visitor(seq_num_vis, path->hdl)};
// writing std::numeric_limits<actor_id>::max() is a hack to get
// this send-to-named-actor feature working with older CAF releases
instance.write(self->context(), apply_visitor(wr_buf_vis, path->hdl), hdr,
......@@ -444,12 +449,37 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) {
});
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id};
0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
apply_visitor(state.seq_num_vis, path->hdl)};
instance.write(self->context(), path->wr_buf, hdr, &writer);
instance.flush(*path);
}
*/
void basp_broker_state::purge(connection_handle h) {
auto i = tcp_ctx.find(h);
if (i != tcp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
tcp_ctx.erase(i);
}
}
void basp_broker_state::purge(dgram_scribe_handle h) {
auto i = udp_ctx.find(h);
if (i != udp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
udp_ctx.erase(i);
}
}
void basp_broker_state::set_context(connection_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl));
auto i = tcp_ctx.find(hdl);
......@@ -463,7 +493,7 @@ void basp_broker_state::set_context(connection_handle hdl) {
0, 0, 0, none, none,
invalid_actor_id, invalid_actor_id},
hdl, none, 0, none,
0, 0
false, 0, 0, {}
}
).first;
}
......@@ -484,37 +514,22 @@ void basp_broker_state::set_context(dgram_scribe_handle hdl) {
0, 0, 0, none, none,
invalid_actor_id, invalid_actor_id},
hdl, none, 0, none,
0, 0
true, 0, 0, {}
}
).first;
}
this_context = &i->second;
}
basp_broker_state::purge_visitor::result_type
basp_broker_state::purge_visitor::operator()(const connection_handle& h) {
auto i = state->tcp_ctx.find(h);
if (i != state->tcp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
state->tcp_ctx.erase(i);
}
uint16_t basp_broker_state::next_sequence_number(connection_handle) {
return 0;
}
basp_broker_state::purge_visitor::result_type
basp_broker_state::purge_visitor::operator()(const dgram_scribe_handle& h) {
auto i = state->udp_ctx.find(h);
if (i != state->udp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
state->udp_ctx.erase(i);
}
uint16_t basp_broker_state::next_sequence_number(dgram_scribe_handle hdl) {
auto i = udp_ctx.find(hdl);
if (i != udp_ctx.end() && i->second.requires_ordering)
return i->second.seq_outgoing++;
return 0;
}
......@@ -607,8 +622,7 @@ behavior basp_broker::make_behavior() {
}
if (src && system().node() == src->node())
system().registry().put(src->id(), src);
if (!state.instance.dispatch(context(), src, fwd_stack,
dest, mid, msg)
if (!state.instance.dispatch(context(), src, fwd_stack, dest, mid, msg)
&& mid.is_request()) {
detail::sync_request_bouncer srb{exit_reason::remote_link_unreachable};
srb(src, mid);
......@@ -640,7 +654,8 @@ behavior basp_broker::make_behavior() {
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id};
dest_node, src->id(), invalid_actor_id,
apply_visitor(state.seq_num_vis, path->hdl)};
state.instance.write(context(), apply_visitor(wr_buf_, path->hdl), hdr,
&writer);
state.instance.flush(*path);
......@@ -771,10 +786,17 @@ behavior basp_broker::make_behavior() {
ctx.hdl = hdl;
ctx.remote_port = port;
ctx.callback = rp;
// assuming all UDP based protocol need ordering ...
// TODO: get this information from the policy
ctx.requires_ordering = true;
ctx.seq_incoming = 0;
ctx.seq_outgoing = 0;
auto& bi = state.instance;
bi.write_client_handshake(context(), wr_buf(hdl), none);
bi.write_client_handshake(context(), wr_buf(hdl), none,
ctx.seq_outgoing++);
flush(hdl);
configure_datagram_size(hdl, 1500);
// TODO: Set timeout for expected answer
} else {
CAF_LOG_DEBUG("failed to assign datagram sink from handle"
<< CAF_ARG(res));
......
This diff is collapsed.
......@@ -1047,7 +1047,8 @@ CAF_TEST(client_handshake_and_dispatch_udp) {
// send a message via `dispatch` from node 0
mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id()},
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(),
1},
std::vector<actor_addr>{},
make_message(1, 2, 3))
.expect(jupiter().connection,
......@@ -1167,7 +1168,7 @@ CAF_TEST(remote_actor_and_send_udp) {
mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id()},
jupiter().dummy_actor->id(), self()->id(), 1},
std::vector<actor_id>{},
make_message("hi there!"));
self()->receive(
......@@ -1204,7 +1205,7 @@ CAF_TEST(actor_serialize_and_deserialize_udp) {
mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0,
prx->node(), this_node(),
prx->id(), testee->id()},
prx->id(), testee->id(), 1},
std::vector<actor_id>{},
msg);
// testee must've responded (process forwarded message in BASP broker)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment