Commit fe9b3ed2 authored by Joseph Noir's avatar Joseph Noir

Add sequence numbers for ordering to basp messages

Work for messages in right order, next up: add out of order delivery
to test multiplexer and ensure buffering and delviery for unordered
messages.
parent 6374186c
...@@ -104,6 +104,12 @@ public: ...@@ -104,6 +104,12 @@ public:
return namespace_.system(); return namespace_.system();
} }
// return the next outgoing sequence number for a connection
virtual uint16_t next_sequence_number(connection_handle hdl) = 0;
// return the next outgoing sequence number for an endpoint
virtual uint16_t next_sequence_number(dgram_scribe_handle hdl) = 0;
protected: protected:
proxy_registry namespace_; proxy_registry namespace_;
}; };
...@@ -197,25 +203,30 @@ public: ...@@ -197,25 +203,30 @@ public:
/// actor published at `port` to `buf`. If `port == none` or /// actor published at `port` to `buf`. If `port == none` or
/// if no actor is published at this port then a standard handshake is /// if no actor is published at this port then a standard handshake is
/// written (e.g. used when establishing direct connections on-the-fly). /// written (e.g. used when establishing direct connections on-the-fly).
void write_server_handshake(execution_unit* ctx, void write_server_handshake(execution_unit* ctx, buffer_type& buf,
buffer_type& buf, optional<uint16_t> port); optional<uint16_t> port,
uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`. /// Writes the client handshake to `buf`.
void write_client_handshake(execution_unit* ctx, void write_client_handshake(execution_unit* ctx, buffer_type& buf,
buffer_type& buf, const node_id& remote_side); const node_id& remote_side,
uint16_t sequence_number = 0);
/// Writes an `announce_proxy` to `buf`. /// Writes an `announce_proxy` to `buf`.
void write_announce_proxy(execution_unit* ctx, buffer_type& buf, void write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid); const node_id& dest_node, actor_id aid,
uint16_t sequence_number = 0);
/// Writes a `kill_proxy` to `buf`. /// Writes a `kill_proxy` to `buf`.
void write_kill_proxy(execution_unit* ctx, buffer_type& buf, void write_kill_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid, const node_id& dest_node, actor_id aid,
const error& fail_state); const error& fail_state,
uint16_t sequence_number = 0);
/// Writes a `heartbeat` to `buf`. /// Writes a `heartbeat` to `buf`.
void write_heartbeat(execution_unit* ctx, void write_heartbeat(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side); buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0);
inline const node_id& this_node() const { inline const node_id& this_node() const {
return this_node_; return this_node_;
...@@ -234,9 +245,7 @@ public: ...@@ -234,9 +245,7 @@ public:
template <class Handle> template <class Handle>
bool handle_msg(execution_unit* ctx, const Handle& hdl, header& hdr, bool handle_msg(execution_unit* ctx, const Handle& hdl, header& hdr,
std::vector<char>* payload, bool tcp_based, std::vector<char>* payload, bool tcp_based,
optional<uint16_t> port) { optional<endpoint_context&> ep, optional<uint16_t> port) {
// std::cerr << "[MSG] From " << hdl.id() << " (" << to_string(hdr.operation)
// << ")" << std::endl;
auto payload_valid = [&]() -> bool { auto payload_valid = [&]() -> bool {
return payload != nullptr && payload->size() == hdr.payload_len; return payload != nullptr && payload->size() == hdr.payload_len;
}; };
...@@ -286,7 +295,8 @@ public: ...@@ -286,7 +295,8 @@ public:
return false; return false;
} }
if (tcp_based) if (tcp_based)
write_client_handshake(ctx, apply_visitor(wr_buf_, path->hdl), hdr.source_node); write_client_handshake(ctx, apply_visitor(wr_buf_, path->hdl),
hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node); callee_.learned_new_node_directly(hdr.source_node);
callee_.finalize_handshake(hdr.source_node, aid, sigs); callee_.finalize_handshake(hdr.source_node, aid, sigs);
flush(*path); flush(*path);
...@@ -319,7 +329,8 @@ public: ...@@ -319,7 +329,8 @@ public:
tbl_.add(hdl, hdr.source_node); tbl_.add(hdl, hdr.source_node);
} }
if (!tcp_based) { if (!tcp_based) {
write_server_handshake(ctx, wr_buf_.ptr->wr_buf(hdl), port); auto seq = (ep && ep->requires_ordering) ? ep->seq_outgoing++ : 0;
write_server_handshake(ctx, wr_buf_.ptr->wr_buf(hdl), port, seq);
wr_buf_.ptr->flush(hdl); wr_buf_.ptr->flush(hdl);
} }
if (!is_known_node) { if (!is_known_node) {
...@@ -390,12 +401,22 @@ public: ...@@ -390,12 +401,22 @@ public:
} }
private: private:
struct sequence_number_visitor {
using result_type = uint16_t;
sequence_number_visitor(instance::callee& c) : cal{c} { }
template <class T>
result_type operator()(const T& hdl) {
return cal.next_sequence_number(hdl);
}
instance::callee& cal;
};
routing_table tbl_; routing_table tbl_;
published_actor_map published_actors_; published_actor_map published_actors_;
node_id this_node_; node_id this_node_;
callee& callee_; callee& callee_;
flush_visitor flush_; flush_visitor flush_;
wr_buf_visitor wr_buf_; wr_buf_visitor wr_buf_;
sequence_number_visitor seq_num_;
}; };
/// @} /// @}
......
...@@ -98,6 +98,13 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -98,6 +98,13 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// nop // nop
} }
// inherited from basp::instance::listener
uint16_t next_sequence_number(connection_handle hdl) override;
uint16_t next_sequence_number(dgram_scribe_handle hdl) override;
void purge(connection_handle hdl);
void purge(dgram_scribe_handle hdl);
void set_context(connection_handle hdl); void set_context(connection_handle hdl);
void set_context(dgram_scribe_handle hdl); void set_context(dgram_scribe_handle hdl);
...@@ -105,12 +112,24 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -105,12 +112,24 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
struct purge_visitor { struct purge_visitor {
using result_type = void; using result_type = void;
purge_visitor(basp_broker_state* ptr) : state{ptr} { } purge_visitor(basp_broker_state* ptr) : state{ptr} { }
result_type operator()(const connection_handle& hdl); template <class T>
result_type operator()(const dgram_scribe_handle& hdl); result_type operator()(const T& hdl) {
return state->purge(hdl);
}
basp_broker_state* state;
};
struct sequence_number_visitor {
using result_type = uint16_t;
sequence_number_visitor(basp_broker_state* ptr) : state{ptr} { }
template <class T>
result_type operator()(const T& hdl) {
return state->next_sequence_number(hdl);
}
basp_broker_state* state; basp_broker_state* state;
}; };
purge_visitor purge_state_vis;
wr_buf_visitor wr_buf_vis; wr_buf_visitor wr_buf_vis;
purge_visitor purge_state_vis;
sequence_number_visitor seq_num_vis;
// pointer to ourselves // pointer to ourselves
broker* self; broker* self;
......
...@@ -48,6 +48,9 @@ struct endpoint_context { ...@@ -48,6 +48,9 @@ struct endpoint_context {
uint16_t remote_port; uint16_t remote_port;
// pending operations to be performed after handshake completed // pending operations to be performed after handshake completed
optional<response_promise> callback; optional<response_promise> callback;
// TODO: introduce some call to ask scribe for such info
// Should be configurable by policies in the future
bool requires_ordering;
uint16_t seq_incoming; uint16_t seq_incoming;
uint16_t seq_outgoing; uint16_t seq_outgoing;
std::unordered_map<uint16_t, std::pair<basp::header,std::vector<char>>> pending; std::unordered_map<uint16_t, std::pair<basp::header,std::vector<char>>> pending;
......
...@@ -32,7 +32,7 @@ struct wr_buf_visitor { ...@@ -32,7 +32,7 @@ struct wr_buf_visitor {
result_type operator()(const Handle& hdl) { return ptr->wr_buf(hdl); } result_type operator()(const Handle& hdl) { return ptr->wr_buf(hdl); }
abstract_broker* ptr; abstract_broker* ptr;
}; };
struct flush_visitor { struct flush_visitor {
using result_type = void; using result_type = void;
flush_visitor(abstract_broker* ptr) : ptr{ptr} { } flush_visitor(abstract_broker* ptr) : ptr{ptr} { }
......
...@@ -50,8 +50,9 @@ const char* basp_broker_state::name = "basp_broker"; ...@@ -50,8 +50,9 @@ const char* basp_broker_state::name = "basp_broker";
basp_broker_state::basp_broker_state(broker* selfptr) basp_broker_state::basp_broker_state(broker* selfptr)
: basp::instance::callee(selfptr->system(), : basp::instance::callee(selfptr->system(),
static_cast<proxy_registry::backend&>(*this)), static_cast<proxy_registry::backend&>(*this)),
purge_state_vis(this),
wr_buf_vis(selfptr), wr_buf_vis(selfptr),
purge_state_vis(this),
seq_num_vis(this),
self(selfptr), self(selfptr),
instance(selfptr, *this) { instance(selfptr, *this) {
CAF_ASSERT(this_node() != none); CAF_ASSERT(this_node() != none);
...@@ -108,10 +109,12 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) { ...@@ -108,10 +109,12 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
CAF_LOG_INFO("successfully created proxy instance, " CAF_LOG_INFO("successfully created proxy instance, "
"write announce_proxy_instance:" "write announce_proxy_instance:"
<< CAF_ARG(nid) << CAF_ARG(aid)); << CAF_ARG(nid) << CAF_ARG(aid));
auto& ctx = *this_context;
// tell remote side we are monitoring this actor now // tell remote side we are monitoring this actor now
instance.write_announce_proxy(self->context(), instance.write_announce_proxy(self->context(),
apply_visitor(wr_buf_vis, this_context->hdl), apply_visitor(wr_buf_vis, ctx.hdl),
nid, aid); nid, aid,
ctx.requires_ordering ? ctx.seq_outgoing++ : 0);
instance.tbl().flush(*path); instance.tbl().flush(*path);
mm->notify<hook::new_remote_actor>(res); mm->notify<hook::new_remote_actor>(res);
return res; return res;
...@@ -175,7 +178,8 @@ void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) { ...@@ -175,7 +178,8 @@ void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) {
} }
instance.write_kill_proxy(self->context(), instance.write_kill_proxy(self->context(),
apply_visitor(wr_buf_vis, path->hdl), apply_visitor(wr_buf_vis, path->hdl),
nid, aid, rsn); nid, aid, rsn,
apply_visitor(seq_num_vis, path->hdl));
instance.tbl().flush(*path); instance.tbl().flush(*path);
}; };
auto ptr = actor_cast<strong_actor_ptr>(entry); auto ptr = actor_cast<strong_actor_ptr>(entry);
...@@ -353,7 +357,8 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -353,7 +357,8 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
// send message to SpawnServ of remote node // send message to SpawnServ of remote node
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id}; 0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
apply_visitor(seq_num_vis, path->hdl)};
// writing std::numeric_limits<actor_id>::max() is a hack to get // writing std::numeric_limits<actor_id>::max() is a hack to get
// this send-to-named-actor feature working with older CAF releases // this send-to-named-actor feature working with older CAF releases
instance.write(self->context(), apply_visitor(wr_buf_vis, path->hdl), hdr, instance.write(self->context(), apply_visitor(wr_buf_vis, path->hdl), hdr,
...@@ -444,12 +449,37 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) { ...@@ -444,12 +449,37 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) {
}); });
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id}; 0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
apply_visitor(state.seq_num_vis, path->hdl)};
instance.write(self->context(), path->wr_buf, hdr, &writer); instance.write(self->context(), path->wr_buf, hdr, &writer);
instance.flush(*path); instance.flush(*path);
} }
*/ */
void basp_broker_state::purge(connection_handle h) {
auto i = tcp_ctx.find(h);
if (i != tcp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
tcp_ctx.erase(i);
}
}
void basp_broker_state::purge(dgram_scribe_handle h) {
auto i = udp_ctx.find(h);
if (i != udp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
udp_ctx.erase(i);
}
}
void basp_broker_state::set_context(connection_handle hdl) { void basp_broker_state::set_context(connection_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl)); CAF_LOG_TRACE(CAF_ARG(hdl));
auto i = tcp_ctx.find(hdl); auto i = tcp_ctx.find(hdl);
...@@ -463,7 +493,7 @@ void basp_broker_state::set_context(connection_handle hdl) { ...@@ -463,7 +493,7 @@ void basp_broker_state::set_context(connection_handle hdl) {
0, 0, 0, none, none, 0, 0, 0, none, none,
invalid_actor_id, invalid_actor_id}, invalid_actor_id, invalid_actor_id},
hdl, none, 0, none, hdl, none, 0, none,
0, 0 false, 0, 0, {}
} }
).first; ).first;
} }
...@@ -484,37 +514,22 @@ void basp_broker_state::set_context(dgram_scribe_handle hdl) { ...@@ -484,37 +514,22 @@ void basp_broker_state::set_context(dgram_scribe_handle hdl) {
0, 0, 0, none, none, 0, 0, 0, none, none,
invalid_actor_id, invalid_actor_id}, invalid_actor_id, invalid_actor_id},
hdl, none, 0, none, hdl, none, 0, none,
0, 0 true, 0, 0, {}
} }
).first; ).first;
} }
this_context = &i->second; this_context = &i->second;
} }
basp_broker_state::purge_visitor::result_type uint16_t basp_broker_state::next_sequence_number(connection_handle) {
basp_broker_state::purge_visitor::operator()(const connection_handle& h) { return 0;
auto i = state->tcp_ctx.find(h);
if (i != state->tcp_ctx.end()) {
auto& ref = i->second;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
state->tcp_ctx.erase(i);
}
} }
basp_broker_state::purge_visitor::result_type uint16_t basp_broker_state::next_sequence_number(dgram_scribe_handle hdl) {
basp_broker_state::purge_visitor::operator()(const dgram_scribe_handle& h) { auto i = udp_ctx.find(hdl);
auto i = state->udp_ctx.find(h); if (i != udp_ctx.end() && i->second.requires_ordering)
if (i != state->udp_ctx.end()) { return i->second.seq_outgoing++;
auto& ref = i->second; return 0;
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
state->udp_ctx.erase(i);
}
} }
...@@ -607,8 +622,7 @@ behavior basp_broker::make_behavior() { ...@@ -607,8 +622,7 @@ behavior basp_broker::make_behavior() {
} }
if (src && system().node() == src->node()) if (src && system().node() == src->node())
system().registry().put(src->id(), src); system().registry().put(src->id(), src);
if (!state.instance.dispatch(context(), src, fwd_stack, if (!state.instance.dispatch(context(), src, fwd_stack, dest, mid, msg)
dest, mid, msg)
&& mid.is_request()) { && mid.is_request()) {
detail::sync_request_bouncer srb{exit_reason::remote_link_unreachable}; detail::sync_request_bouncer srb{exit_reason::remote_link_unreachable};
srb(src, mid); srb(src, mid);
...@@ -640,7 +654,8 @@ behavior basp_broker::make_behavior() { ...@@ -640,7 +654,8 @@ behavior basp_broker::make_behavior() {
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(), 0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id}; dest_node, src->id(), invalid_actor_id,
apply_visitor(state.seq_num_vis, path->hdl)};
state.instance.write(context(), apply_visitor(wr_buf_, path->hdl), hdr, state.instance.write(context(), apply_visitor(wr_buf_, path->hdl), hdr,
&writer); &writer);
state.instance.flush(*path); state.instance.flush(*path);
...@@ -771,10 +786,17 @@ behavior basp_broker::make_behavior() { ...@@ -771,10 +786,17 @@ behavior basp_broker::make_behavior() {
ctx.hdl = hdl; ctx.hdl = hdl;
ctx.remote_port = port; ctx.remote_port = port;
ctx.callback = rp; ctx.callback = rp;
// assuming all UDP based protocol need ordering ...
// TODO: get this information from the policy
ctx.requires_ordering = true;
ctx.seq_incoming = 0;
ctx.seq_outgoing = 0;
auto& bi = state.instance; auto& bi = state.instance;
bi.write_client_handshake(context(), wr_buf(hdl), none); bi.write_client_handshake(context(), wr_buf(hdl), none,
ctx.seq_outgoing++);
flush(hdl); flush(hdl);
configure_datagram_size(hdl, 1500); configure_datagram_size(hdl, 1500);
// TODO: Set timeout for expected answer
} else { } else {
CAF_LOG_DEBUG("failed to assign datagram sink from handle" CAF_LOG_DEBUG("failed to assign datagram sink from handle"
<< CAF_ARG(res)); << CAF_ARG(res));
......
This diff is collapsed.
...@@ -1047,7 +1047,8 @@ CAF_TEST(client_handshake_and_dispatch_udp) { ...@@ -1047,7 +1047,8 @@ CAF_TEST(client_handshake_and_dispatch_udp) {
// send a message via `dispatch` from node 0 // send a message via `dispatch` from node 0
mock(jupiter().connection, mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id()}, jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(),
1},
std::vector<actor_addr>{}, std::vector<actor_addr>{},
make_message(1, 2, 3)) make_message(1, 2, 3))
.expect(jupiter().connection, .expect(jupiter().connection,
...@@ -1167,7 +1168,7 @@ CAF_TEST(remote_actor_and_send_udp) { ...@@ -1167,7 +1168,7 @@ CAF_TEST(remote_actor_and_send_udp) {
mock(jupiter().connection, mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id()}, jupiter().dummy_actor->id(), self()->id(), 1},
std::vector<actor_id>{}, std::vector<actor_id>{},
make_message("hi there!")); make_message("hi there!"));
self()->receive( self()->receive(
...@@ -1204,7 +1205,7 @@ CAF_TEST(actor_serialize_and_deserialize_udp) { ...@@ -1204,7 +1205,7 @@ CAF_TEST(actor_serialize_and_deserialize_udp) {
mock(jupiter().connection, mock(jupiter().connection,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
prx->node(), this_node(), prx->node(), this_node(),
prx->id(), testee->id()}, prx->id(), testee->id(), 1},
std::vector<actor_id>{}, std::vector<actor_id>{},
msg); msg);
// testee must've responded (process forwarded message in BASP broker) // testee must've responded (process forwarded message in BASP broker)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment