Commit aa3f2518 authored by Joseph Noir's avatar Joseph Noir

Buffer messages if connection is pending

parent 24073a6d
...@@ -130,6 +130,11 @@ public: ...@@ -130,6 +130,11 @@ public:
/// Returns a reference to the sent buffer. /// Returns a reference to the sent buffer.
virtual buffer_type& get_buffer(connection_handle hdl) = 0; virtual buffer_type& get_buffer(connection_handle hdl) = 0;
/// Returns a reference to a buffer to be sent to node with `nid`.
/// If communication with the node is esstablished, it picks the first
/// available handle, otherwise a buffer for a pending message is returned.
virtual buffer_type& get_buffer(node_id nid) = 0;
/// Returns the buffer accessed through a call to `get_buffer` when /// Returns the buffer accessed through a call to `get_buffer` when
/// passing a datagram handle and removes it from the callee. /// passing a datagram handle and removes it from the callee.
......
...@@ -109,6 +109,9 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -109,6 +109,9 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// inherited from basp::instance::callee // inherited from basp::instance::callee
buffer_type& get_buffer(connection_handle hdl) override; buffer_type& get_buffer(connection_handle hdl) override;
// inherited from basp::instance::callee
buffer_type& get_buffer(node_id nid) override;
// inherited from basp::instance::callee // inherited from basp::instance::callee
buffer_type pop_datagram_buffer(datagram_handle hdl) override; buffer_type pop_datagram_buffer(datagram_handle hdl) override;
...@@ -169,9 +172,13 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -169,9 +172,13 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// maximum queue size for pending messages of endpoints with ordering // maximum queue size for pending messages of endpoints with ordering
const size_t max_pending_messages; const size_t max_pending_messages;
// buffer messages for nodes while connectivity is established
std::unordered_map<node_id, std::vector<buffer_type>> pending_connectivity;
// timeout for delivery of pending messages of endpoints with ordering // timeout for delivery of pending messages of endpoints with ordering
const std::chrono::milliseconds pending_to = std::chrono::milliseconds(100); const std::chrono::milliseconds pending_timeout
= std::chrono::milliseconds(100);
// returns the node identifier of the underlying BASP instance // returns the node identifier of the underlying BASP instance
const node_id& this_node() const { const node_id& this_node() const {
......
...@@ -201,9 +201,11 @@ void basp_broker_state::send_kill_proxy_instance(const node_id& nid, ...@@ -201,9 +201,11 @@ void basp_broker_state::send_kill_proxy_instance(const node_id& nid,
visit(seq_num_visitor{this}, *c.hdl)); visit(seq_num_visitor{this}, *c.hdl));
instance.flush(*c.hdl); instance.flush(*c.hdl);
} else { } else {
// TODO: Buffer message! // TODO: Buffer message until communication is enstablished.
CAF_CRITICAL("ibasp_broker_state::send_kill_proxy_instance with buffering " buffer_type buf;
"not implemented!"); instance.write_kill_proxy(self->context(), buf,
nid, aid, rsn, 0);
pending_connectivity[nid].emplace_back(std::move(buf));
} }
} }
...@@ -383,12 +385,13 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -383,12 +385,13 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
return; return;
} }
auto c = std::move(*ec); auto c = std::move(*ec);
// send message to SpawnServ of remote node
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
0}; // sequence number only available with connectivity
if (c.conn == basp::routing_table::connectivity::established) { if (c.conn == basp::routing_table::connectivity::established) {
// send message to SpawnServ of remote node hdr.sequence_number = visit(seq_num_visitor{this}, *c.hdl);
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
visit(seq_num_visitor{this}, *c.hdl)};
// writing std::numeric_limits<actor_id>::max() is a hack to get // writing std::numeric_limits<actor_id>::max() is a hack to get
// this send-to-named-actor feature working with older CAF releases // this send-to-named-actor feature working with older CAF releases
instance.write(self->context(), get_buffer(*c.hdl), instance.write(self->context(), get_buffer(*c.hdl),
...@@ -396,8 +399,9 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -396,8 +399,9 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
instance.flush(*c.hdl); instance.flush(*c.hdl);
} else { } else {
// TODO: Implement this, can it happen? // TODO: Implement this, can it happen?
CAF_CRITICAL("basp_broker_state::learned_new_node with buffering not " buffer_type buf;
"implemented!"); instance.write(self->context(), buf, hdr, &writer);
pending_connectivity[nid].emplace_back(std::move(buf));
} }
} }
...@@ -510,7 +514,7 @@ void basp_broker_state::add_pending(execution_unit* ctx, ...@@ -510,7 +514,7 @@ void basp_broker_state::add_pending(execution_unit* ctx,
if (ep.pending.size() >= max_pending_messages) if (ep.pending.size() >= max_pending_messages)
deliver_pending(ctx, ep, true); deliver_pending(ctx, ep, true);
else if (!ep.did_set_timeout) else if (!ep.did_set_timeout)
self->delayed_send(self, pending_to, pending_atom::value, self->delayed_send(self, pending_timeout, pending_atom::value,
get<datagram_handle>(ep.hdl)); get<datagram_handle>(ep.hdl));
} }
...@@ -535,7 +539,7 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx, ...@@ -535,7 +539,7 @@ bool basp_broker_state::deliver_pending(execution_unit* ctx,
} }
// Set a timeout if there are still pending messages. // Set a timeout if there are still pending messages.
if (!ep.pending.empty() && !ep.did_set_timeout) if (!ep.pending.empty() && !ep.did_set_timeout)
self->delayed_send(self, pending_to, pending_atom::value, self->delayed_send(self, pending_timeout, pending_atom::value,
get<datagram_handle>(ep.hdl)); get<datagram_handle>(ep.hdl));
return true; return true;
} }
...@@ -567,6 +571,17 @@ basp_broker_state::get_buffer(connection_handle hdl) { ...@@ -567,6 +571,17 @@ basp_broker_state::get_buffer(connection_handle hdl) {
return self->wr_buf(hdl); return self->wr_buf(hdl);
} }
basp_broker_state::buffer_type&
basp_broker_state::get_buffer(node_id nid) {
auto ec = instance.tbl().lookup(nid);
if (ec && ec->conn == basp::routing_table::connectivity::established && ec->hdl) {
return get_buffer(*(ec->hdl));
}
auto msgs = pending_connectivity[nid];
msgs.emplace_back();
return msgs.back();
}
basp_broker_state::buffer_type basp_broker_state::buffer_type
basp_broker_state::pop_datagram_buffer(datagram_handle) { basp_broker_state::pop_datagram_buffer(datagram_handle) {
std::vector<char> res; std::vector<char> res;
...@@ -737,24 +752,26 @@ behavior basp_broker::make_behavior() { ...@@ -737,24 +752,26 @@ behavior basp_broker::make_behavior() {
return sec::no_route_to_receiving_node; return sec::no_route_to_receiving_node;
} }
auto c = std::move(*ec); auto c = std::move(*ec);
if (system().node() == src->node())
system().registry().put(src->id(), src);
auto writer = make_callback([&](serializer& sink) -> error {
return sink(dest_name, cme->stages, const_cast<message&>(msg));
});
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id,
0};
if (c.conn == basp::routing_table::connectivity::established) { if (c.conn == basp::routing_table::connectivity::established) {
if (system().node() == src->node()) hdr.sequence_number = visit(seq_num_visitor{&state}, *c.hdl);
system().registry().put(src->id(), src);
auto writer = make_callback([&](serializer& sink) -> error {
return sink(dest_name, cme->stages, const_cast<message&>(msg));
});
basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id,
visit(seq_num_visitor{&state}, *c.hdl)};
state.instance.write(context(), state.get_buffer(*c.hdl), state.instance.write(context(), state.get_buffer(*c.hdl),
hdr, &writer); hdr, &writer);
state.instance.flush(*c.hdl); state.instance.flush(*c.hdl);
} else { } else {
// TODO: Buffer the message in the basp broker. // TODO: Buffer the message in the basp broker.
CAF_CRITICAL("basp_broker forward_atom with buffering " std::vector<char> buf;
"not implemented!"); state.instance.write(context(), buf, hdr, &writer);
state.pending_connectivity[dest_node].emplace_back(buf);
} }
return delegated<message>(); return delegated<message>();
}, },
......
...@@ -254,20 +254,21 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, ...@@ -254,20 +254,21 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
CAF_ASSERT(receiver && system().node() != receiver->node()); CAF_ASSERT(receiver && system().node() != receiver->node());
auto ec = tbl_.lookup(receiver->node()); auto ec = tbl_.lookup(receiver->node());
/// TODO: Let's assume that the handle is valid if the status is established. /// TODO: Let's assume that the handle is valid if the status is established.
if (!ec || ec->cs == routing_table::connectivity::failed) { if (!ec || ec->conn == routing_table::connectivity::failed) {
notify<hook::message_sending_failed>(sender, receiver, mid, msg); notify<hook::message_sending_failed>(sender, receiver, mid, msg);
return false; return false;
} }
auto c = std::move(*ec); auto c = std::move(*ec);
if (c.cs == routing_table::connectivity::established) { auto writer = make_callback([&](serializer& sink) -> error {
auto writer = make_callback([&](serializer& sink) -> error { return sink(const_cast<std::vector<strong_actor_ptr>&>(forwarding_stack),
return sink(const_cast<std::vector<strong_actor_ptr>&>(forwarding_stack), const_cast<message&>(msg));
const_cast<message&>(msg)); });
}); header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(),
header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(), sender ? sender->node() : this_node(), receiver->node(),
sender ? sender->node() : this_node(), receiver->node(), sender ? sender->id() : invalid_actor_id, receiver->id(),
sender ? sender->id() : invalid_actor_id, receiver->id(), 0};
visit(seq_num_visitor{callee_}, *c.hdl)}; if (c.conn == routing_table::connectivity::established) {
hdr.sequence_number = visit(seq_num_visitor{callee_}, *c.hdl);
write(ctx, callee_.get_buffer(*c.hdl), hdr, &writer); write(ctx, callee_.get_buffer(*c.hdl), hdr, &writer);
flush(*c.hdl); flush(*c.hdl);
notify<hook::message_sent>(sender, receiver->node(), receiver, mid, msg); notify<hook::message_sent>(sender, receiver->node(), receiver, mid, msg);
...@@ -275,8 +276,11 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, ...@@ -275,8 +276,11 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
} else { } else {
// lr.cs == routing_table::communication::pending // lr.cs == routing_table::communication::pending
// TODO: Buffer the message in the basp broker. // TODO: Buffer the message in the basp broker.
CAF_CRITICAL("instance::disaptch with buffering not implemented!"); write(ctx, callee_.get_buffer(receiver->node()), hdr, &writer);
return false; // TODO: should the hook really be called here, or should we delay this
// until communication is established?
notify<hook::message_sent>(sender, receiver->node(), receiver, mid, msg);
return true;
} }
} }
......
...@@ -85,7 +85,7 @@ void routing_table::add(const node_id& nid) { ...@@ -85,7 +85,7 @@ void routing_table::add(const node_id& nid) {
bool routing_table::reachable(const node_id& dest) { bool routing_table::reachable(const node_id& dest) {
auto i = node_information_base_.find(dest); auto i = node_information_base_.find(dest);
if (i != node_information_base_.end()) if (i != node_information_base_.end())
return i->second.details.cs == connectivity::established; return i->second.details.conn == connectivity::established;
return false; return false;
} }
...@@ -94,7 +94,7 @@ bool routing_table::status(const node_id& nid, ...@@ -94,7 +94,7 @@ bool routing_table::status(const node_id& nid,
auto i = node_information_base_.find(nid); auto i = node_information_base_.find(nid);
if (i == node_information_base_.end()) if (i == node_information_base_.end())
return false; return false;
i->second.details.cs = new_status; i->second.details.conn = new_status;
return true; return true;
} }
...@@ -103,7 +103,7 @@ routing_table::status(const node_id& nid) { ...@@ -103,7 +103,7 @@ routing_table::status(const node_id& nid) {
auto i = node_information_base_.find(nid); auto i = node_information_base_.find(nid);
if (i == node_information_base_.end()) if (i == node_information_base_.end())
return none; return none;
return i->second.details.cs; return i->second.details.conn;
} }
bool routing_table::forwarder(const node_id& nid, bool routing_table::forwarder(const node_id& nid,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment