Unverified Commit bec8b3fe authored by Dominik Charousset's avatar Dominik Charousset Committed by GitHub

Merge pull request #626

Add basic UDP support
parents 078d2db1 42dcb781
...@@ -53,6 +53,10 @@ detach-utility-actors=true ...@@ -53,6 +53,10 @@ detach-utility-actors=true
; setting this to false allows fully deterministic execution in unit test and ; setting this to false allows fully deterministic execution in unit test and
; requires the user to trigger I/O manually ; requires the user to trigger I/O manually
detach-multiplexer=true detach-multiplexer=true
; enable or disable communication via the TCP transport protocol
middleman_enable_tcp=true
; enable or disable communication via the UDP transport protocol
middleman_enable_udp=false
; when compiling with logging enabled ; when compiling with logging enabled
[logger] [logger]
......
...@@ -257,6 +257,7 @@ public: ...@@ -257,6 +257,7 @@ public:
actor_system_config& set(const char* cn, config_value cv); actor_system_config& set(const char* cn, config_value cv);
// -- config parameters of the scheduler ------------------------------------- // -- config parameters of the scheduler -------------------------------------
atom_value scheduler_policy; atom_value scheduler_policy;
size_t scheduler_max_threads; size_t scheduler_max_threads;
size_t scheduler_max_throughput; size_t scheduler_max_throughput;
...@@ -298,6 +299,9 @@ public: ...@@ -298,6 +299,9 @@ public:
size_t middleman_heartbeat_interval; size_t middleman_heartbeat_interval;
bool middleman_detach_utility_actors; bool middleman_detach_utility_actors;
bool middleman_detach_multiplexer; bool middleman_detach_multiplexer;
bool middleman_enable_tcp;
bool middleman_enable_udp;
size_t middleman_cached_udp_buffers;
// -- config parameters of the OpenCL module --------------------------------- // -- config parameters of the OpenCL module ---------------------------------
......
...@@ -141,9 +141,15 @@ using unlink_atom = atom_constant<atom("unlink")>; ...@@ -141,9 +141,15 @@ using unlink_atom = atom_constant<atom("unlink")>;
/// Used for publishing actors at a given port. /// Used for publishing actors at a given port.
using publish_atom = atom_constant<atom("publish")>; using publish_atom = atom_constant<atom("publish")>;
/// Used for publishing actors at a given port.
using publish_udp_atom = atom_constant<atom("pub_udp")>;
/// Used for removing an actor/port mapping. /// Used for removing an actor/port mapping.
using unpublish_atom = atom_constant<atom("unpublish")>; using unpublish_atom = atom_constant<atom("unpublish")>;
/// Used for removing an actor/port mapping.
using unpublish_udp_atom = atom_constant<atom("unpub_udp")>;
/// Used for signalizing group membership. /// Used for signalizing group membership.
using subscribe_atom = atom_constant<atom("subscribe")>; using subscribe_atom = atom_constant<atom("subscribe")>;
...@@ -153,6 +159,9 @@ using unsubscribe_atom = atom_constant<atom("unsubscrib")>; ...@@ -153,6 +159,9 @@ using unsubscribe_atom = atom_constant<atom("unsubscrib")>;
/// Used for establishing network connections. /// Used for establishing network connections.
using connect_atom = atom_constant<atom("connect")>; using connect_atom = atom_constant<atom("connect")>;
/// Used for contacting a remote UDP endpoint
using contact_atom = atom_constant<atom("contact")>;
/// Used for opening ports or files. /// Used for opening ports or files.
using open_atom = atom_constant<atom("open")>; using open_atom = atom_constant<atom("open")>;
...@@ -168,6 +177,9 @@ using migrate_atom = atom_constant<atom("migrate")>; ...@@ -168,6 +177,9 @@ using migrate_atom = atom_constant<atom("migrate")>;
/// Used for triggering periodic operations. /// Used for triggering periodic operations.
using tick_atom = atom_constant<atom("tick")>; using tick_atom = atom_constant<atom("tick")>;
/// Used for pending out of order messages.
using pending_atom = atom_constant<atom("pending")>;
} // namespace caf } // namespace caf
namespace std { namespace std {
......
...@@ -111,7 +111,9 @@ enum class sec : uint8_t { ...@@ -111,7 +111,9 @@ enum class sec : uint8_t {
/// Stream aborted due to unexpected error. /// Stream aborted due to unexpected error.
unhandled_stream_error, unhandled_stream_error,
/// A function view was called without assigning an actor first. /// A function view was called without assigning an actor first.
bad_function_call = 40 bad_function_call = 40,
/// Feature is disabled in the actor system config.
feature_disabled,
}; };
/// @relates sec /// @relates sec
......
...@@ -137,6 +137,9 @@ actor_system_config::actor_system_config() ...@@ -137,6 +137,9 @@ actor_system_config::actor_system_config()
middleman_heartbeat_interval = 0; middleman_heartbeat_interval = 0;
middleman_detach_utility_actors = true; middleman_detach_utility_actors = true;
middleman_detach_multiplexer = true; middleman_detach_multiplexer = true;
middleman_enable_tcp = true;
middleman_enable_udp = false;
middleman_cached_udp_buffers = 10;
// fill our options vector for creating INI and CLI parsers // fill our options vector for creating INI and CLI parsers
opt_group{options_, "scheduler"} opt_group{options_, "scheduler"}
.add(scheduler_policy, "policy", .add(scheduler_policy, "policy",
...@@ -199,7 +202,14 @@ actor_system_config::actor_system_config() ...@@ -199,7 +202,14 @@ actor_system_config::actor_system_config()
.add(middleman_detach_utility_actors, "detach-utility-actors", .add(middleman_detach_utility_actors, "detach-utility-actors",
"enables or disables detaching of utility actors") "enables or disables detaching of utility actors")
.add(middleman_detach_multiplexer, "detach-multiplexer", .add(middleman_detach_multiplexer, "detach-multiplexer",
"enables or disables background activity of the multiplexer"); "enables or disables background activity of the multiplexer")
.add(middleman_enable_tcp, "enable-tcp",
"enable communication via TCP (on by default)")
.add(middleman_enable_udp, "enable-udp",
"enable communication via UDP (off by default)")
.add(middleman_cached_udp_buffers, "cached-udp-buffers",
"sets the max number of UDP send buffers that will be cached for reuse "
"(default: 10)");
opt_group(options_, "opencl") opt_group(options_, "opencl")
.add(opencl_device_ids, "device-ids", .add(opencl_device_ids, "device-ids",
"restricts which OpenCL devices are accessed by CAF"); "restricts which OpenCL devices are accessed by CAF");
......
...@@ -65,7 +65,8 @@ const char* sec_strings[] = { ...@@ -65,7 +65,8 @@ const char* sec_strings[] = {
"no_downstream_stages_defined", "no_downstream_stages_defined",
"stream_init_failed", "stream_init_failed",
"invalid_stream_state", "invalid_stream_state",
"bad_function_call" "bad_function_call",
"feature_disabled",
}; };
} // namespace <anonymous> } // namespace <anonymous>
......
...@@ -24,6 +24,13 @@ set (LIBCAF_IO_SRCS ...@@ -24,6 +24,13 @@ set (LIBCAF_IO_SRCS
src/scribe.cpp src/scribe.cpp
src/stream_manager.cpp src/stream_manager.cpp
src/test_multiplexer.cpp src/test_multiplexer.cpp
src/acceptor_manager.cpp
src/multiplexer.cpp
src/datagram_servant.cpp
src/datagram_manager.cpp
src/ip_endpoint.cpp
src/connection_helper.cpp
src/receive_buffer.cpp
# BASP files # BASP files
src/header.cpp src/header.cpp
src/message_type.cpp src/message_type.cpp
......
...@@ -30,12 +30,15 @@ ...@@ -30,12 +30,15 @@
#include "caf/io/fwd.hpp" #include "caf/io/fwd.hpp"
#include "caf/io/accept_handle.hpp" #include "caf/io/accept_handle.hpp"
#include "caf/io/receive_policy.hpp" #include "caf/io/receive_policy.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/system_messages.hpp" #include "caf/io/system_messages.hpp"
#include "caf/io/connection_handle.hpp" #include "caf/io/connection_handle.hpp"
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/io/network/native_socket.hpp" #include "caf/io/network/native_socket.hpp"
#include "caf/io/network/stream_manager.hpp" #include "caf/io/network/stream_manager.hpp"
#include "caf/io/network/acceptor_manager.hpp" #include "caf/io/network/acceptor_manager.hpp"
#include "caf/io/network/datagram_manager.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
...@@ -62,7 +65,7 @@ class middleman; ...@@ -62,7 +65,7 @@ class middleman;
/// sending its content via the network. Instead of actively receiving data, /// sending its content via the network. Instead of actively receiving data,
/// brokers configure a scribe to asynchronously receive data, e.g., /// brokers configure a scribe to asynchronously receive data, e.g.,
/// `self->configure_read(hdl, receive_policy::exactly(1024))` would /// `self->configure_read(hdl, receive_policy::exactly(1024))` would
/// configure the scribe associated to `hdl` to receive *exactly* 1024 bytes /// configure the scribe associated with `hdl` to receive *exactly* 1024 bytes
/// and generate a `new_data_msg` message for the broker once the /// and generate a `new_data_msg` message for the broker once the
/// data is available. The buffer in this message will be re-used by the /// data is available. The buffer in this message will be re-used by the
/// scribe to minimize memory usage and heap allocations. /// scribe to minimize memory usage and heap allocations.
...@@ -82,6 +85,7 @@ public: ...@@ -82,6 +85,7 @@ public:
// even brokers need friends // even brokers need friends
friend class scribe; friend class scribe;
friend class doorman; friend class doorman;
friend class datagram_servant;
// -- overridden modifiers of abstract_actor --------------------------------- // -- overridden modifiers of abstract_actor ---------------------------------
...@@ -136,23 +140,38 @@ public: ...@@ -136,23 +140,38 @@ public:
} }
} }
/// Modifies the receive policy for given connection. /// Modifies the receive policy for a given connection.
/// @param hdl Identifies the affected connection. /// @param hdl Identifies the affected connection.
/// @param cfg Contains the new receive policy. /// @param cfg Contains the new receive policy.
void configure_read(connection_handle hdl, receive_policy::config cfg); void configure_read(connection_handle hdl, receive_policy::config cfg);
/// Enables or disables write notifications for given connection. /// Enables or disables write notifications for a given connection.
void ack_writes(connection_handle hdl, bool enable); void ack_writes(connection_handle hdl, bool enable);
/// Returns the write buffer for given connection. /// Returns the write buffer for a given connection.
std::vector<char>& wr_buf(connection_handle hdl); std::vector<char>& wr_buf(connection_handle hdl);
/// Writes `data` into the buffer for given connection. /// Writes `data` into the buffer for a given connection.
void write(connection_handle hdl, size_t bs, const void* buf); void write(connection_handle hdl, size_t bs, const void* buf);
/// Sends the content of the buffer for given connection. /// Sends the content of the buffer for a given connection.
void flush(connection_handle hdl); void flush(connection_handle hdl);
/// Enables or disables write notifications for a given datagram socket.
void ack_writes(datagram_handle hdl, bool enable);
/// Returns the write buffer for a given sink.
std::vector<char>& wr_buf(datagram_handle hdl);
/// Enqueue a buffer to be sent as a datagram via a given endpoint.
void enqueue_datagram(datagram_handle, std::vector<char>);
/// Writes `data` into the buffer of a given sink.
void write(datagram_handle hdl, size_t data_size, const void* data);
/// Sends the content of the buffer to a UDP endpoint.
void flush(datagram_handle hdl);
/// Returns the middleman instance this broker belongs to. /// Returns the middleman instance this broker belongs to.
inline middleman& parent() { inline middleman& parent() {
return system().middleman(); return system().middleman();
...@@ -191,24 +210,75 @@ public: ...@@ -191,24 +210,75 @@ public:
add_tcp_doorman(uint16_t port = 0, const char* in = nullptr, add_tcp_doorman(uint16_t port = 0, const char* in = nullptr,
bool reuse_addr = false); bool reuse_addr = false);
/// Returns the remote address associated to `hdl` /// Adds a `datagram_servant` to this broker.
void add_datagram_servant(datagram_servant_ptr ptr);
/// Adds the `datagram_servant` under an additional `hdl`.
void add_hdl_for_datagram_servant(datagram_servant_ptr ptr,
datagram_handle hdl);
/// Creates and assigns a new `datagram_servant` from a given socket `fd`.
datagram_handle add_datagram_servant(network::native_socket fd);
/// Creates and assigns a new `datagram_servant` from a given socket `fd`
/// for the remote endpoint `ep`.
datagram_handle
add_datagram_servant_for_endpoint(network::native_socket fd,
const network::ip_endpoint& ep);
/// Creates a new `datagram_servant` for the remote endpoint `host` and `port`.
/// @returns The handle to the new `datagram_servant`.
expected<datagram_handle>
add_udp_datagram_servant(const std::string& host, uint16_t port);
/// Tries to open a local port and creates a `datagram_servant` managing it on
/// success. If `port == 0`, then the broker will ask the operating system to
/// pick a random port.
/// @returns The handle of the new `datagram_servant` and the assigned port.
expected<std::pair<datagram_handle, uint16_t>>
add_udp_datagram_servant(uint16_t port = 0, const char* in = nullptr,
bool reuse_addr = false);
/// Moves an initialized `datagram_servant` instance `ptr` from another broker
/// to this one.
void move_datagram_servant(datagram_servant_ptr ptr);
/// Returns the remote address associated with `hdl`
/// or empty string if `hdl` is invalid. /// or empty string if `hdl` is invalid.
std::string remote_addr(connection_handle hdl); std::string remote_addr(connection_handle hdl);
/// Returns the remote port associated to `hdl` /// Returns the remote port associated with `hdl`
/// or `0` if `hdl` is invalid. /// or `0` if `hdl` is invalid.
uint16_t remote_port(connection_handle hdl); uint16_t remote_port(connection_handle hdl);
/// Returns the local address associated to `hdl` /// Returns the local address associated with `hdl`
/// or empty string if `hdl` is invalid. /// or empty string if `hdl` is invalid.
std::string local_addr(accept_handle hdl); std::string local_addr(accept_handle hdl);
/// Returns the local port associated to `hdl` or `0` if `hdl` is invalid. /// Returns the local port associated with `hdl` or `0` if `hdl` is invalid.
uint16_t local_port(accept_handle hdl); uint16_t local_port(accept_handle hdl);
/// Returns the handle associated to given local `port` or `none`. /// Returns the handle associated with given local `port` or `none`.
accept_handle hdl_by_port(uint16_t port); accept_handle hdl_by_port(uint16_t port);
/// Returns the dgram handle associated with given local `port` or `none`.
datagram_handle datagram_hdl_by_port(uint16_t port);
/// Returns the remote address associated with `hdl`
/// or an empty string if `hdl` is invalid.
std::string remote_addr(datagram_handle hdl);
/// Returns the remote port associated with `hdl`
/// or `0` if `hdl` is invalid.
uint16_t remote_port(datagram_handle hdl);
/// Returns the remote port associated with `hdl`
/// or `0` if `hdl` is invalid.
uint16_t local_port(datagram_handle hdl);
/// Remove the endpoint `hdl` from the broker.
bool remove_endpoint(datagram_handle hdl);
/// Closes all connections and acceptors. /// Closes all connections and acceptors.
void close_all(); void close_all();
...@@ -237,6 +307,30 @@ public: ...@@ -237,6 +307,30 @@ public:
if (i != elements.end()) if (i != elements.end())
elements.erase(i); elements.erase(i);
} }
// meta programming utility (not implemented)
static intrusive_ptr<doorman> ptr_of(accept_handle);
// meta programming utility (not implemented)
static intrusive_ptr<scribe> ptr_of(connection_handle);
// meta programming utility (not implemented)
static intrusive_ptr<datagram_servant> ptr_of(datagram_handle);
/// Returns an intrusive pointer to a `scribe` or `doorman`
/// identified by `hdl` and remove it from this broker.
template <class Handle>
auto take(Handle hdl) -> decltype(ptr_of(hdl)) {
using std::swap;
auto& elements = get_map(hdl);
decltype(ptr_of(hdl)) result;
auto i = elements.find(hdl);
if (i == elements.end())
return nullptr;
swap(result, i->second);
elements.erase(i);
return result;
}
/// @endcond /// @endcond
// -- overridden observers of abstract_actor --------------------------------- // -- overridden observers of abstract_actor ---------------------------------
...@@ -270,6 +364,8 @@ protected: ...@@ -270,6 +364,8 @@ protected:
using scribe_map = std::unordered_map<connection_handle, using scribe_map = std::unordered_map<connection_handle,
intrusive_ptr<scribe>>; intrusive_ptr<scribe>>;
using datagram_servant_map
= std::unordered_map<datagram_handle, intrusive_ptr<datagram_servant>>;
/// @cond PRIVATE /// @cond PRIVATE
// meta programming utility // meta programming utility
...@@ -282,12 +378,9 @@ protected: ...@@ -282,12 +378,9 @@ protected:
return scribes_; return scribes_;
} }
// meta programming utility (not implemented) inline datagram_servant_map& get_map(datagram_handle) {
static intrusive_ptr<doorman> ptr_of(accept_handle); return datagram_servants_;
}
// meta programming utility (not implemented)
static intrusive_ptr<scribe> ptr_of(connection_handle);
/// @endcond /// @endcond
/// Returns a `scribe` or `doorman` identified by `hdl`. /// Returns a `scribe` or `doorman` identified by `hdl`.
...@@ -300,21 +393,6 @@ protected: ...@@ -300,21 +393,6 @@ protected:
return *(i->second); return *(i->second);
} }
/// Returns an intrusive pointer to a `scribe` or `doorman`
/// identified by `hdl` and remove it from this broker.
template <class Handle>
auto take(Handle hdl) -> decltype(ptr_of(hdl)) {
using std::swap;
auto& elements = get_map(hdl);
decltype(ptr_of(hdl)) result;
auto i = elements.find(hdl);
if (i == elements.end())
return nullptr;
swap(result, i->second);
elements.erase(i);
return result;
}
private: private:
inline void launch_servant(scribe_ptr&) { inline void launch_servant(scribe_ptr&) {
// nop // nop
...@@ -322,6 +400,8 @@ private: ...@@ -322,6 +400,8 @@ private:
void launch_servant(doorman_ptr& ptr); void launch_servant(doorman_ptr& ptr);
void launch_servant(datagram_servant_ptr& ptr);
template <class T> template <class T>
typename T::handle_type add_servant(intrusive_ptr<T>&& ptr) { typename T::handle_type add_servant(intrusive_ptr<T>&& ptr) {
CAF_ASSERT(ptr != nullptr); CAF_ASSERT(ptr != nullptr);
...@@ -345,6 +425,7 @@ private: ...@@ -345,6 +425,7 @@ private:
scribe_map scribes_; scribe_map scribes_;
doorman_map doormen_; doorman_map doormen_;
datagram_servant_map datagram_servants_;
detail::intrusive_partitioned_list<mailbox_element, detail::disposer> cache_; detail::intrusive_partitioned_list<mailbox_element, detail::disposer> cache_;
std::vector<char> dummy_wr_buf_; std::vector<char> dummy_wr_buf_;
}; };
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2016 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_BASP_ENDPOINT_CONTEXT_HPP
#define CAF_IO_BASP_ENDPOINT_CONTEXT_HPP
#include <unordered_map>
#include "caf/variant.hpp"
#include "caf/response_promise.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/connection_handle.hpp"
#include "caf/io/basp/header.hpp"
#include "caf/io/basp/connection_state.hpp"
namespace caf {
namespace io {
namespace basp {
// stores meta information for active endpoints
struct endpoint_context {
using pending_map = std::unordered_map<uint16_t,
std::pair<basp::header,
std::vector<char>>>;
// denotes what message we expect from the remote node next
basp::connection_state cstate;
// our currently processed BASP header
basp::header hdr;
// the handle for I/O operations
variant<connection_handle, datagram_handle> hdl;
// network-agnostic node identifier
node_id id;
// ports
uint16_t remote_port;
uint16_t local_port;
// pending operations to be performed after handshake completed
optional<response_promise> callback;
// protocols that do not implement ordering are ordered by CAF
bool requires_ordering;
// sequence numbers and a buffer to establish order
uint16_t seq_incoming;
uint16_t seq_outgoing;
pending_map pending;
};
} // namespace basp
} // namespace io
} // namespace caf
#endif // CAF_IO_BASP_ENDPOINT_CONTEXT_HPP
...@@ -37,6 +37,9 @@ namespace basp { ...@@ -37,6 +37,9 @@ namespace basp {
/// @addtogroup BASP /// @addtogroup BASP
/// Sequence number type for BASP headers.
using sequence_type = uint16_t;
/// The header of a Binary Actor System Protocol (BASP) message. /// The header of a Binary Actor System Protocol (BASP) message.
/// A BASP header consists of a routing part, i.e., source and /// A BASP header consists of a routing part, i.e., source and
/// destination, as well as an operation and operation data. Several /// destination, as well as an operation and operation data. Several
...@@ -52,6 +55,7 @@ struct header { ...@@ -52,6 +55,7 @@ struct header {
node_id dest_node; node_id dest_node;
actor_id source_actor; actor_id source_actor;
actor_id dest_actor; actor_id dest_actor;
sequence_type sequence_number;
inline header(message_type m_operation, uint8_t m_flags, inline header(message_type m_operation, uint8_t m_flags,
uint32_t m_payload_len, uint64_t m_operation_data, uint32_t m_payload_len, uint64_t m_operation_data,
...@@ -64,7 +68,25 @@ struct header { ...@@ -64,7 +68,25 @@ struct header {
source_node(std::move(m_source_node)), source_node(std::move(m_source_node)),
dest_node(std::move(m_dest_node)), dest_node(std::move(m_dest_node)),
source_actor(m_source_actor), source_actor(m_source_actor),
dest_actor(m_dest_actor) { dest_actor(m_dest_actor),
sequence_number(0) {
// nop
}
inline header(message_type m_operation, uint8_t m_flags,
uint32_t m_payload_len, uint64_t m_operation_data,
node_id m_source_node, node_id m_dest_node,
actor_id m_source_actor, actor_id m_dest_actor,
sequence_type m_sequence_number)
: operation(m_operation),
flags(m_flags),
payload_len(m_payload_len),
operation_data(m_operation_data),
source_node(std::move(m_source_node)),
dest_node(std::move(m_dest_node)),
source_actor(m_source_actor),
dest_actor(m_dest_actor),
sequence_number(m_sequence_number) {
// nop // nop
} }
...@@ -87,8 +109,10 @@ typename Inspector::result_type inspect(Inspector& f, header& hdr) { ...@@ -87,8 +109,10 @@ typename Inspector::result_type inspect(Inspector& f, header& hdr) {
hdr.operation, hdr.operation,
meta::omittable(), pad, meta::omittable(), pad,
meta::omittable(), pad, meta::omittable(), pad,
hdr.flags, hdr.payload_len, hdr.operation_data, hdr.source_node, hdr.flags, hdr.payload_len, hdr.operation_data,
hdr.dest_node, hdr.source_actor, hdr.dest_actor); hdr.source_node, hdr.dest_node,
hdr.source_actor, hdr.dest_actor,
hdr.sequence_number);
} }
/// @relates header /// @relates header
...@@ -118,7 +142,8 @@ bool valid(const header& hdr); ...@@ -118,7 +142,8 @@ bool valid(const header& hdr);
constexpr size_t header_size = node_id::serialized_size * 2 constexpr size_t header_size = node_id::serialized_size * 2
+ sizeof(actor_id) * 2 + sizeof(actor_id) * 2
+ sizeof(uint32_t) * 2 + sizeof(uint32_t) * 2
+ sizeof(uint64_t); + sizeof(uint64_t)
+ sizeof(sequence_type);
/// @} /// @}
......
...@@ -20,7 +20,12 @@ ...@@ -20,7 +20,12 @@
#ifndef CAF_IO_BASP_INSTANCE_HPP #ifndef CAF_IO_BASP_INSTANCE_HPP
#define CAF_IO_BASP_INSTANCE_HPP #define CAF_IO_BASP_INSTANCE_HPP
#include <limits>
#include "caf/error.hpp" #include "caf/error.hpp"
#include "caf/variant.hpp"
#include "caf/actor_system_config.hpp"
#include "caf/binary_deserializer.hpp"
#include "caf/io/hook.hpp" #include "caf/io/hook.hpp"
#include "caf/io/middleman.hpp" #include "caf/io/middleman.hpp"
...@@ -30,6 +35,7 @@ ...@@ -30,6 +35,7 @@
#include "caf/io/basp/message_type.hpp" #include "caf/io/basp/message_type.hpp"
#include "caf/io/basp/routing_table.hpp" #include "caf/io/basp/routing_table.hpp"
#include "caf/io/basp/connection_state.hpp" #include "caf/io/basp/connection_state.hpp"
#include "caf/io/basp/endpoint_context.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
...@@ -42,6 +48,9 @@ class instance { ...@@ -42,6 +48,9 @@ class instance {
public: public:
/// Provides a callback-based interface for certain BASP events. /// Provides a callback-based interface for certain BASP events.
class callee { class callee {
protected:
using buffer_type = std::vector<char>;
using endpoint_handle = variant<connection_handle, datagram_handle>;
public: public:
explicit callee(actor_system& sys, proxy_registry::backend& backend); explicit callee(actor_system& sys, proxy_registry::backend& backend);
...@@ -96,6 +105,52 @@ public: ...@@ -96,6 +105,52 @@ public:
return namespace_.system(); return namespace_.system();
} }
/// Returns the next outgoing sequence number for a connection.
virtual sequence_type next_sequence_number(connection_handle hdl) = 0;
/// Returns the next outgoing sequence number for an endpoint.
virtual sequence_type next_sequence_number(datagram_handle hdl) = 0;
/// Adds a message with a future sequence number to the pending messages
/// of a given endpoint context.
virtual void add_pending(sequence_type seq, endpoint_context& ep,
header hdr, std::vector<char> payload) = 0;
/// Delivers a pending incoming messages for an endpoint with
/// application layer ordering.
virtual bool deliver_pending(execution_unit* ctx,
endpoint_context& ep) = 0;
/// Drop pending messages with sequence number `seq`.
virtual void drop_pending(sequence_type seq, endpoint_context& ep) = 0;
/// Returns a reference to the current sent buffer, dispatching the call
/// based on the type contained in `hdl`.
virtual buffer_type& get_buffer(endpoint_handle hdl) = 0;
/// Returns a reference to the current sent buffer. The callee may cache
/// buffers to reuse them for multiple datagrams. Subsequent calls will
/// return the same buffer until `pop_datagram_buffer` is called.
virtual buffer_type& get_buffer(datagram_handle hdl) = 0;
/// Returns a reference to the sent buffer.
virtual buffer_type& get_buffer(connection_handle hdl) = 0;
/// Returns the buffer accessed through a call to `get_buffer` when
/// passing a datagram handle and removes it from the callee.
virtual buffer_type pop_datagram_buffer(datagram_handle hdl) = 0;
/// Flushes the underlying write buffer of `hdl`, dispatches the call based
/// on the type contained in `hdl`.
virtual void flush(endpoint_handle hdl) = 0;
/// Flushes the underlying write buffer of `hdl`. Implicitly pops the
/// current buffer and enqueues it for sending.
virtual void flush(datagram_handle hdl) = 0;
/// Flushes the underlying write buffer of `hdl`.
virtual void flush(connection_handle hdl) = 0;
protected: protected:
proxy_registry namespace_; proxy_registry namespace_;
}; };
...@@ -114,6 +169,9 @@ public: ...@@ -114,6 +169,9 @@ public:
connection_state handle(execution_unit* ctx, connection_state handle(execution_unit* ctx,
new_data_msg& dm, header& hdr, bool is_payload); new_data_msg& dm, header& hdr, bool is_payload);
/// Handles a received datagram.
bool handle(execution_unit* ctx, new_datagram_msg& dm, endpoint_context& ep);
/// Sends heartbeat messages to all valid nodes those are directly connected. /// Sends heartbeat messages to all valid nodes those are directly connected.
void handle_heartbeat(execution_unit* ctx); void handle_heartbeat(execution_unit* ctx);
...@@ -142,6 +200,12 @@ public: ...@@ -142,6 +200,12 @@ public:
size_t remove_published_actor(const actor_addr& whom, uint16_t port, size_t remove_published_actor(const actor_addr& whom, uint16_t port,
removed_published_actor* cb = nullptr); removed_published_actor* cb = nullptr);
/// Compare two sequence numbers
static bool is_greater(sequence_type lhs, sequence_type rhs,
sequence_type max_distance
= std::numeric_limits<sequence_type>::max() / 2);
/// Returns `true` if a path to destination existed, `false` otherwise. /// Returns `true` if a path to destination existed, `false` otherwise.
bool dispatch(execution_unit* ctx, const strong_actor_ptr& sender, bool dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
const std::vector<strong_actor_ptr>& forwarding_stack, const std::vector<strong_actor_ptr>& forwarding_stack,
...@@ -172,7 +236,7 @@ public: ...@@ -172,7 +236,7 @@ public:
} }
/// Writes a header followed by its payload to `storage`. /// Writes a header followed by its payload to `storage`.
void write(execution_unit* ctx, buffer_type& buf, header& hdr, static void write(execution_unit* ctx, buffer_type& buf, header& hdr,
payload_writer* pw = nullptr); payload_writer* pw = nullptr);
/// Writes the server handshake containing the information of the /// Writes the server handshake containing the information of the
...@@ -180,24 +244,36 @@ public: ...@@ -180,24 +244,36 @@ public:
/// if no actor is published at this port then a standard handshake is /// if no actor is published at this port then a standard handshake is
/// written (e.g. used when establishing direct connections on-the-fly). /// written (e.g. used when establishing direct connections on-the-fly).
void write_server_handshake(execution_unit* ctx, void write_server_handshake(execution_unit* ctx,
buffer_type& out_buf, optional<uint16_t> port); buffer_type& out_buf, optional<uint16_t> port,
uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`.
static void write_client_handshake(execution_unit* ctx,
buffer_type& buf,
const node_id& remote_side,
const node_id& this_node,
const std::string& app_identifier,
uint16_t sequence_number = 0);
/// Writes the client handshake to `buf`. /// Writes the client handshake to `buf`.
void write_client_handshake(execution_unit* ctx, void write_client_handshake(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side); buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0);
/// Writes an `announce_proxy` to `buf`. /// Writes an `announce_proxy` to `buf`.
void write_announce_proxy(execution_unit* ctx, buffer_type& buf, void write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid); const node_id& dest_node, actor_id aid,
uint16_t sequence_number = 0);
/// Writes a `kill_proxy` to `buf`. /// Writes a `kill_proxy` to `buf`.
void write_kill_proxy(execution_unit* ctx, buffer_type& buf, void write_kill_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid, const node_id& dest_node, actor_id aid,
const error& rsn); const error& rsn, uint16_t sequence_number = 0);
/// Writes a `heartbeat` to `buf`. /// Writes a `heartbeat` to `buf`.
void write_heartbeat(execution_unit* ctx, void write_heartbeat(execution_unit* ctx,
buffer_type& buf, const node_id& remote_side); buffer_type& buf, const node_id& remote_side,
uint16_t sequence_number = 0);
inline const node_id& this_node() const { inline const node_id& this_node() const {
return this_node_; return this_node_;
...@@ -213,6 +289,177 @@ public: ...@@ -213,6 +289,177 @@ public:
return callee_.system(); return callee_.system();
} }
template <class Handle>
bool handle(execution_unit* ctx, const Handle& hdl, header& hdr,
std::vector<char>* payload, bool tcp_based,
optional<endpoint_context&> ep, optional<uint16_t> port) {
// function object for checking payload validity
auto payload_valid = [&]() -> bool {
return payload != nullptr && payload->size() == hdr.payload_len;
};
// handle message to ourselves
switch (hdr.operation) {
case message_type::server_handshake: {
actor_id aid = invalid_actor_id;
std::set<std::string> sigs;
if (!payload_valid()) {
CAF_LOG_ERROR("fail to receive the app identifier");
return false;
} else {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return false;
if (remote_appid != callee_.system().config().middleman_app_identifier) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
e = bd(aid, sigs);
if (e)
return false;
}
// close self connection after handshake is done
if (hdr.source_node == this_node_) {
CAF_LOG_INFO("close connection to self immediately");
callee_.finalize_handshake(hdr.source_node, aid, sigs);
return false;
}
// close this connection if we already have a direct connection
if (tbl_.lookup_direct(hdr.source_node)) {
CAF_LOG_INFO("close connection since we already have a "
"direct connection: " << CAF_ARG(hdr.source_node));
callee_.finalize_handshake(hdr.source_node, aid, sigs);
return false;
}
// add direct route to this node and remove any indirect entry
CAF_LOG_INFO("new direct connection:" << CAF_ARG(hdr.source_node));
tbl_.add_direct(hdl, hdr.source_node);
auto was_indirect = tbl_.erase_indirect(hdr.source_node);
// write handshake as client in response
auto path = tbl_.lookup(hdr.source_node);
if (!path) {
CAF_LOG_ERROR("no route to host after server handshake");
return false;
}
if (tcp_based) {
auto ch = get<connection_handle>(path->hdl);
write_client_handshake(ctx, callee_.get_buffer(ch),
hdr.source_node);
}
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
callee_.finalize_handshake(hdr.source_node, aid, sigs);
flush(*path);
break;
}
case message_type::client_handshake: {
if (!payload_valid()) {
CAF_LOG_ERROR("fail to receive the app identifier");
return false;
} else {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return false;
if (remote_appid != callee_.system().config().middleman_app_identifier) {
CAF_LOG_ERROR("app identifier mismatch");
return false;
}
}
if (tcp_based) {
if (tbl_.lookup_direct(hdr.source_node)) {
CAF_LOG_INFO("received second client handshake:"
<< CAF_ARG(hdr.source_node));
break;
}
// add direct route to this node and remove any indirect entry
CAF_LOG_INFO("new direct connection:" << CAF_ARG(hdr.source_node));
tbl_.add_direct(hdl, hdr.source_node);
auto was_indirect = tbl_.erase_indirect(hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
} else {
auto new_node = (this_node() != hdr.source_node
&& !tbl_.lookup_direct(hdr.source_node));
if (new_node) {
// add direct route to this node and remove any indirect entry
CAF_LOG_INFO("new direct connection:" << CAF_ARG(hdr.source_node));
tbl_.add_direct(hdl, hdr.source_node);
}
uint16_t seq = (ep && ep->requires_ordering) ? ep->seq_outgoing++ : 0;
write_server_handshake(ctx,
callee_.get_buffer(hdl),
port, seq);
callee_.flush(hdl);
if (new_node) {
auto was_indirect = tbl_.erase_indirect(hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
}
}
break;
}
case message_type::dispatch_message: {
if (!payload_valid())
return false;
// in case the sender of this message was received via a third node,
// we assume that that node to offers a route to the original source
auto last_hop = tbl_.lookup_direct(hdl);
if (hdr.source_node != none
&& hdr.source_node != this_node_
&& last_hop != hdr.source_node
&& !tbl_.lookup_direct(hdr.source_node)
&& tbl_.add_indirect(last_hop, hdr.source_node))
callee_.learned_new_node_indirectly(hdr.source_node);
binary_deserializer bd{ctx, *payload};
auto receiver_name = static_cast<atom_value>(0);
std::vector<strong_actor_ptr> forwarding_stack;
message msg;
if (hdr.has(header::named_receiver_flag)) {
auto e = bd(receiver_name);
if (e)
return false;
}
auto e = bd(forwarding_stack, msg);
if (e)
return false;
CAF_LOG_DEBUG(CAF_ARG(forwarding_stack) << CAF_ARG(msg));
if (hdr.has(header::named_receiver_flag))
callee_.deliver(hdr.source_node, hdr.source_actor, receiver_name,
message_id::make(hdr.operation_data),
forwarding_stack, msg);
else
callee_.deliver(hdr.source_node, hdr.source_actor, hdr.dest_actor,
message_id::make(hdr.operation_data),
forwarding_stack, msg);
break;
}
case message_type::announce_proxy:
callee_.proxy_announced(hdr.source_node, hdr.dest_actor);
break;
case message_type::kill_proxy: {
if (!payload_valid())
return false;
binary_deserializer bd{ctx, *payload};
error fail_state;
auto e = bd(fail_state);
if (e)
return false;
callee_.proxies().erase(hdr.source_node, hdr.source_actor,
std::move(fail_state));
break;
}
case message_type::heartbeat: {
CAF_LOG_TRACE("received heartbeat: " << CAF_ARG(hdr.source_node));
callee_.handle_heartbeat(hdr.source_node);
break;
}
default:
CAF_LOG_ERROR("invalid operation");
return false;
}
return true;
}
private: private:
routing_table tbl_; routing_table tbl_;
published_actor_map published_actors_; published_actor_map published_actors_;
...@@ -227,4 +474,3 @@ private: ...@@ -227,4 +474,3 @@ private:
} // namespace caf } // namespace caf
#endif // CAF_IO_BASP_INSTANCE_HPP #endif // CAF_IO_BASP_INSTANCE_HPP
...@@ -26,10 +26,23 @@ ...@@ -26,10 +26,23 @@
#include "caf/node_id.hpp" #include "caf/node_id.hpp"
#include "caf/callback.hpp" #include "caf/callback.hpp"
#include "caf/io/visitors.hpp"
#include "caf/io/abstract_broker.hpp" #include "caf/io/abstract_broker.hpp"
#include "caf/io/basp/buffer_type.hpp" #include "caf/io/basp/buffer_type.hpp"
namespace std {
template<>
struct hash<caf::variant<caf::io::connection_handle,caf::io::datagram_handle>> {
size_t operator()(const caf::variant<caf::io::connection_handle,
caf::io::datagram_handle>& hdl) const {
return caf::visit(caf::io::hash_visitor{}, hdl);
}
};
} // namespace std
namespace caf { namespace caf {
namespace io { namespace io {
namespace basp { namespace basp {
...@@ -40,15 +53,16 @@ namespace basp { ...@@ -40,15 +53,16 @@ namespace basp {
/// BASP peer and provides both direct and indirect paths. /// BASP peer and provides both direct and indirect paths.
class routing_table { class routing_table {
public: public:
using endpoint_handle = variant<connection_handle, datagram_handle>;
explicit routing_table(abstract_broker* parent); explicit routing_table(abstract_broker* parent);
virtual ~routing_table(); virtual ~routing_table();
/// Describes a routing path to a node. /// Describes a routing path to a node.
struct route { struct route {
buffer_type& wr_buf;
const node_id& next_hop; const node_id& next_hop;
connection_handle hdl; endpoint_handle hdl;
}; };
/// Describes a function object for erase operations that /// Describes a function object for erase operations that
...@@ -60,22 +74,19 @@ public: ...@@ -60,22 +74,19 @@ public:
/// Returns the ID of the peer connected via `hdl` or /// Returns the ID of the peer connected via `hdl` or
/// `none` if `hdl` is unknown. /// `none` if `hdl` is unknown.
node_id lookup_direct(const connection_handle& hdl) const; node_id lookup_direct(const endpoint_handle& hdl) const;
/// Returns the handle offering a direct connection to `nid` or /// Returns the handle offering a direct connection to `nid` or
/// `invalid_connection_handle` if no direct connection to `nid` exists. /// `invalid_connection_handle` if no direct connection to `nid` exists.
connection_handle lookup_direct(const node_id& nid) const; optional<endpoint_handle> lookup_direct(const node_id& nid) const;
/// Returns the next hop that would be chosen for `nid` /// Returns the next hop that would be chosen for `nid`
/// or `none` if there's no indirect route to `nid`. /// or `none` if there's no indirect route to `nid`.
node_id lookup_indirect(const node_id& nid) const; node_id lookup_indirect(const node_id& nid) const;
/// Flush output buffer for `r`.
void flush(const route& r);
/// Adds a new direct route to the table. /// Adds a new direct route to the table.
/// @pre `hdl != invalid_connection_handle && nid != none` /// @pre `hdl != invalid_connection_handle && nid != none`
void add_direct(const connection_handle& hdl, const node_id& nid); void add_direct(const endpoint_handle& hdl, const node_id& nid);
/// Adds a new indirect route to the table. /// Adds a new indirect route to the table.
bool add_indirect(const node_id& hop, const node_id& dest); bool add_indirect(const node_id& hop, const node_id& dest);
...@@ -86,7 +97,7 @@ public: ...@@ -86,7 +97,7 @@ public:
/// Removes a direct connection and calls `cb` for any node /// Removes a direct connection and calls `cb` for any node
/// that became unreachable as a result of this operation, /// that became unreachable as a result of this operation,
/// including the node that is assigned as direct path for `hdl`. /// including the node that is assigned as direct path for `hdl`.
void erase_direct(const connection_handle& hdl, erase_callback& cb); void erase_direct(const endpoint_handle& hdl, erase_callback& cb);
/// Removes any entry for indirect connection to `dest` and returns /// Removes any entry for indirect connection to `dest` and returns
/// `true` if `dest` had an indirect route, otherwise `false`. /// `true` if `dest` had an indirect route, otherwise `false`.
...@@ -101,6 +112,11 @@ public: ...@@ -101,6 +112,11 @@ public:
/// @returns the number of removed routes (direct and indirect) /// @returns the number of removed routes (direct and indirect)
size_t erase(const node_id& dest, erase_callback& cb); size_t erase(const node_id& dest, erase_callback& cb);
/// Returns the parent broker.
inline abstract_broker* parent() {
return parent_;
}
public: public:
template <class Map, class Fallback> template <class Map, class Fallback>
typename Map::mapped_type typename Map::mapped_type
...@@ -117,8 +133,8 @@ public: ...@@ -117,8 +133,8 @@ public:
node_id_set>; // hop node_id_set>; // hop
abstract_broker* parent_; abstract_broker* parent_;
std::unordered_map<connection_handle, node_id> direct_by_hdl_; std::unordered_map<endpoint_handle, node_id> direct_by_hdl_;
std::unordered_map<node_id, connection_handle> direct_by_nid_; std::unordered_map<node_id, endpoint_handle> direct_by_nid_;
indirect_entries indirect_; indirect_entries indirect_;
indirect_entries blacklist_; indirect_entries blacklist_;
}; };
...@@ -130,4 +146,3 @@ public: ...@@ -130,4 +146,3 @@ public:
} // namespace caf } // namespace caf
#endif // CAF_IO_BASP_ROUTING_TABLE_HPP #endif // CAF_IO_BASP_ROUTING_TABLE_HPP
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <map> #include <map>
#include <set> #include <set>
#include <stack>
#include <string> #include <string>
#include <future> #include <future>
#include <vector> #include <vector>
...@@ -36,8 +37,11 @@ ...@@ -36,8 +37,11 @@
#include "caf/io/basp/all.hpp" #include "caf/io/basp/all.hpp"
#include "caf/io/broker.hpp" #include "caf/io/broker.hpp"
#include "caf/io/visitors.hpp"
#include "caf/io/typed_broker.hpp" #include "caf/io/typed_broker.hpp"
#include "caf/io/basp/endpoint_context.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
...@@ -52,22 +56,22 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -52,22 +56,22 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// inherited from proxy_registry::backend // inherited from proxy_registry::backend
execution_unit* registry_context() override; execution_unit* registry_context() override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void finalize_handshake(const node_id& nid, actor_id aid, void finalize_handshake(const node_id& nid, actor_id aid,
std::set<std::string>& sigs) override; std::set<std::string>& sigs) override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void purge_state(const node_id& nid) override; void purge_state(const node_id& nid) override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void proxy_announced(const node_id& nid, actor_id aid) override; void proxy_announced(const node_id& nid, actor_id aid) override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void deliver(const node_id& src_nid, actor_id src_aid, void deliver(const node_id& src_nid, actor_id src_aid,
actor_id dest_aid, message_id mid, actor_id dest_aid, message_id mid,
std::vector<strong_actor_ptr>& stages, message& msg) override; std::vector<strong_actor_ptr>& stages, message& msg) override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void deliver(const node_id& src_nid, actor_id src_aid, void deliver(const node_id& src_nid, actor_id src_aid,
atom_value dest_name, message_id mid, atom_value dest_name, message_id mid,
std::vector<strong_actor_ptr>& stages, message& msg) override; std::vector<strong_actor_ptr>& stages, message& msg) override;
...@@ -80,38 +84,62 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -80,38 +84,62 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// performs bookkeeping such as managing `spawn_servers` // performs bookkeeping such as managing `spawn_servers`
void learned_new_node(const node_id& nid); void learned_new_node(const node_id& nid);
// inherited from basp::instance::listener // inherited from basp::instance::callee
void learned_new_node_directly(const node_id& nid, void learned_new_node_directly(const node_id& nid,
bool was_indirectly_before) override; bool was_indirectly_before) override;
// inherited from basp::instance::listener // inherited from basp::instance::callee
void learned_new_node_indirectly(const node_id& nid) override; void learned_new_node_indirectly(const node_id& nid) override;
// inherited from basp::instance::callee
uint16_t next_sequence_number(connection_handle hdl) override;
// inherited from basp::instance::callee
uint16_t next_sequence_number(datagram_handle hdl) override;
// inherited from basp::instance::callee
void add_pending(uint16_t seq, basp::endpoint_context& ep, basp::header hdr,
std::vector<char> payload) override;
// inherited from basp::instance::callee
bool deliver_pending(execution_unit* ctx,
basp::endpoint_context& ep) override;
// inherited from basp::instance::callee
void drop_pending(uint16_t seq, basp::endpoint_context& ep) override;
// inherited from basp::instance::callee
buffer_type& get_buffer(endpoint_handle hdl) override;
// inherited from basp::instance::callee
buffer_type& get_buffer(datagram_handle hdl) override;
// inherited from basp::instance::callee
buffer_type& get_buffer(connection_handle hdl) override;
// inherited from basp::instance::callee
buffer_type pop_datagram_buffer(datagram_handle hdl) override;
// inherited from basp::instance::callee
void flush(endpoint_handle hdl) override;
// inherited from basp::instance::callee
void flush(datagram_handle hdl) override;
// inherited from basp::instance::callee
void flush(connection_handle hdl) override;
void handle_heartbeat(const node_id&) override { void handle_heartbeat(const node_id&) override {
// nop // nop
} }
// stores meta information for open connections
struct connection_context {
// denotes what message we expect from the remote node next
basp::connection_state cstate;
// our currently processed BASP header
basp::header hdr;
// the connection handle for I/O operations
connection_handle hdl;
// network-agnostic node identifier
node_id id;
// connected port
uint16_t remote_port;
// pending operations to be performed after handhsake completed
optional<response_promise> callback;
};
/// Sets `this_context` by either creating or accessing state for `hdl`. /// Sets `this_context` by either creating or accessing state for `hdl`.
void set_context(connection_handle hdl); void set_context(connection_handle hdl);
void set_context(datagram_handle hdl);
/// Cleans up any state for `hdl`. /// Cleans up any state for `hdl`.
void cleanup(connection_handle hdl); void cleanup(connection_handle hdl);
void cleanup(datagram_handle hdl);
// pointer to ourselves // pointer to ourselves
broker* self; broker* self;
...@@ -119,13 +147,17 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -119,13 +147,17 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// protocol instance of BASP // protocol instance of BASP
basp::instance instance; basp::instance instance;
using ctx_map = std::unordered_map<connection_handle, connection_context>; using ctx_tcp_map = std::unordered_map<connection_handle,
basp::endpoint_context>;
using ctx_udp_map = std::unordered_map<datagram_handle,
basp::endpoint_context>;
// keeps context information for all open connections // keeps context information for all open connections
ctx_map ctx; ctx_tcp_map ctx_tcp;
ctx_udp_map ctx_udp;
// points to the current context for callbacks such as `make_proxy` // points to the current context for callbacks such as `make_proxy`
connection_context* this_context = nullptr; basp::endpoint_context* this_context = nullptr;
// stores handles to spawn servers for other nodes; these servers // stores handles to spawn servers for other nodes; these servers
// are spawned whenever the broker learns a new node ID and try to // are spawned whenever the broker learns a new node ID and try to
...@@ -136,6 +168,12 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee { ...@@ -136,6 +168,12 @@ struct basp_broker_state : proxy_registry::backend, basp::instance::callee {
// to establish new connections at runtime to optimize // to establish new connections at runtime to optimize
// routing paths by forming a mesh between all nodes // routing paths by forming a mesh between all nodes
bool enable_automatic_connections = false; bool enable_automatic_connections = false;
bool enable_tcp = true;
bool enable_udp = false;
// reusable send buffers for UDP communication
const size_t max_buffers;
std::stack<buffer_type> cached_buffers;
// returns the node identifier of the underlying BASP instance // returns the node identifier of the underlying BASP instance
const node_id& this_node() const { const node_id& this_node() const {
......
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include "caf/io/scribe.hpp" #include "caf/io/scribe.hpp"
#include "caf/io/doorman.hpp" #include "caf/io/doorman.hpp"
#include "caf/io/abstract_broker.hpp" #include "caf/io/abstract_broker.hpp"
#include "caf/io/datagram_servant.hpp"
#include "caf/mixin/sender.hpp" #include "caf/mixin/sender.hpp"
#include "caf/mixin/requester.hpp" #include "caf/mixin/requester.hpp"
......
...@@ -102,7 +102,11 @@ protected: ...@@ -102,7 +102,11 @@ protected:
typename std::conditional< typename std::conditional<
std::is_same<handle_type, connection_handle>::value, std::is_same<handle_type, connection_handle>::value,
connection_passivated_msg, connection_passivated_msg,
acceptor_passivated_msg typename std::conditional<
std::is_same<handle_type, accept_handle>::value,
acceptor_passivated_msg,
datagram_servant_passivated_msg
>::type
>::type; >::type;
using tmp_t = mailbox_element_vals<passiv_t>; using tmp_t = mailbox_element_vals<passiv_t>;
tmp_t tmp{strong_actor_ptr{}, message_id::make(), tmp_t tmp{strong_actor_ptr{}, message_id::make(),
......
...@@ -71,7 +71,7 @@ public: ...@@ -71,7 +71,7 @@ public:
} // namespace io } // namespace io
} // namespace caf } // namespace caf
namespace std{ namespace std {
template<> template<>
struct hash<caf::io::connection_handle> { struct hash<caf::io::connection_handle> {
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_CONNECTION_HELPER_HPP
#define CAF_IO_CONNECTION_HELPER_HPP
#include <chrono>
#include "caf/stateful_actor.hpp"
#include "caf/io/network/interfaces.hpp"
#include "caf/after.hpp"
#include "caf/event_based_actor.hpp"
#include "caf/actor_system_config.hpp"
#include "caf/io/broker.hpp"
#include "caf/io/middleman.hpp"
#include "caf/io/basp_broker.hpp"
#include "caf/io/system_messages.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/basp/all.hpp"
#include "caf/io/network/datagram_manager.hpp"
#include "caf/io/network/default_multiplexer.hpp"
namespace caf {
namespace io {
struct connection_helper_state {
static const char* name;
};
behavior datagram_connection_broker(broker* self,
uint16_t port,
network::address_listing addresses,
actor system_broker);
behavior connection_helper(stateful_actor<connection_helper_state>* self,
actor b);
} // namespace io
} // namespace caf
#endif // CAF_IO_CONNECTION_HELPER_HPP
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_DATAGRAM_HANDLE_HPP
#define CAF_IO_DATAGRAM_HANDLE_HPP
#include <functional>
#include "caf/error.hpp"
#include "caf/io/handle.hpp"
#include "caf/meta/type_name.hpp"
namespace caf {
namespace io {
struct invalid_datagram_handle_t {
constexpr invalid_datagram_handle_t() {
// nop
}
};
constexpr invalid_datagram_handle_t invalid_datagram_handle
= invalid_datagram_handle_t{};
/// Generic handle type for identifying datagram endpoints
class datagram_handle : public handle<datagram_handle,
invalid_datagram_handle_t> {
public:
friend class handle<datagram_handle, invalid_datagram_handle_t>;
using super = handle<datagram_handle, invalid_datagram_handle_t>;
constexpr datagram_handle() {
// nop
}
constexpr datagram_handle(const invalid_datagram_handle_t&) {
// nop
}
template <class Inspector>
friend typename Inspector::result_type inspect(Inspector& f,
datagram_handle& x) {
return f(meta::type_name("datagram_handle"), x.id_);
}
private:
inline datagram_handle(int64_t handle_id) : super{handle_id} {
// nop
}
};
} // namespace io
} // namespace caf
namespace std {
template<>
struct hash<caf::io::datagram_handle> {
size_t operator()(const caf::io::datagram_handle& hdl) const {
return std::hash<int64_t>{}(hdl.id());
}
};
} // namespace std
#endif // CAF_IO_DATAGRAM_HANDLE_HPP
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_DATAGRAM_SERVANT_HPP
#define CAF_IO_DATAGRAM_SERVANT_HPP
#include <vector>
#include "caf/message.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/broker_servant.hpp"
#include "caf/io/system_messages.hpp"
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/io/network/datagram_manager.hpp"
#include "caf/io/network/receive_buffer.hpp"
namespace caf {
namespace io {
using datagram_servant_base = broker_servant<network::datagram_manager,
datagram_handle, new_datagram_msg>;
/// Manages writing to a datagram sink.
/// @ingroup Broker
class datagram_servant : public datagram_servant_base {
public:
datagram_servant(datagram_handle hdl);
~datagram_servant() override;
/// Enables or disables write notifications.
virtual void ack_writes(bool enable) = 0;
/// Returns a new output buffer.
virtual std::vector<char>& wr_buf(datagram_handle) = 0;
/// Enqueue a buffer to be sent as a datagram.
virtual void enqueue_datagram(datagram_handle, std::vector<char>) = 0;
/// Returns the current input buffer.
virtual network::receive_buffer& rd_buf() = 0;
/// Flushes the output buffer, i.e., sends the
/// content of the buffer via the network.
virtual void flush() = 0;
/// Returns the local port of associated socket.
virtual uint16_t local_port() const = 0;
/// Returns all the handles associated with this servant
virtual std::vector<datagram_handle> hdls() const = 0;
/// Adds a new remote endpoint identified by the `ip_endpoint` to
/// the related manager.
virtual void add_endpoint(const network::ip_endpoint& ep,
datagram_handle hdl) = 0;
virtual void remove_endpoint(datagram_handle hdl) = 0;
void io_failure(execution_unit* ctx, network::operation op) override;
bool consume(execution_unit*, datagram_handle hdl,
network::receive_buffer& buf) override;
void datagram_sent(execution_unit*, datagram_handle hdl, size_t,
std::vector<char> buffer) override;
virtual void detach_handles() = 0;
using datagram_servant_base::new_endpoint;
virtual void launch() = 0;
protected:
message detach_message() override;
};
using datagram_servant_ptr = intrusive_ptr<datagram_servant>;
} // namespace io
} // namespace caf
// Allows the `middleman_actor` to create an `datagram_servant` and then send it
// to the BASP broker.
CAF_ALLOW_UNSAFE_MESSAGE_TYPE(caf::io::datagram_servant_ptr)
#endif // CAF_IO_DATAGRAM_SERVANT_HPP
...@@ -42,11 +42,13 @@ class middleman; ...@@ -42,11 +42,13 @@ class middleman;
class basp_broker; class basp_broker;
class receive_policy; class receive_policy;
class abstract_broker; class abstract_broker;
class datagram_servant;
// -- aliases ------------------------------------------------------------------ // -- aliases ------------------------------------------------------------------
using scribe_ptr = intrusive_ptr<scribe>; using scribe_ptr = intrusive_ptr<scribe>;
using doorman_ptr = intrusive_ptr<doorman>; using doorman_ptr = intrusive_ptr<doorman>;
using datagram_servant_ptr = intrusive_ptr<datagram_servant>;
// -- nested namespaces -------------------------------------------------------- // -- nested namespaces --------------------------------------------------------
......
...@@ -77,6 +77,22 @@ public: ...@@ -77,6 +77,22 @@ public:
system().message_types(tk), port, in, reuse); system().message_types(tk), port, in, reuse);
} }
/// Tries to publish `whom` at `port` and returns either an
/// `error` or the bound port.
/// @param whom Actor that should be published at `port`.
/// @param port Unused TCP port.
/// @param in The IP address to listen to or `INADDR_ANY` if `in == nullptr`.
/// @param reuse Create socket using `SO_REUSEADDR`.
/// @returns The actual port the OS uses after `bind()`. If `port == 0`
/// the OS chooses a random high-level port.
template <class Handle>
expected<uint16_t> publish_udp(Handle&& whom, uint16_t port,
const char* in = nullptr, bool reuse = false) {
detail::type_list<typename std::decay<Handle>::type> tk;
return publish_udp(actor_cast<strong_actor_ptr>(std::forward<Handle>(whom)),
system().message_types(tk), port, in, reuse);
}
/// Makes *all* local groups accessible via network /// Makes *all* local groups accessible via network
/// on address `addr` and `port`. /// on address `addr` and `port`.
/// @returns The actual port the OS uses after `bind()`. If `port == 0` /// @returns The actual port the OS uses after `bind()`. If `port == 0`
...@@ -93,6 +109,14 @@ public: ...@@ -93,6 +109,14 @@ public:
return unpublish(whom.address(), port); return unpublish(whom.address(), port);
} }
/// Unpublishes `whom` by closing `port` or all assigned ports if `port == 0`.
/// @param whom Actor that should be unpublished at `port`.
/// @param port UDP port.
template <class Handle>
expected<void> unpublish_udp(const Handle& whom, uint16_t port = 0) {
return unpublish_udp(whom.address(), port);
}
/// Establish a new connection to the actor at `host` on given `port`. /// Establish a new connection to the actor at `host` on given `port`.
/// @param host Valid hostname or IP address. /// @param host Valid hostname or IP address.
/// @param port TCP port. /// @param port TCP port.
...@@ -108,6 +132,21 @@ public: ...@@ -108,6 +132,21 @@ public:
return actor_cast<ActorHandle>(std::move(*x)); return actor_cast<ActorHandle>(std::move(*x));
} }
/// Contacts the actor at `host` on given `port`.
/// @param host Valid hostname or IP address.
/// @param port TCP port.
/// @returns An `actor` to the proxy instance representing
/// a remote actor or an `error`.
template <class ActorHandle = actor>
expected<ActorHandle> remote_actor_udp(std::string host, uint16_t port) {
detail::type_list<ActorHandle> tk;
auto x = remote_actor_udp(system().message_types(tk), std::move(host), port);
if (!x)
return x.error();
CAF_ASSERT(x && *x);
return actor_cast<ActorHandle>(std::move(*x));
}
/// <group-name>@<host>:<port> /// <group-name>@<host>:<port>
expected<group> remote_group(const std::string& group_uri); expected<group> remote_group(const std::string& group_uri);
...@@ -323,12 +362,20 @@ private: ...@@ -323,12 +362,20 @@ private:
std::set<std::string> sigs, std::set<std::string> sigs,
uint16_t port, const char* cstr, bool ru); uint16_t port, const char* cstr, bool ru);
expected<uint16_t> publish_udp(const strong_actor_ptr& whom,
std::set<std::string> sigs,
uint16_t port, const char* cstr, bool ru);
expected<void> unpublish(const actor_addr& whom, uint16_t port); expected<void> unpublish(const actor_addr& whom, uint16_t port);
expected<void> unpublish_udp(const actor_addr& whom, uint16_t port);
expected<strong_actor_ptr> remote_actor(std::set<std::string> ifs, expected<strong_actor_ptr> remote_actor(std::set<std::string> ifs,
std::string host, uint16_t port); std::string host, uint16_t port);
expected<strong_actor_ptr> remote_actor_udp(std::set<std::string> ifs,
std::string host, uint16_t port);
static int exec_slave_mode(actor_system&, const actor_system_config&); static int exec_slave_mode(actor_system&, const actor_system_config&);
// environment // environment
......
...@@ -70,6 +70,12 @@ namespace io { ...@@ -70,6 +70,12 @@ namespace io {
/// (unpublish_atom, strong_actor_ptr whom, uint16_t port) /// (unpublish_atom, strong_actor_ptr whom, uint16_t port)
/// -> void /// -> void
/// ///
/// // Closes `port` if it is mapped to `whom`.
/// // whom: A published actor.
/// // port: Used UDP port.
/// (unpublish_udp_atom, strong_actor_ptr whom, uint16_t port)
/// -> void
///
/// // Unconditionally closes `port`, removing any actor /// // Unconditionally closes `port`, removing any actor
/// // published at this port. /// // published at this port.
/// // port: Used TCP port. /// // port: Used TCP port.
...@@ -93,14 +99,24 @@ using middleman_actor = ...@@ -93,14 +99,24 @@ using middleman_actor =
std::set<std::string>, std::string, bool> std::set<std::string>, std::string, bool>
::with<uint16_t>, ::with<uint16_t>,
replies_to<publish_udp_atom, uint16_t, strong_actor_ptr,
std::set<std::string>, std::string, bool>
::with<uint16_t>,
replies_to<open_atom, uint16_t, std::string, bool> replies_to<open_atom, uint16_t, std::string, bool>
::with<uint16_t>, ::with<uint16_t>,
replies_to<connect_atom, std::string, uint16_t> replies_to<connect_atom, std::string, uint16_t>
::with<node_id, strong_actor_ptr, std::set<std::string>>, ::with<node_id, strong_actor_ptr, std::set<std::string>>,
replies_to<contact_atom, std::string, uint16_t>
::with<node_id, strong_actor_ptr, std::set<std::string>>,
reacts_to<unpublish_atom, actor_addr, uint16_t>, reacts_to<unpublish_atom, actor_addr, uint16_t>,
reacts_to<unpublish_udp_atom, actor_addr, uint16_t>,
reacts_to<close_atom, uint16_t>, reacts_to<close_atom, uint16_t>,
replies_to<spawn_atom, node_id, std::string, message, std::set<std::string>> replies_to<spawn_atom, node_id, std::string, message, std::set<std::string>>
......
...@@ -38,9 +38,11 @@ public: ...@@ -38,9 +38,11 @@ public:
using mpi_set = std::set<std::string>; using mpi_set = std::set<std::string>;
using get_res = delegated<node_id, strong_actor_ptr, mpi_set>; using get_res = result<node_id, strong_actor_ptr, mpi_set>;
using del_res = delegated<void>; using get_delegated = delegated<node_id, strong_actor_ptr, mpi_set>;
using del_res = result<void>;
using endpoint_data = std::tuple<node_id, strong_actor_ptr, mpi_set>; using endpoint_data = std::tuple<node_id, strong_actor_ptr, mpi_set>;
...@@ -59,21 +61,36 @@ protected: ...@@ -59,21 +61,36 @@ protected:
/// calls `system().middleman().backend().new_tcp_scribe(host, port)`. /// calls `system().middleman().backend().new_tcp_scribe(host, port)`.
virtual expected<scribe_ptr> connect(const std::string& host, uint16_t port); virtual expected<scribe_ptr> connect(const std::string& host, uint16_t port);
/// Tries to connect to given `host` and `port`. The default implementation
/// calls `system().middleman().backend().new_udp`.
virtual expected<datagram_servant_ptr> contact(const std::string& host,
uint16_t port);
/// Tries to open a local port. The default implementation calls /// Tries to open a local port. The default implementation calls
/// `system().middleman().backend().new_tcp_doorman(port, addr, reuse)`. /// `system().middleman().backend().new_tcp_doorman(port, addr, reuse)`.
virtual expected<doorman_ptr> open(uint16_t port, const char* addr, virtual expected<doorman_ptr> open(uint16_t port, const char* addr,
bool reuse); bool reuse);
/// Tries to open a local port. The default implementation calls
/// `system().middleman().backend().new_tcp_doorman(port, addr, reuse)`.
virtual expected<datagram_servant_ptr> open_udp(uint16_t port,
const char* addr, bool reuse);
private: private:
result<uint16_t> put(uint16_t port, strong_actor_ptr& whom, mpi_set& sigs, put_res put(uint16_t port, strong_actor_ptr& whom, mpi_set& sigs,
const char* in = nullptr, bool reuse_addr = false);
put_res put_udp(uint16_t port, strong_actor_ptr& whom, mpi_set& sigs,
const char* in = nullptr, bool reuse_addr = false); const char* in = nullptr, bool reuse_addr = false);
optional<endpoint_data&> cached(const endpoint& ep); optional<endpoint_data&> cached_tcp(const endpoint& ep);
optional<endpoint_data&> cached_udp(const endpoint& ep);
optional<std::vector<response_promise>&> pending(const endpoint& ep); optional<std::vector<response_promise>&> pending(const endpoint& ep);
actor broker_; actor broker_;
std::map<endpoint, endpoint_data> cached_; std::map<endpoint, endpoint_data> cached_tcp_;
std::map<endpoint, endpoint_data> cached_udp_;
std::map<endpoint, std::vector<response_promise>> pending_; std::map<endpoint, std::vector<response_promise>> pending_;
}; };
......
...@@ -37,6 +37,9 @@ public: ...@@ -37,6 +37,9 @@ public:
/// @returns `true` if the manager accepts further connections, /// @returns `true` if the manager accepts further connections,
/// otherwise `false`. /// otherwise `false`.
virtual bool new_connection() = 0; virtual bool new_connection() = 0;
/// Get the port of the underlying I/O device.
virtual uint16_t port() const = 0;
}; };
} // namespace network } // namespace network
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_NETWORK_DATAGRAM_MANAGER_HPP
#define CAF_IO_NETWORK_DATAGRAM_MANAGER_HPP
#include "caf/io/datagram_handle.hpp"
#include "caf/io/network/manager.hpp"
#include "caf/io/network/receive_buffer.hpp"
namespace caf {
namespace io {
namespace network {
/// A datagram manager provides callbacks for outgoing
/// datagrams as well as for error handling.
class datagram_manager : public manager {
public:
~datagram_manager() override;
/// Called by the underlying I/O device whenever it received data.
/// @returns `true` if the manager accepts further reads, otherwise `false`.
virtual bool consume(execution_unit*, datagram_handle hdl,
receive_buffer& buf) = 0;
/// Called by the underlying I/O device whenever it sent data.
virtual void datagram_sent(execution_unit*, datagram_handle hdl, size_t,
std::vector<char> buffer) = 0;
/// Called by the underlying I/O device to indicate that a new remote
/// endpoint has been detected, passing in the received datagram.
/// @returns `true` if the manager accepts further enpoints,
/// otherwise `false`.
virtual bool new_endpoint(receive_buffer& buf) = 0;
/// Get the port of the underlying I/O device.
virtual uint16_t port(datagram_handle) const = 0;
};
} // namespace network
} // namespace io
} // namespace caf
#endif // CAF_IO_NETWORK_DATAGRAM_MANAGER_HPP
...@@ -35,11 +35,16 @@ ...@@ -35,11 +35,16 @@
#include "caf/io/doorman.hpp" #include "caf/io/doorman.hpp"
#include "caf/io/accept_handle.hpp" #include "caf/io/accept_handle.hpp"
#include "caf/io/receive_policy.hpp" #include "caf/io/receive_policy.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/datagram_servant.hpp"
#include "caf/io/connection_handle.hpp" #include "caf/io/connection_handle.hpp"
#include "caf/io/network/operation.hpp" #include "caf/io/network/operation.hpp"
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/io/network/multiplexer.hpp" #include "caf/io/network/multiplexer.hpp"
#include "caf/io/network/receive_buffer.hpp"
#include "caf/io/network/stream_manager.hpp" #include "caf/io/network/stream_manager.hpp"
#include "caf/io/network/acceptor_manager.hpp" #include "caf/io/network/acceptor_manager.hpp"
#include "caf/io/network/datagram_manager.hpp"
#include "caf/io/network/native_socket.hpp" #include "caf/io/network/native_socket.hpp"
...@@ -67,6 +72,8 @@ ...@@ -67,6 +72,8 @@
# include <unistd.h> # include <unistd.h>
# include <cerrno> # include <cerrno>
# include <sys/socket.h> # include <sys/socket.h>
# include <netinet/in.h>
# include <netinet/ip.h>
#endif #endif
// poll xs epoll backend // poll xs epoll backend
...@@ -93,6 +100,7 @@ namespace network { ...@@ -93,6 +100,7 @@ namespace network {
// annoying platform-dependent bootstrapping // annoying platform-dependent bootstrapping
#ifdef CAF_WINDOWS #ifdef CAF_WINDOWS
using setsockopt_ptr = const char*; using setsockopt_ptr = const char*;
using getsockopt_ptr = char*;
using socket_send_ptr = const char*; using socket_send_ptr = const char*;
using socket_recv_ptr = char*; using socket_recv_ptr = char*;
using socklen_t = int; using socklen_t = int;
...@@ -105,6 +113,7 @@ namespace network { ...@@ -105,6 +113,7 @@ namespace network {
constexpr int ec_interrupted_syscall = WSAEINTR; constexpr int ec_interrupted_syscall = WSAEINTR;
#else #else
using setsockopt_ptr = const void*; using setsockopt_ptr = const void*;
using getsockopt_ptr = void*;
using socket_send_ptr = const void*; using socket_send_ptr = const void*;
using socket_recv_ptr = void*; using socket_recv_ptr = void*;
inline void closesocket(int fd) { close(fd); } inline void closesocket(int fd) { close(fd); }
...@@ -176,6 +185,12 @@ expected<void> tcp_nodelay(native_socket fd, bool new_value); ...@@ -176,6 +185,12 @@ expected<void> tcp_nodelay(native_socket fd, bool new_value);
/// Enables or disables `SIGPIPE` events from `fd`. /// Enables or disables `SIGPIPE` events from `fd`.
expected<void> allow_sigpipe(native_socket fd, bool new_value); expected<void> allow_sigpipe(native_socket fd, bool new_value);
/// Get the socket buffer size for `fd`.
expected<int> send_buffer_size(native_socket fd);
/// Set the socket buffer size for `fd`.
expected<void> send_buffer_size(native_socket fd, int new_value);
/// Denotes the returned state of read and write operations on sockets. /// Denotes the returned state of read and write operations on sockets.
enum class rw_state { enum class rw_state {
/// Reports that bytes could be read or written. /// Reports that bytes could be read or written.
...@@ -223,6 +238,32 @@ struct tcp_policy { ...@@ -223,6 +238,32 @@ struct tcp_policy {
static try_accept_fun try_accept; static try_accept_fun try_accept;
}; };
/// Write a datagram containing `buf_len` bytes to `fd` addressed
/// at the endpoint in `sa` with size `sa_len`. Returns true as long
/// as no IO error occurs. The number of written bytes is stored in
/// `result` and the sender is stored in `ep`.
bool read_datagram(size_t& result, native_socket fd, void* buf, size_t buf_len,
ip_endpoint& ep);
/// Reveice a datagram of up to `len` bytes. Larger datagrams are truncated.
/// Up to `sender_len` bytes of the receiver address is written into
/// `sender_addr`. Returns `true` if no IO error occurred. The number of
/// received bytes is stored in `result` (can be 0).
bool write_datagram(size_t& result, native_socket fd, void* buf, size_t buf_len,
const ip_endpoint& ep);
/// Function signature of read_datagram
using read_datagram_fun = decltype(read_datagram)*;
/// Function signature of write_datagram
using write_datagram_fun = decltype(write_datagram)*;
/// Policy object for wrapping default UDP operations
struct udp_policy {
static read_datagram_fun read_datagram;
static write_datagram_fun write_datagram;
};
/// Returns the locally assigned port of `fd`. /// Returns the locally assigned port of `fd`.
expected<uint16_t> local_port_of_fd(native_socket fd); expected<uint16_t> local_port_of_fd(native_socket fd);
...@@ -339,6 +380,19 @@ public: ...@@ -339,6 +380,19 @@ public:
expected<doorman_ptr> new_tcp_doorman(uint16_t port, const char* in, expected<doorman_ptr> new_tcp_doorman(uint16_t port, const char* in,
bool reuse_addr) override; bool reuse_addr) override;
datagram_servant_ptr new_datagram_servant(native_socket fd) override;
datagram_servant_ptr
new_datagram_servant_for_endpoint(native_socket fd,
const ip_endpoint& ep) override;
expected<datagram_servant_ptr>
new_remote_udp_endpoint(const std::string& host, uint16_t port) override;
expected<datagram_servant_ptr>
new_local_udp_endpoint(uint16_t port,const char* in = nullptr,
bool reuse_addr = false) override;
void exec_later(resumable* ptr) override; void exec_later(resumable* ptr) override;
explicit default_multiplexer(actor_system* sys); explicit default_multiplexer(actor_system* sys);
...@@ -364,6 +418,9 @@ public: ...@@ -364,6 +418,9 @@ public:
/// Calls `ptr->resume`. /// Calls `ptr->resume`.
void resume(intrusive_ptr<resumable> ptr); void resume(intrusive_ptr<resumable> ptr);
/// Get the next id to create a new datagram handle
int64_t next_endpoint_id();
private: private:
/// Calls `epoll`, `kqueue`, or `poll` with or without blocking. /// Calls `epoll`, `kqueue`, or `poll` with or without blocking.
bool poll_once_impl(bool block); bool poll_once_impl(bool block);
...@@ -443,6 +500,9 @@ private: ...@@ -443,6 +500,9 @@ private:
/// avoids a possible deadlock where the multiplexer is blocked in /// avoids a possible deadlock where the multiplexer is blocked in
/// `wr_dispatch_request` when the pipe's buffer is full. /// `wr_dispatch_request` when the pipe's buffer is full.
std::vector<intrusive_ptr<resumable>> internally_posted_; std::vector<intrusive_ptr<resumable>> internally_posted_;
/// Sequential ids for handles of datagram servants
int64_t servant_ids_;
}; };
inline connection_handle conn_hdl_from_socket(native_socket fd) { inline connection_handle conn_hdl_from_socket(native_socket fd) {
...@@ -543,7 +603,8 @@ protected: ...@@ -543,7 +603,8 @@ protected:
return; return;
collected_ += rb; collected_ += rb;
if (collected_ >= read_threshold_) { if (collected_ >= read_threshold_) {
auto res = reader_->consume(&backend(), rd_buf_.data(), collected_); auto res = reader_->consume(&backend(), rd_buf_.data(),
collected_);
prepare_next_read(); prepare_next_read();
if (!res) { if (!res) {
passivate(); passivate();
...@@ -701,12 +762,205 @@ private: ...@@ -701,12 +762,205 @@ private:
ProtocolPolicy policy_; ProtocolPolicy policy_;
}; };
class datagram_handler : public event_handler {
public:
/// A smart pointer to a datagram manager.
using manager_ptr = intrusive_ptr<datagram_manager>;
/// A buffer class providing a compatible interface to `std::vector`.
using write_buffer_type = std::vector<char>;
using read_buffer_type = network::receive_buffer;
/// A job for sending a datagram consisting of the sender and a buffer.
using job_type = std::pair<datagram_handle, write_buffer_type>;
datagram_handler(default_multiplexer& backend_ref, native_socket sockfd);
/// Starts reading data from the socket, forwarding incoming data to `mgr`.
void start(datagram_manager* mgr);
/// Activates the datagram handler.
void activate(datagram_manager* mgr);
void ack_writes(bool x);
/// Copies data to the write buffer.
/// @warning Not thread safe.
void write(datagram_handle hdl, const void* buf, size_t num_bytes);
/// Returns the write buffer of this enpoint.
/// @warning Must not be modified outside the IO multiplexers event loop
/// once the stream has been started.
inline write_buffer_type& wr_buf(datagram_handle hdl) {
wr_offline_buf_.emplace_back();
wr_offline_buf_.back().first = hdl;
return wr_offline_buf_.back().second;
}
/// Enqueues a buffer to be sent as a datagram.
/// @warning Must not be modified outside the IO multiplexers event loop
/// once the stream has been started.
inline void enqueue_datagram(datagram_handle hdl, std::vector<char> buf) {
wr_offline_buf_.emplace_back(hdl, move(buf));
}
/// Returns the read buffer of this stream.
/// @warning Must not be modified outside the IO multiplexers event loop
/// once the stream has been started.
inline read_buffer_type& rd_buf() {
return rd_buf_;
}
/// Sends the content of the write buffer, calling the `io_failure`
/// member function of `mgr` in case of an error.
/// @warning Must not be called outside the IO multiplexers event loop
/// once the stream has been started.
void flush(const manager_ptr& mgr);
/// Closes the read channel of the underlying socket and removes
/// this handler from its parent.
void stop_reading();
void removed_from_loop(operation op) override;
void add_endpoint(datagram_handle hdl, const ip_endpoint& ep,
const manager_ptr mgr);
std::unordered_map<datagram_handle, ip_endpoint>& endpoints();
const std::unordered_map<datagram_handle, ip_endpoint>& endpoints() const;
void remove_endpoint(datagram_handle hdl);
inline ip_endpoint& sending_endpoint() {
return sender_;
}
protected:
template <class Policy>
void handle_event_impl(io::network::operation op, Policy& policy) {
CAF_LOG_TRACE(CAF_ARG(op));
auto mcr = max_consecutive_reads();
switch (op) {
case io::network::operation::read: {
// Loop until an error occurs or we have nothing more to read
// or until we have handled `mcr` reads.
for (size_t i = 0; i < mcr; ++i) {
if (!policy.read_datagram(num_bytes_, fd(), rd_buf_.data(),
rd_buf_.size(), sender_)) {
reader_->io_failure(&backend(), operation::read);
passivate();
return;
}
if (num_bytes_ > 0) {
rd_buf_.resize(num_bytes_);
auto itr = hdl_by_ep_.find(sender_);
bool consumed = false;
if (itr == hdl_by_ep_.end())
consumed = reader_->new_endpoint(rd_buf_);
else
consumed = reader_->consume(&backend(), itr->second, rd_buf_);
prepare_next_read();
if (!consumed) {
passivate();
return;
}
}
}
break;
}
case io::network::operation::write: {
size_t wb; // written bytes
auto itr = ep_by_hdl_.find(wr_buf_.first);
// maybe this could be an assert?
if (itr == ep_by_hdl_.end())
CAF_RAISE_ERROR("got write event for undefined endpoint");
auto& id = itr->first;
auto& ep = itr->second;
std::vector<char> buf;
std::swap(buf, wr_buf_.second);
auto size_as_int = static_cast<int>(buf.size());
if (size_as_int > send_buffer_size_) {
send_buffer_size_ = size_as_int;
send_buffer_size(fd(), size_as_int);
}
if (!policy.write_datagram(wb, fd(), buf.data(),
buf.size(), ep)) {
writer_->io_failure(&backend(), operation::write);
backend().del(operation::write, fd(), this);
} else if (wb > 0) {
CAF_ASSERT(wb == buf.size());
if (ack_writes_)
writer_->datagram_sent(&backend(), id, wb, std::move(buf));
prepare_next_write();
} else {
if (writer_)
writer_->io_failure(&backend(), operation::write);
}
break;
}
case operation::propagate_error:
if (reader_)
reader_->io_failure(&backend(), operation::read);
if (writer_)
writer_->io_failure(&backend(), operation::write);
// backend will delete this handler anyway,
// no need to call backend().del() here
}
}
private:
size_t max_consecutive_reads();
void prepare_next_read();
void prepare_next_write();
// known endpoints and broker servants
std::unordered_map<ip_endpoint, datagram_handle> hdl_by_ep_;
std::unordered_map<datagram_handle, ip_endpoint> ep_by_hdl_;
// state for reading
const size_t max_datagram_size_;
size_t num_bytes_;
read_buffer_type rd_buf_;
manager_ptr reader_;
ip_endpoint sender_;
// state for writing
int send_buffer_size_;
bool ack_writes_;
bool writing_;
std::deque<job_type> wr_offline_buf_;
job_type wr_buf_;
manager_ptr writer_;
};
/// A concrete datagram_handler with a technology-dependent policy.
template <class ProtocolPolicy>
class datagram_handler_impl : public datagram_handler {
public:
template <class... Ts>
datagram_handler_impl(default_multiplexer& mpx, native_socket sockfd,
Ts&&... xs)
: datagram_handler(mpx, sockfd),
policy_(std::forward<Ts>(xs)...) {
// nop
}
void handle_event(io::network::operation op) override {
this->handle_event_impl(op, policy_);
}
private:
ProtocolPolicy policy_;
};
expected<native_socket> expected<native_socket>
new_tcp_connection(const std::string& host, uint16_t port, new_tcp_connection(const std::string& host, uint16_t port,
optional<protocol::network> preferred = none); optional<protocol::network> preferred = none);
expected<native_socket> new_tcp_acceptor_impl(uint16_t port, const char* addr, expected<native_socket>
bool reuse_addr); new_tcp_acceptor_impl(uint16_t port, const char* addr, bool reuse_addr);
/// Default doorman implementation. /// Default doorman implementation.
class doorman_impl : public doorman { class doorman_impl : public doorman {
...@@ -763,6 +1017,62 @@ protected: ...@@ -763,6 +1017,62 @@ protected:
stream_impl<tcp_policy> stream_; stream_impl<tcp_policy> stream_;
}; };
/// Default datagram servant implementation
class datagram_servant_impl : public datagram_servant {
using id_type = int64_t;
public:
datagram_servant_impl(default_multiplexer& mx, native_socket sockfd,
int64_t id);
bool new_endpoint(network::receive_buffer& buf) override;
void ack_writes(bool enable) override;
std::vector<char>& wr_buf(datagram_handle hdl) override;
void enqueue_datagram(datagram_handle hdl, std::vector<char> buf) override;
network::receive_buffer& rd_buf() override;
void stop_reading() override;
void flush() override;
std::string addr() const override;
uint16_t port(datagram_handle hdl) const override;
uint16_t local_port() const override;
std::vector<datagram_handle> hdls() const override;
void add_endpoint(const ip_endpoint& ep, datagram_handle hdl) override;
void remove_endpoint(datagram_handle hdl) override;
void launch() override;
void add_to_loop() override;
void remove_from_loop() override;
void detach_handles() override;
private:
bool launched_;
datagram_handler_impl<udp_policy> handler_;
};
expected<std::pair<native_socket, ip_endpoint>>
new_remote_udp_endpoint_impl(const std::string& host, uint16_t port,
optional<protocol::network> preferred = none);
expected<std::pair<native_socket, protocol::network>>
new_local_udp_endpoint_impl(uint16_t port, const char* addr,
bool reuse_addr = false,
optional<protocol::network> preferred = none);
} // namespace network } // namespace network
} // namespace io } // namespace io
} // namespace caf } // namespace caf
......
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "caf/optional.hpp" #include "caf/optional.hpp"
#include "caf/io/network/protocol.hpp" #include "caf/io/network/protocol.hpp"
#include "caf/io/network/ip_endpoint.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
...@@ -78,6 +79,11 @@ public: ...@@ -78,6 +79,11 @@ public:
static std::vector<std::pair<std::string, protocol::network>> static std::vector<std::pair<std::string, protocol::network>>
server_address(uint16_t port, const char* host, server_address(uint16_t port, const char* host,
optional<protocol::network> preferred = none); optional<protocol::network> preferred = none);
/// Writes datagram endpoint info for `host`:`port` into ep
static bool
get_endpoint(const std::string& host, uint16_t port, ip_endpoint& ep,
optional<protocol::network> preferred = none);
}; };
} // namespace network } // namespace network
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_IP_ENDPOINT_HPP
#define CAF_IO_IP_ENDPOINT_HPP
#include <deque>
#include <vector>
#include <string>
#include <functional>
#include "caf/error.hpp"
#include "caf/meta/type_name.hpp"
#include "caf/meta/save_callback.hpp"
#include "caf/meta/load_callback.hpp"
struct sockaddr;
struct sockaddr_storage;
struct sockaddr_in;
struct sockaddr_in6;
namespace caf {
namespace io {
namespace network {
// hash for char*, see:
// - https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
// - http://www.isthe.com/chongo/tech/comp/fnv/index.html
// Always hash 128 bit address, for v4 we use the embedded addr.
class ep_hash {
public:
ep_hash();
size_t operator()(const sockaddr& sa) const noexcept;
size_t hash(const sockaddr_in* sa) const noexcept;
size_t hash(const sockaddr_in6* sa) const noexcept;
};
/// A hashable wrapper for a sockaddr storage.
struct ip_endpoint {
public:
/// Default constructor for sockaddr storage which reserves memory for the
/// internal data structure on creation.
ip_endpoint();
/// Move constructor.
ip_endpoint(ip_endpoint&&) = default;
/// Copy constructor.
ip_endpoint(const ip_endpoint&);
/// Destructor
~ip_endpoint() = default;
/// Copy assignment operator.
ip_endpoint& operator=(const ip_endpoint&);
/// Move assignment operator.
ip_endpoint& operator=(ip_endpoint&&) = default;
/// Returns a pointer to the internal address storage.
sockaddr* address();
/// Returns a constant pointer to the internal address storage.
const sockaddr* caddress() const;
/// Returns the length of the stored address.
size_t* length();
/// Returns the length of the stored address.
const size_t* clength() const;
/// Null internal storage and length.
void clear();
private:
struct impl;
struct impl_deleter { void operator()(impl*) const; };
std::unique_ptr<impl,impl_deleter> ptr_;
};
bool operator==(const ip_endpoint& lhs, const ip_endpoint& rhs);
std::string to_string(const ip_endpoint& ep);
std::string host(const ip_endpoint& ep);
uint16_t port(const ip_endpoint& ep);
uint32_t family(const ip_endpoint& ep);
error load_endpoint(ip_endpoint& ep, uint32_t& f, std::string& h,
uint16_t& p, size_t& l);
error save_endpoint(ip_endpoint& ep, uint32_t& f, std::string& h,
uint16_t& p, size_t& l);
template <class Inspector>
typename Inspector::result_type inspect(Inspector& fun, ip_endpoint& ep) {
uint32_t f;
std::string h;
uint16_t p;
size_t l;
if (*ep.length() > 0) {
f = family(ep);
h = host(ep);
p = port(ep);
l = *ep.length();
}
auto load = [&] { return load_endpoint(ep, f, h, p, l); };
auto save = [&] { return save_endpoint(ep, f, h, p, l); };
return fun(meta::type_name("ip_endpoint"), f, h, p, l,
meta::load_callback(load), meta::save_callback(save));
}
} // namespace network
} // namespace io
} // namespace caf
namespace std {
template <>
struct hash<caf::io::network::ip_endpoint> {
using argument_type = caf::io::network::ip_endpoint;
using result_type = size_t;
result_type operator()(const argument_type& ep) const {
auto ptr = ep.caddress();
return caf::io::network::ep_hash{}(*ptr);
}
};
} // namespace std
#endif // CAF_IO_IP_ENDPOINT_HPP
...@@ -71,9 +71,6 @@ public: ...@@ -71,9 +71,6 @@ public:
/// Get the address of the underlying I/O device. /// Get the address of the underlying I/O device.
virtual std::string addr() const = 0; virtual std::string addr() const = 0;
/// Get the port of the underlying I/O device.
virtual uint16_t port() const = 0;
protected: protected:
/// Creates a message signalizing a disconnect to the parent. /// Creates a message signalizing a disconnect to the parent.
virtual message detach_message() = 0; virtual message detach_message() = 0;
......
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
#include "caf/io/connection_handle.hpp" #include "caf/io/connection_handle.hpp"
#include "caf/io/network/protocol.hpp" #include "caf/io/network/protocol.hpp"
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/io/network/native_socket.hpp" #include "caf/io/network/native_socket.hpp"
namespace caf { namespace caf {
...@@ -69,6 +70,26 @@ public: ...@@ -69,6 +70,26 @@ public:
const char* in = nullptr, const char* in = nullptr,
bool reuse_addr = false) = 0; bool reuse_addr = false) = 0;
/// Creates a new `datagram_servant` from a native socket handle.
/// @threadsafe
virtual datagram_servant_ptr new_datagram_servant(native_socket fd) = 0;
virtual datagram_servant_ptr
new_datagram_servant_for_endpoint(native_socket fd, const ip_endpoint& ep) = 0;
/// Create a new `datagram_servant` to contact a remote endpoint `host` and
/// `port`.
/// @warning Do not call from outside the multiplexer's event loop.
virtual expected<datagram_servant_ptr>
new_remote_udp_endpoint(const std::string& host, uint16_t port) = 0;
/// Create a new `datagram_servant` that receives datagrams on the local
/// `port`, optionally only accepting connections from IP address `in`.
/// @warning Do not call from outside the multiplexer's event loop.
virtual expected<datagram_servant_ptr>
new_local_udp_endpoint(uint16_t port, const char* in = nullptr,
bool reuse_addr = false) = 0;
/// Simple wrapper for runnables /// Simple wrapper for runnables
class runnable : public resumable, public ref_counted { class runnable : public resumable, public ref_counted {
public: public:
......
...@@ -20,6 +20,8 @@ ...@@ -20,6 +20,8 @@
#ifndef CAF_IO_NETWORK_OPERATION_HPP #ifndef CAF_IO_NETWORK_OPERATION_HPP
#define CAF_IO_NETWORK_OPERATION_HPP #define CAF_IO_NETWORK_OPERATION_HPP
#include <string>
namespace caf { namespace caf {
namespace io { namespace io {
namespace network { namespace network {
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_NETWORK_RECEIVE_BUFFER_HPP
#define CAF_IO_NETWORK_RECEIVE_BUFFER_HPP
#include <memory>
#include <cstddef>
#include "caf/allowed_unsafe_message_type.hpp"
namespace caf {
namespace io {
namespace network {
/// A container that does not call constructors and destructors for its values.
class receive_buffer {
public:
using value_type = char;
using size_type = size_t;
using difference_type = std::ptrdiff_t;
using reference = value_type&;
using const_reference = const value_type&;
using pointer = value_type*;
using const_pointer = std::pointer_traits<pointer>::rebind<const value_type>;
using iterator = pointer;
using const_iterator = const_pointer;
using reverse_iterator = std::reverse_iterator<iterator>;
using const_reverse_iterator = std::reverse_iterator<const_iterator>;
using buffer_ptr = std::unique_ptr<value_type[],
std::default_delete<value_type[]>>;
/// Create an empty container.
receive_buffer();
/// Create an empty container with `size` storage. Data in the storage is
/// not initialized.
receive_buffer(size_t size);
/// Move constructor.
receive_buffer(receive_buffer&& other) noexcept;
/// Copy constructor.
receive_buffer(const receive_buffer& other);
/// Move assignment operator.
receive_buffer& operator=(receive_buffer&& other) noexcept;
/// Copy assignment operator.
receive_buffer& operator=(const receive_buffer& other);
/// Returns a pointer to the underlying buffer.
inline pointer data() noexcept {
return buffer_.get();
}
/// Returns a const pointer to the data.
inline const_pointer data() const noexcept {
return buffer_.get();
}
/// Returns the number of stored elements.
size_type size() const noexcept {
return size_;
}
/// Returns the number of elements that the container has allocated space for.
size_type capacity() const noexcept {
return capacity_;
}
/// Returns the maximum possible number of elements the container
/// could theoretically hold.
size_type max_size() const noexcept {
return std::numeric_limits<size_t>::max();
}
/// Resize the container to `new_size`. While this may increase its storage,
/// no storage will be released.
void resize(size_type new_size);
/// Set the size of the storage to `new_size`. If `new_size` is smaller than
/// the current capacity nothing happens. If `new_size` is larger than the
/// current capacity all iterators are invalidated.
void reserve(size_type new_size);
/// Shrink the container to its current size.
void shrink_to_fit();
/// Check if the container is empty.
inline bool empty() const noexcept {
return size_ == 0;
}
/// Clears the content of the container and releases the allocated storage.
void clear();
/// Swap contents with `other` receive buffer.
void swap(receive_buffer& other) noexcept;
/// Returns an iterator to the beginning.
inline iterator begin() noexcept {
return buffer_.get();
}
/// Returns an iterator to the end.
inline iterator end() noexcept {
return buffer_.get() + size_;
}
/// Returns an iterator to the beginning.
inline const_iterator begin() const noexcept {
return buffer_.get();
}
/// Returns an iterator to the end.
inline const_iterator end() const noexcept {
return buffer_.get() + size_;
}
/// Returns an iterator to the beginning.
inline const_iterator cbegin() const noexcept {
return buffer_.get();
}
/// Returns an iterator to the end.
inline const_iterator cend() const noexcept {
return buffer_.get() + size_;
}
/// Returns an iterator to the reverse beginning.
inline reverse_iterator rbegin() noexcept {
return reverse_iterator{buffer_.get() + size_};
}
/// Returns an iterator to the reverse end of the data.
inline reverse_iterator rend() noexcept {
return reverse_iterator{buffer_.get()};
}
/// Returns an iterator to the reverse beginning.
inline const_reverse_iterator rbegin() const noexcept {
return const_reverse_iterator{buffer_.get() + size_};
}
/// Returns an iterator to the reverse end of the data.
inline const_reverse_iterator rend() const noexcept {
return const_reverse_iterator{buffer_.get()};
}
/// Returns an iterator to the reverse beginning.
inline const_reverse_iterator crbegin() const noexcept {
return const_reverse_iterator{buffer_.get() + size_};
}
/// Returns an iterator to the reverse end of the data.
inline const_reverse_iterator crend() const noexcept {
return const_reverse_iterator{buffer_.get()};
}
/// Insert `value` before `pos`.
iterator insert(iterator pos, value_type value);
/// Append `value`.
void push_back(value_type value);
private:
// Increse the buffer capacity, maintaining its data. May invalidate iterators.
void increase_by(size_t bytes);
// Reduce the buffer capacity, maintaining its data. May invalidate iterators.
void shrink_by(size_t bytes);
buffer_ptr buffer_;
size_type capacity_;
size_type size_;
};
} // namepsace network
} // namespace io
} // namespace caf
#endif // CAF_IO_NETWORK_RECEIVE_BUFFER_HPP
...@@ -41,6 +41,9 @@ public: ...@@ -41,6 +41,9 @@ public:
/// Called by the underlying I/O device whenever it sent data. /// Called by the underlying I/O device whenever it sent data.
virtual void data_transferred(execution_unit* ctx, size_t num_bytes, virtual void data_transferred(execution_unit* ctx, size_t num_bytes,
size_t remaining_bytes) = 0; size_t remaining_bytes) = 0;
/// Get the port of the underlying I/O device.
virtual uint16_t port() const = 0;
}; };
} // namespace network } // namespace network
......
...@@ -26,12 +26,16 @@ ...@@ -26,12 +26,16 @@
#include "caf/io/abstract_broker.hpp" #include "caf/io/abstract_broker.hpp"
#include "caf/io/network/multiplexer.hpp" #include "caf/io/network/multiplexer.hpp"
#include "caf/io/network/receive_buffer.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
namespace network { namespace network {
class test_multiplexer : public multiplexer { class test_multiplexer : public multiplexer {
private:
struct datagram_data;
public: public:
explicit test_multiplexer(actor_system* sys); explicit test_multiplexer(actor_system* sys);
...@@ -47,6 +51,19 @@ public: ...@@ -47,6 +51,19 @@ public:
expected<doorman_ptr> new_tcp_doorman(uint16_t prt, const char* in, expected<doorman_ptr> new_tcp_doorman(uint16_t prt, const char* in,
bool reuse_addr) override; bool reuse_addr) override;
datagram_servant_ptr new_datagram_servant(native_socket fd) override;
datagram_servant_ptr
new_datagram_servant_for_endpoint(native_socket fd,
const ip_endpoint& ep) override;
expected<datagram_servant_ptr>
new_remote_udp_endpoint(const std::string& host, uint16_t port) override;
expected<datagram_servant_ptr>
new_local_udp_endpoint(uint16_t port, const char* in = nullptr,
bool reuse_addr = false) override;
/// Checks whether `x` is assigned to any known doorman or is user-provided /// Checks whether `x` is assigned to any known doorman or is user-provided
/// for future assignment. /// for future assignment.
bool is_known_port(uint16_t x) const; bool is_known_port(uint16_t x) const;
...@@ -55,6 +72,8 @@ public: ...@@ -55,6 +72,8 @@ public:
/// for future assignment. /// for future assignment.
bool is_known_handle(accept_handle x) const; bool is_known_handle(accept_handle x) const;
bool is_known_handle(datagram_handle x) const;
supervisor_ptr make_supervisor() override; supervisor_ptr make_supervisor() override;
bool try_run_once() override; bool try_run_once() override;
...@@ -67,43 +86,101 @@ public: ...@@ -67,43 +86,101 @@ public:
doorman_ptr new_doorman(accept_handle, uint16_t port); doorman_ptr new_doorman(accept_handle, uint16_t port);
public:
datagram_servant_ptr new_datagram_servant(datagram_handle, uint16_t port);
datagram_servant_ptr new_datagram_servant(datagram_handle,
const std::string& host,
uint16_t port);
void provide_scribe(std::string host, uint16_t desired_port, void provide_scribe(std::string host, uint16_t desired_port,
connection_handle hdl); connection_handle hdl);
void provide_acceptor(uint16_t desired_port, accept_handle hdl); void provide_acceptor(uint16_t desired_port, accept_handle hdl);
/// A buffer storing bytes. void provide_datagram_servant(uint16_t desired_port, datagram_handle hdl);
void provide_datagram_servant(std::string host, uint16_t desired_port,
datagram_handle hdl);
/// Generate an id for a new servant.
int64_t next_endpoint_id();
/// A buffer storing bytes used for TCP related components.
using buffer_type = std::vector<char>; using buffer_type = std::vector<char>;
/// Buffers storing bytes for UDP related components.
using read_buffer_type = network::receive_buffer;
using write_buffer_type = buffer_type;
using read_job_type = std::pair<datagram_handle, read_buffer_type>;
using write_job_type = std::pair<datagram_handle, write_buffer_type>;
using write_job_queue_type = std::deque<write_job_type>;
using shared_buffer_type = std::shared_ptr<buffer_type>; using shared_buffer_type = std::shared_ptr<buffer_type>;
using shared_job_queue_type = std::shared_ptr<write_job_queue_type>;
/// Models pending data on the network, i.e., the network /// Models pending data on the network, i.e., the network
/// input buffer usually managed by the operating system. /// input buffer usually managed by the operating system.
buffer_type& virtual_network_buffer(connection_handle hdl); buffer_type& virtual_network_buffer(connection_handle hdl);
/// Models pending data on the network, i.e., the network
/// input buffer usually managed by the operating system.
write_job_queue_type& virtual_network_buffer(datagram_handle hdl);
/// Returns the output buffer of the scribe identified by `hdl`. /// Returns the output buffer of the scribe identified by `hdl`.
buffer_type& output_buffer(connection_handle hdl); buffer_type& output_buffer(connection_handle hdl);
/// Returns the input buffer of the scribe identified by `hdl`. /// Returns the input buffer of the scribe identified by `hdl`.
buffer_type& input_buffer(connection_handle hdl); buffer_type& input_buffer(connection_handle hdl);
/// Returns the output buffer of the dgram servant identified by `hdl`.
write_job_type& output_buffer(datagram_handle hdl);
/// Returns the queue with all outgoing datagrams for the dgram servant
/// identified by `hdl`.
write_job_queue_type& output_queue(datagram_handle hdl);
/// Returns the input buffer of the dgram servant identified by `hdl`.
read_job_type& input_buffer(datagram_handle hdl);
/// Returns the configured read policy of the scribe identified by `hdl`. /// Returns the configured read policy of the scribe identified by `hdl`.
receive_policy::config& read_config(connection_handle hdl); receive_policy::config& read_config(connection_handle hdl);
/// Returns whether the scribe identified by `hdl` receives write ACKs. /// Returns whether the scribe identified by `hdl` receives write ACKs.
bool& ack_writes(connection_handle hdl); bool& ack_writes(connection_handle hdl);
/// Returns whether the dgram servant identified by `hdl` receives write ACKs.
bool& ack_writes(datagram_handle hdl);
/// Returns `true` if this handle has been closed /// Returns `true` if this handle has been closed
/// for reading, `false` otherwise. /// for reading, `false` otherwise.
bool& stopped_reading(connection_handle hdl); bool& stopped_reading(connection_handle hdl);
/// Returns `true` if this handle has been closed
/// for reading, `false` otherwise.
bool& stopped_reading(datagram_handle hdl);
/// Returns `true` if this handle is inactive, otherwise `false`. /// Returns `true` if this handle is inactive, otherwise `false`.
bool& passive_mode(connection_handle hdl); bool& passive_mode(connection_handle hdl);
/// Returns `true` if this handle is inactive, otherwise `false`.
bool& passive_mode(datagram_handle hdl);
scribe_ptr& impl_ptr(connection_handle hdl); scribe_ptr& impl_ptr(connection_handle hdl);
uint16_t& port(accept_handle hdl); uint16_t& port(accept_handle hdl);
uint16_t& port(datagram_handle hdl);
uint16_t& local_port(datagram_handle hdl);
size_t& datagram_size(datagram_handle hdl);
datagram_servant_ptr& impl_ptr(datagram_handle hdl);
/// Returns a map with all servants related to the servant `hdl`.
std::set<datagram_handle>& servants(datagram_handle hdl);
/// Returns `true` if this handle has been closed /// Returns `true` if this handle has been closed
/// for reading, `false` otherwise. /// for reading, `false` otherwise.
bool& stopped_reading(accept_handle hdl); bool& stopped_reading(accept_handle hdl);
...@@ -123,18 +200,33 @@ public: ...@@ -123,18 +200,33 @@ public:
test_multiplexer& peer, std::string host, test_multiplexer& peer, std::string host,
uint16_t port, connection_handle peer_hdl); uint16_t port, connection_handle peer_hdl);
/// Stores `hdl` as a pending endpoint for `src`.
void add_pending_endpoint(datagram_handle src, datagram_handle hdl);
using pending_connects_map = std::unordered_multimap<accept_handle, using pending_connects_map = std::unordered_multimap<accept_handle,
connection_handle>; connection_handle>;
pending_connects_map& pending_connects(); pending_connects_map& pending_connects();
using pending_endpoints_map = std::unordered_map<int64_t, datagram_handle>;
pending_endpoints_map& pending_endpoints();
using pending_scribes_map = std::map<std::pair<std::string, uint16_t>, using pending_scribes_map = std::map<std::pair<std::string, uint16_t>,
connection_handle>; connection_handle>;
using pending_doorman_map = std::unordered_map<uint16_t, accept_handle>; using pending_doorman_map = std::unordered_map<uint16_t, accept_handle>;
using pending_local_datagram_endpoints_map = std::map<uint16_t,
datagram_handle>;
using pending_remote_datagram_endpoints_map
= std::map<std::pair<std::string, uint16_t>, datagram_handle>;
bool has_pending_scribe(std::string x, uint16_t y); bool has_pending_scribe(std::string x, uint16_t y);
bool has_pending_remote_endpoint(std::string x, uint16_t y);
/// Accepts a pending connect on `hdl`. /// Accepts a pending connect on `hdl`.
void accept_connection(accept_handle hdl); void accept_connection(accept_handle hdl);
...@@ -154,10 +246,18 @@ public: ...@@ -154,10 +246,18 @@ public:
/// the configured read policy no longer allows receiving. /// the configured read policy no longer allows receiving.
bool read_data(connection_handle hdl); bool read_data(connection_handle hdl);
/// Reads the next datagram from the external input buffer.
bool read_data(datagram_handle hdl);
/// Appends `buf` to the virtual network buffer of `hdl` /// Appends `buf` to the virtual network buffer of `hdl`
/// and calls `read_data(hdl)` afterwards. /// and calls `read_data(hdl)` afterwards.
void virtual_send(connection_handle hdl, const buffer_type& buf); void virtual_send(connection_handle hdl, const buffer_type& buf);
/// Appends `buf` to the virtual network buffer of `hdl`
/// and calls `read_data(hdl)` afterwards.
void virtual_send(datagram_handle src, datagram_handle ep,
const buffer_type&);
/// Waits until a `runnable` is available and executes it. /// Waits until a `runnable` is available and executes it.
void exec_runnable(); void exec_runnable();
...@@ -197,6 +297,8 @@ private: ...@@ -197,6 +297,8 @@ private:
using guard_type = std::unique_lock<std::mutex>; using guard_type = std::unique_lock<std::mutex>;
std::shared_ptr<datagram_data> data_for_hdl(datagram_handle hdl);
struct scribe_data { struct scribe_data {
shared_buffer_type vn_buf_ptr; shared_buffer_type vn_buf_ptr;
shared_buffer_type wr_buf_ptr; shared_buffer_type wr_buf_ptr;
...@@ -223,19 +325,49 @@ private: ...@@ -223,19 +325,49 @@ private:
doorman_data(); doorman_data();
}; };
struct datagram_data {
shared_job_queue_type vn_buf_ptr;
shared_job_queue_type wr_buf_ptr;
write_job_queue_type& vn_buf;
write_job_queue_type& wr_buf;
read_job_type rd_buf;
datagram_servant_ptr ptr;
bool stopped_reading;
bool passive_mode;
bool ack_writes;
uint16_t port;
uint16_t local_port;
std::set<datagram_handle> servants;
size_t datagram_size;
// Allows creating an entangled scribes where the input of this scribe is
// the output of another scribe and vice versa.
datagram_data(
shared_job_queue_type input = std::make_shared<write_job_queue_type>(),
shared_job_queue_type output = std::make_shared<write_job_queue_type>()
);
};
using scribe_data_map = std::unordered_map<connection_handle, scribe_data>; using scribe_data_map = std::unordered_map<connection_handle, scribe_data>;
using doorman_data_map = std::unordered_map<accept_handle, doorman_data>; using doorman_data_map = std::unordered_map<accept_handle, doorman_data>;
using datagram_data_map = std::unordered_map<datagram_handle,
std::shared_ptr<datagram_data>>;
// guards resumables_ and scribes_ // guards resumables_ and scribes_
std::mutex mx_; std::mutex mx_;
std::condition_variable cv_; std::condition_variable cv_;
std::list<resumable_ptr> resumables_; std::list<resumable_ptr> resumables_;
pending_scribes_map scribes_; pending_scribes_map scribes_;
std::unordered_map<uint16_t, accept_handle> doormen_; pending_doorman_map doormen_;
scribe_data_map scribe_data_; scribe_data_map scribe_data_;
doorman_data_map doorman_data_; doorman_data_map doorman_data_;
pending_local_datagram_endpoints_map local_endpoints_;
pending_remote_datagram_endpoints_map remote_endpoints_;
pending_connects_map pending_connects_; pending_connects_map pending_connects_;
pending_endpoints_map pending_endpoints_;
datagram_data_map datagram_data_;
// extra state for making sure the test multiplexer is not used in a // extra state for making sure the test multiplexer is not used in a
// multithreaded setup // multithreaded setup
...@@ -246,6 +378,8 @@ private: ...@@ -246,6 +378,8 @@ private:
// Configures a one-shot handler for the next inlined runnable. // Configures a one-shot handler for the next inlined runnable.
std::function<void()> inline_runnable_callback_; std::function<void()> inline_runnable_callback_;
int64_t servant_ids_;
}; };
} // namespace network } // namespace network
......
...@@ -30,7 +30,9 @@ ...@@ -30,7 +30,9 @@
#include "caf/io/handle.hpp" #include "caf/io/handle.hpp"
#include "caf/io/accept_handle.hpp" #include "caf/io/accept_handle.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/connection_handle.hpp" #include "caf/io/connection_handle.hpp"
#include "caf/io/network/receive_buffer.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
...@@ -128,6 +130,60 @@ inspect(Inspector& f, acceptor_passivated_msg& x) { ...@@ -128,6 +130,60 @@ inspect(Inspector& f, acceptor_passivated_msg& x) {
return f(meta::type_name("acceptor_passivated_msg"), x.handle); return f(meta::type_name("acceptor_passivated_msg"), x.handle);
} }
/// Signalizes that a datagram with a certain size has been sent.
struct new_datagram_msg {
// Handle to the endpoint used.
datagram_handle handle;
// Buffer containing received data.
network::receive_buffer buf;
};
/// @relates new_datagram_msg
template <class Inspector>
typename Inspector::result_type inspect(Inspector& f, new_datagram_msg& x) {
return f(meta::type_name("new_datagram_msg"), x.handle, x.buf);
}
/// Signalizes that a datagram with a certain size has been sent.
struct datagram_sent_msg {
// Handle to the endpoint used.
datagram_handle handle;
// Number of bytes written.
uint64_t written;
// Buffer of the sent datagram, for reuse.
std::vector<char> buf;
};
/// @relates datagram_sent_msg
template <class Inspector>
typename Inspector::result_type inspect(Inspector& f, datagram_sent_msg& x) {
return f(meta::type_name("datagram_sent_msg"), x.handle, x.written, x.buf);
}
/// Signalizes that a datagram sink has entered passive mode.
struct datagram_servant_passivated_msg {
datagram_handle handle;
};
/// @relates datagram_servant_passivated_msg
template <class Inspector>
typename Inspector::result_type
inspect(Inspector& f, datagram_servant_passivated_msg& x) {
return f(meta::type_name("datagram_servant_passivated_msg"), x.handle);
}
/// Signalizes that a datagram endpoint has entered passive mode.
struct datagram_servant_closed_msg {
std::vector<datagram_handle> handles;
};
/// @relates datagram_servant_closed_msg
template <class Inspector>
typename Inspector::result_type
inspect(Inspector& f, datagram_servant_closed_msg& x) {
return f(meta::type_name("datagram_servant_closed_msg"), x.handles);
}
} // namespace io } // namespace io
} // namespace caf } // namespace caf
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2016 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#ifndef CAF_IO_VISITORS_HPP
#define CAF_IO_VISITORS_HPP
#include "caf/io/abstract_broker.hpp"
namespace caf {
namespace io {
struct addr_visitor {
using result_type = std::string;
addr_visitor(abstract_broker* ptr) : ptr_(ptr) { }
template <class Handle>
result_type operator()(const Handle& hdl) { return ptr_->remote_addr(hdl); }
abstract_broker* ptr_;
};
struct port_visitor {
using result_type = uint16_t;
port_visitor(abstract_broker* ptr) : ptr_(ptr) { }
template <class Handle>
result_type operator()(const Handle& hdl) { return ptr_->remote_port(hdl); }
abstract_broker* ptr_;
};
struct id_visitor {
using result_type = int64_t;
template <class Handle>
result_type operator()(const Handle& hdl) { return hdl.id(); }
};
struct hash_visitor {
using result_type = size_t;
template <class Handle>
result_type operator()(const Handle& hdl) const {
std::hash<Handle> f;
return f(hdl);
}
};
} // namespace io
} // namespace caf
#endif // CAF_IO_VISITORS_HPP
...@@ -64,6 +64,7 @@ bool abstract_broker::cleanup(error&& reason, execution_unit* host) { ...@@ -64,6 +64,7 @@ bool abstract_broker::cleanup(error&& reason, execution_unit* host) {
close_all(); close_all();
CAF_ASSERT(doormen_.empty()); CAF_ASSERT(doormen_.empty());
CAF_ASSERT(scribes_.empty()); CAF_ASSERT(scribes_.empty());
CAF_ASSERT(datagram_servants_.empty());
cache_.clear(); cache_.clear();
return local_actor::cleanup(std::move(reason), host); return local_actor::cleanup(std::move(reason), host);
} }
...@@ -109,6 +110,46 @@ void abstract_broker::flush(connection_handle hdl) { ...@@ -109,6 +110,46 @@ void abstract_broker::flush(connection_handle hdl) {
x->flush(); x->flush();
} }
void abstract_broker::ack_writes(datagram_handle hdl, bool enable) {
CAF_LOG_TRACE(CAF_ARG(hdl) << CAF_ARG(enable));
auto x = by_id(hdl);
if (x)
x->ack_writes(enable);
}
std::vector<char>& abstract_broker::wr_buf(datagram_handle hdl) {
auto x = by_id(hdl);
if (!x) {
CAF_LOG_ERROR("tried to access wr_buf() of an unknown"
"datagram_handle");
return dummy_wr_buf_;
}
return x->wr_buf(hdl);
}
void abstract_broker::enqueue_datagram(datagram_handle hdl,
std::vector<char> buf) {
auto x = by_id(hdl);
if (!x)
CAF_LOG_ERROR("tried to access datagram_buffer() of an unknown"
"datagram_handle");
x->enqueue_datagram(hdl, std::move(buf));
}
void abstract_broker::write(datagram_handle hdl, size_t bs,
const void* buf) {
auto& out = wr_buf(hdl);
auto first = reinterpret_cast<const char*>(buf);
auto last = first + bs;
out.insert(out.end(), first, last);
}
void abstract_broker::flush(datagram_handle hdl) {
auto x = by_id(hdl);
if (x)
x->flush();
}
std::vector<connection_handle> abstract_broker::connections() const { std::vector<connection_handle> abstract_broker::connections() const {
std::vector<connection_handle> result; std::vector<connection_handle> result;
result.reserve(scribes_.size()); result.reserve(scribes_.size());
...@@ -164,6 +205,83 @@ abstract_broker::add_tcp_doorman(uint16_t port, const char* in, ...@@ -164,6 +205,83 @@ abstract_broker::add_tcp_doorman(uint16_t port, const char* in,
return std::move(eptr.error()); return std::move(eptr.error());
} }
void abstract_broker::add_datagram_servant(datagram_servant_ptr ptr) {
CAF_LOG_TRACE(CAF_ARG(ptr));
CAF_ASSERT(ptr != nullptr);
CAF_ASSERT(ptr->parent() == nullptr);
ptr->set_parent(this);
auto hdls = ptr->hdls();
launch_servant(ptr);
for (auto& hdl : hdls)
add_hdl_for_datagram_servant(ptr, hdl);
}
void abstract_broker::add_hdl_for_datagram_servant(datagram_servant_ptr ptr,
datagram_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(hdl));
CAF_ASSERT(ptr != nullptr);
CAF_ASSERT(ptr->parent() == this);
get_map(hdl).emplace(hdl, std::move(ptr));
}
datagram_handle abstract_broker::add_datagram_servant(network::native_socket fd) {
CAF_LOG_TRACE(CAF_ARG(fd));
auto ptr = backend().new_datagram_servant(fd);
auto hdl = ptr->hdl();
add_datagram_servant(std::move(ptr));
return hdl;
}
datagram_handle
abstract_broker::add_datagram_servant_for_endpoint(network::native_socket fd,
const network::ip_endpoint& ep) {
CAF_LOG_TRACE(CAF_ARG(fd));
auto ptr = backend().new_datagram_servant_for_endpoint(fd, ep);
auto hdl = ptr->hdl();
add_datagram_servant(std::move(ptr));
return hdl;
}
expected<datagram_handle>
abstract_broker::add_udp_datagram_servant(const std::string& host,
uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port));
auto eptr = backend().new_remote_udp_endpoint(host, port);
if (eptr) {
auto ptr = std::move(*eptr);
auto hdl = ptr->hdl();
add_datagram_servant(std::move(ptr));
return hdl;
}
return std::move(eptr.error());
}
expected<std::pair<datagram_handle, uint16_t>>
abstract_broker::add_udp_datagram_servant(uint16_t port, const char* in,
bool reuse_addr) {
CAF_LOG_TRACE(CAF_ARG(port) << CAF_ARG(in) << CAF_ARG(reuse_addr));
auto eptr = backend().new_local_udp_endpoint(port, in, reuse_addr);
if (eptr) {
auto ptr = std::move(*eptr);
auto p = ptr->local_port();
auto hdl = ptr->hdl();
add_datagram_servant(std::move(ptr));
return std::make_pair(hdl, p);
}
return std::move(eptr.error());
}
void abstract_broker::move_datagram_servant(datagram_servant_ptr ptr) {
CAF_LOG_TRACE(CAF_ARG(ptr));
CAF_ASSERT(ptr != nullptr);
CAF_ASSERT(ptr->parent() != nullptr && ptr->parent() != this);
ptr->set_parent(this);
CAF_ASSERT(ptr->parent() == this);
auto hdls = ptr->hdls();
for (auto& hdl : hdls)
add_hdl_for_datagram_servant(ptr, hdl);
}
std::string abstract_broker::remote_addr(connection_handle hdl) { std::string abstract_broker::remote_addr(connection_handle hdl) {
auto i = scribes_.find(hdl); auto i = scribes_.find(hdl);
return i != scribes_.end() ? i->second->addr() : std::string{}; return i != scribes_.end() ? i->second->addr() : std::string{};
...@@ -191,6 +309,36 @@ accept_handle abstract_broker::hdl_by_port(uint16_t port) { ...@@ -191,6 +309,36 @@ accept_handle abstract_broker::hdl_by_port(uint16_t port) {
return invalid_accept_handle; return invalid_accept_handle;
} }
datagram_handle abstract_broker::datagram_hdl_by_port(uint16_t port) {
for (auto& kvp : datagram_servants_)
if (kvp.second->port(kvp.first) == port)
return kvp.first;
return invalid_datagram_handle;
}
std::string abstract_broker::remote_addr(datagram_handle hdl) {
auto i = datagram_servants_.find(hdl);
return i != datagram_servants_.end() ? i->second->addr() : std::string{};
}
uint16_t abstract_broker::remote_port(datagram_handle hdl) {
auto i = datagram_servants_.find(hdl);
return i != datagram_servants_.end() ? i->second->port(hdl) : 0;
}
uint16_t abstract_broker::local_port(datagram_handle hdl) {
auto i = datagram_servants_.find(hdl);
return i != datagram_servants_.end() ? i->second->local_port() : 0;
}
bool abstract_broker::remove_endpoint(datagram_handle hdl) {
auto x = by_id(hdl);
if (!x)
return false;
x->remove_endpoint(hdl);
return true;
}
void abstract_broker::close_all() { void abstract_broker::close_all() {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
while (!doormen_.empty()) { while (!doormen_.empty()) {
...@@ -201,6 +349,10 @@ void abstract_broker::close_all() { ...@@ -201,6 +349,10 @@ void abstract_broker::close_all() {
// stop_reading will remove the scribe from scribes_ // stop_reading will remove the scribe from scribes_
scribes_.begin()->second->stop_reading(); scribes_.begin()->second->stop_reading();
} }
while (!datagram_servants_.empty()) {
// stop reading will remove dgram servants from datagram_servants_
datagram_servants_.begin()->second->stop_reading();
}
} }
resumable::subtype_t abstract_broker::subtype() const { resumable::subtype_t abstract_broker::subtype() const {
...@@ -225,7 +377,6 @@ void abstract_broker::init_broker() { ...@@ -225,7 +377,6 @@ void abstract_broker::init_broker() {
// might call functions like add_connection // might call functions like add_connection
for (auto& kvp : doormen_) for (auto& kvp : doormen_)
kvp.second->launch(); kvp.second->launch();
} }
abstract_broker::abstract_broker(actor_config& cfg) : scheduled_actor(cfg) { abstract_broker::abstract_broker(actor_config& cfg) : scheduled_actor(cfg) {
...@@ -243,5 +394,10 @@ void abstract_broker::launch_servant(doorman_ptr& ptr) { ...@@ -243,5 +394,10 @@ void abstract_broker::launch_servant(doorman_ptr& ptr) {
ptr->launch(); ptr->launch();
} }
void abstract_broker::launch_servant(datagram_servant_ptr& ptr) {
if (getf(is_initialized_flag))
ptr->launch();
}
} // namespace io } // namespace io
} // namespace caf } // namespace caf
...@@ -35,12 +35,38 @@ ...@@ -35,12 +35,38 @@
#include "caf/io/basp/all.hpp" #include "caf/io/basp/all.hpp"
#include "caf/io/middleman.hpp" #include "caf/io/middleman.hpp"
#include "caf/io/connection_helper.hpp"
#include "caf/io/network/interfaces.hpp" #include "caf/io/network/interfaces.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
namespace {
// visitors to access handle variant of the context
struct seq_num_visitor {
using result_type = basp::sequence_type;
seq_num_visitor(basp_broker_state* ptr) : state(ptr) { }
template <class T>
result_type operator()(const T& hdl) {
return state->next_sequence_number(hdl);
}
basp_broker_state* state;
};
struct close_visitor {
using result_type = void;
close_visitor(broker* ptr) : b(ptr) { }
template <class T>
result_type operator()(const T& hdl) {
b->close(hdl);
}
broker* b;
};
} // namespace anonymous
const char* basp_broker_state::name = "basp_broker"; const char* basp_broker_state::name = "basp_broker";
/****************************************************************************** /******************************************************************************
...@@ -51,7 +77,8 @@ basp_broker_state::basp_broker_state(broker* selfptr) ...@@ -51,7 +77,8 @@ basp_broker_state::basp_broker_state(broker* selfptr)
: basp::instance::callee(selfptr->system(), : basp::instance::callee(selfptr->system(),
static_cast<proxy_registry::backend&>(*this)), static_cast<proxy_registry::backend&>(*this)),
self(selfptr), self(selfptr),
instance(selfptr, *this) { instance(selfptr, *this),
max_buffers(self->system().config().middleman_cached_udp_buffers) {
CAF_ASSERT(this_node() != none); CAF_ASSERT(this_node() != none);
} }
...@@ -70,7 +97,7 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) { ...@@ -70,7 +97,7 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
// payload received from a remote node; if a remote node A sends // payload received from a remote node; if a remote node A sends
// us a handle to a third node B, then we assume that A offers a route to B // us a handle to a third node B, then we assume that A offers a route to B
if (nid != this_context->id if (nid != this_context->id
&& instance.tbl().lookup_direct(nid) == invalid_connection_handle && !instance.tbl().lookup_direct(nid)
&& instance.tbl().add_indirect(this_context->id, nid)) && instance.tbl().add_indirect(this_context->id, nid))
learned_new_node_indirectly(nid); learned_new_node_indirectly(nid);
// we need to tell remote side we are watching this actor now; // we need to tell remote side we are watching this actor now;
...@@ -103,10 +130,13 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) { ...@@ -103,10 +130,13 @@ strong_actor_ptr basp_broker_state::make_proxy(node_id nid, actor_id aid) {
CAF_LOG_INFO("successfully created proxy instance, " CAF_LOG_INFO("successfully created proxy instance, "
"write announce_proxy_instance:" "write announce_proxy_instance:"
<< CAF_ARG(nid) << CAF_ARG(aid)); << CAF_ARG(nid) << CAF_ARG(aid));
auto& ctx = *this_context;
// tell remote side we are monitoring this actor now // tell remote side we are monitoring this actor now
instance.write_announce_proxy(self->context(), instance.write_announce_proxy(self->context(),
self->wr_buf(this_context->hdl), nid, aid); get_buffer(this_context->hdl),
instance.tbl().flush(*path); nid, aid,
ctx.requires_ordering ? ctx.seq_outgoing++ : 0);
instance.flush(*path);
mm->notify<hook::new_remote_actor>(res); mm->notify<hook::new_remote_actor>(res);
return res; return res;
} }
...@@ -159,9 +189,11 @@ void basp_broker_state::send_kill_proxy_instance(const node_id& nid, ...@@ -159,9 +189,11 @@ void basp_broker_state::send_kill_proxy_instance(const node_id& nid,
<< CAF_ARG(nid)); << CAF_ARG(nid));
return; return;
} }
instance.write_kill_proxy(self->context(), path->wr_buf, instance.write_kill_proxy(self->context(),
nid, aid, rsn); get_buffer(path->hdl),
instance.tbl().flush(*path); nid, aid, rsn,
visit(seq_num_visitor{this}, path->hdl));
instance.flush(*path);
} }
void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) { void basp_broker_state::proxy_announced(const node_id& nid, actor_id aid) {
...@@ -230,7 +262,7 @@ void basp_broker_state::deliver(const node_id& src_nid, actor_id src_aid, ...@@ -230,7 +262,7 @@ void basp_broker_state::deliver(const node_id& src_nid, actor_id src_aid,
break; break;
case link_atom::value.uint_value(): { case link_atom::value.uint_value(): {
if (src_nid != this_node()) { if (src_nid != this_node()) {
CAF_LOG_WARNING("received link message for an other node"); CAF_LOG_WARNING("received link message for another node");
return; return;
} }
auto ptr = msg.get_as<strong_actor_ptr>(1); auto ptr = msg.get_as<strong_actor_ptr>(1);
...@@ -342,10 +374,12 @@ void basp_broker_state::learned_new_node(const node_id& nid) { ...@@ -342,10 +374,12 @@ void basp_broker_state::learned_new_node(const node_id& nid) {
// send message to SpawnServ of remote node // send message to SpawnServ of remote node
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id}; 0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
visit(seq_num_visitor{this}, path->hdl)};
// writing std::numeric_limits<actor_id>::max() is a hack to get // writing std::numeric_limits<actor_id>::max() is a hack to get
// this send-to-named-actor feature working with older CAF releases // this send-to-named-actor feature working with older CAF releases
instance.write(self->context(), path->wr_buf, hdr, &writer); instance.write(self->context(), get_buffer(path->hdl),
hdr, &writer);
instance.flush(*path); instance.flush(*path);
} }
...@@ -357,16 +391,6 @@ void basp_broker_state::learned_new_node_directly(const node_id& nid, ...@@ -357,16 +391,6 @@ void basp_broker_state::learned_new_node_directly(const node_id& nid,
learned_new_node(nid); learned_new_node(nid);
} }
namespace {
struct connection_helper_state {
static const char* name;
};
const char* connection_helper_state::name = "connection_helper";
} // namespace <anonymous>
void basp_broker_state::learned_new_node_indirectly(const node_id& nid) { void basp_broker_state::learned_new_node_indirectly(const node_id& nid) {
CAF_ASSERT(this_context != nullptr); CAF_ASSERT(this_context != nullptr);
CAF_LOG_TRACE(CAF_ARG(nid)); CAF_LOG_TRACE(CAF_ARG(nid));
...@@ -377,48 +401,6 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) { ...@@ -377,48 +401,6 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) {
// indirect connection to the routing table; hence, spawning // indirect connection to the routing table; hence, spawning
// our helper here exactly once and there is no need to track // our helper here exactly once and there is no need to track
// in-flight connection requests // in-flight connection requests
auto connection_helper = [=](stateful_actor<connection_helper_state>* helper,
actor s) -> behavior {
CAF_LOG_TRACE(CAF_ARG(s));
helper->monitor(s);
helper->set_down_handler([=](down_msg& dm) {
CAF_LOG_TRACE(CAF_ARG(dm));
helper->quit(std::move(dm.reason));
});
return {
// this config is send from the remote `ConfigServ`
[=](const std::string&, message& msg) {
CAF_LOG_TRACE(CAF_ARG(msg));
CAF_LOG_DEBUG("received requested config:" << CAF_ARG(msg));
// whatever happens, we are done afterwards
helper->quit();
msg.apply({
[&](uint16_t port, network::address_listing& addresses) {
auto& mx = system().middleman().backend();
for (auto& kvp : addresses)
for (auto& addr : kvp.second) {
auto hdl = mx.new_tcp_scribe(addr, port);
if (hdl) {
// gotcha! send scribe to our BASP broker
// to initiate handshake etc.
CAF_LOG_INFO("connected directly:" << CAF_ARG(addr));
helper->send(s, connect_atom::value, *hdl, port);
return;
}
}
CAF_LOG_INFO("could not connect to node directly:" << CAF_ARG(nid));
}
});
},
after(std::chrono::minutes(10)) >> [=] {
CAF_LOG_TRACE(CAF_ARG(""));
// nothing heard in about 10 minutes... just a call it a day, then
CAF_LOG_INFO("aborted direct connection attempt after 10min:"
<< CAF_ARG(nid));
helper->quit(exit_reason::user_shutdown);
}
};
};
auto path = instance.tbl().lookup(nid); auto path = instance.tbl().lookup(nid);
if (!path) { if (!path) {
CAF_LOG_ERROR("learned_new_node_indirectly called, but no route to nid"); CAF_LOG_ERROR("learned_new_node_indirectly called, but no route to nid");
...@@ -429,38 +411,67 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) { ...@@ -429,38 +411,67 @@ void basp_broker_state::learned_new_node_indirectly(const node_id& nid) {
return; return;
} }
using namespace detail; using namespace detail;
auto try_connect = [&](std::string item) {
auto tmp = system().config().middleman_detach_utility_actors auto tmp = system().config().middleman_detach_utility_actors
? system().spawn<detached + hidden>(connection_helper, self) ? system().spawn<detached + hidden>(connection_helper, self)
: system().spawn<hidden>(connection_helper, self); : system().spawn<hidden>(connection_helper, self);
system().registry().put(tmp.id(), actor_cast<strong_actor_ptr>(tmp)); system().registry().put(tmp.id(), actor_cast<strong_actor_ptr>(tmp));
auto writer = make_callback([](serializer& sink) -> error { auto writer = make_callback([&item](serializer& sink) -> error {
auto name_atm = atom("ConfigServ"); auto name_atm = atom("ConfigServ");
std::vector<actor_id> stages; std::vector<actor_id> stages;
auto msg = make_message(get_atom::value, "basp.default-connectivity"); auto msg = make_message(get_atom::value, std::move(item));
return sink(name_atm, stages, msg); return sink(name_atm, stages, msg);
}); });
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, 0, this_node(), nid, tmp.id(), invalid_actor_id}; 0, 0, this_node(), nid, tmp.id(), invalid_actor_id,
instance.write(self->context(), path->wr_buf, hdr, &writer); visit(seq_num_visitor{this}, path->hdl)};
instance.write(self->context(), get_buffer(path->hdl),
hdr, &writer);
instance.flush(*path); instance.flush(*path);
};
if (enable_tcp)
try_connect("basp.default-connectivity-tcp");
if (enable_udp)
try_connect("basp.default-connectivity-udp");
} }
void basp_broker_state::set_context(connection_handle hdl) { void basp_broker_state::set_context(connection_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl)); CAF_LOG_TRACE(CAF_ARG(hdl));
auto i = ctx.find(hdl); auto i = ctx_tcp.find(hdl);
if (i == ctx.end()) { if (i == ctx_tcp.end()) {
CAF_LOG_INFO("create new BASP context:" << CAF_ARG(hdl)); CAF_LOG_INFO("create new BASP context:" << CAF_ARG(hdl));
i = ctx.emplace(hdl, i = ctx_tcp.emplace(
connection_context{ hdl,
basp::endpoint_context{
basp::await_header, basp::await_header,
basp::header{basp::message_type::server_handshake, 0, basp::header{basp::message_type::server_handshake, 0,
0, 0, none, none, 0, 0, none, none,
invalid_actor_id, invalid_actor_id}, invalid_actor_id, invalid_actor_id},
hdl, none, 0, 0, none,
false, 0, 0, basp::endpoint_context::pending_map()
}
).first;
}
this_context = &i->second;
}
void basp_broker_state::set_context(datagram_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl));
auto i = ctx_udp.find(hdl);
if (i == ctx_udp.end()) {
CAF_LOG_INFO("create new BASP context:" << CAF_ARG(hdl));
i = ctx_udp.emplace(
hdl, hdl,
none, basp::endpoint_context{
0, basp::await_header,
none}).first; basp::header{basp::message_type::server_handshake,
0, 0, 0, none, none,
invalid_actor_id, invalid_actor_id},
hdl, none, 0, 0, none,
true, 0, 0, basp::endpoint_context::pending_map()
}
).first;
} }
this_context = &i->second; this_context = &i->second;
} }
...@@ -476,18 +487,134 @@ void basp_broker_state::cleanup(connection_handle hdl) { ...@@ -476,18 +487,134 @@ void basp_broker_state::cleanup(connection_handle hdl) {
instance.tbl().erase_direct(hdl, cb); instance.tbl().erase_direct(hdl, cb);
// Remove the context for `hdl`, making sure clients receive an error in case // Remove the context for `hdl`, making sure clients receive an error in case
// this connection was closed during handshake. // this connection was closed during handshake.
auto i = ctx.find(hdl); auto i = ctx_tcp.find(hdl);
if (i != ctx.end()) { if (i != ctx_tcp.end()) {
auto& ref = i->second;
CAF_ASSERT(i->first == ref.hdl);
if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake);
}
ctx_tcp.erase(i);
}
}
void basp_broker_state::cleanup(datagram_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl));
// Remove handle from the routing table and clean up any node-specific state
// we might still have.
auto cb = make_callback([&](const node_id& nid) -> error {
purge_state(nid);
return none;
});
instance.tbl().erase_direct(hdl, cb);
// Remove the context for `hdl`, making sure clients receive an error in case
// this connection was closed during handshake.
auto i = ctx_udp.find(hdl);
if (i != ctx_udp.end()) {
auto& ref = i->second; auto& ref = i->second;
CAF_ASSERT(i->first == ref.hdl); CAF_ASSERT(i->first == ref.hdl);
if (ref.callback) { if (ref.callback) {
CAF_LOG_DEBUG("connection closed during handshake"); CAF_LOG_DEBUG("connection closed during handshake");
ref.callback->deliver(sec::disconnect_during_handshake); ref.callback->deliver(sec::disconnect_during_handshake);
} }
ctx.erase(i); ctx_udp.erase(i);
} }
} }
basp::sequence_type basp_broker_state::next_sequence_number(connection_handle) {
return 0;
}
basp::sequence_type
basp_broker_state::next_sequence_number(datagram_handle hdl) {
auto i = ctx_udp.find(hdl);
if (i != ctx_udp.end() && i->second.requires_ordering)
return i->second.seq_outgoing++;
return 0;
}
void basp_broker_state::add_pending(basp::sequence_type seq,
basp::endpoint_context& ep,
basp::header hdr,
std::vector<char> payload) {
ep.pending.emplace(seq, std::make_pair(std::move(hdr), std::move(payload)));
// TODO: choose reasonable default timeout, make configurable
self->delayed_send(self, std::chrono::milliseconds(20), pending_atom::value,
get<datagram_handle>(ep.hdl), seq);
}
bool basp_broker_state::deliver_pending(execution_unit* ctx,
basp::endpoint_context& ep) {
if (!ep.requires_ordering)
return true;
std::vector<char>* payload = nullptr;
auto itr = ep.pending.find(ep.seq_incoming);
while (itr != ep.pending.end()) {
ep.hdr = std::move(itr->second.first);
payload = &itr->second.second;
if (!instance.handle(ctx, get<datagram_handle>(ep.hdl),
ep.hdr, payload, false, ep, none))
return false;
ep.pending.erase(itr);
ep.seq_incoming += 1;
itr = ep.pending.find(ep.seq_incoming);
}
return true;
}
void basp_broker_state::drop_pending(basp::sequence_type seq,
basp::endpoint_context& ep) {
if (!ep.requires_ordering)
return;
ep.pending.erase(seq);
}
basp_broker_state::buffer_type&
basp_broker_state::get_buffer(endpoint_handle hdl) {
if (hdl.is<connection_handle>())
return get_buffer(get<connection_handle>(hdl));
else
return get_buffer(get<datagram_handle>(hdl));
}
basp_broker_state::buffer_type&
basp_broker_state::get_buffer(datagram_handle) {
if (cached_buffers.empty())
cached_buffers.emplace();
return cached_buffers.top();
}
basp_broker_state::buffer_type&
basp_broker_state::get_buffer(connection_handle hdl) {
return self->wr_buf(hdl);
}
basp_broker_state::buffer_type
basp_broker_state::pop_datagram_buffer(datagram_handle) {
std::vector<char> res;
std::swap(res, cached_buffers.top());
cached_buffers.pop();
return res;
}
void basp_broker_state::flush(endpoint_handle hdl) {
if (hdl.is<connection_handle>())
flush(get<connection_handle>(hdl));
else
flush(get<datagram_handle>(hdl));
}
void basp_broker_state::flush(datagram_handle hdl) {
if (!cached_buffers.empty() && !cached_buffers.top().empty())
self->enqueue_datagram(hdl, pop_datagram_buffer(hdl));
self->flush(hdl);
}
void basp_broker_state::flush(connection_handle hdl) {
self->flush(hdl);
}
/****************************************************************************** /******************************************************************************
* basp_broker * * basp_broker *
******************************************************************************/ ******************************************************************************/
...@@ -501,22 +628,36 @@ basp_broker::basp_broker(actor_config& cfg) ...@@ -501,22 +628,36 @@ basp_broker::basp_broker(actor_config& cfg)
behavior basp_broker::make_behavior() { behavior basp_broker::make_behavior() {
CAF_LOG_TRACE(CAF_ARG(system().node())); CAF_LOG_TRACE(CAF_ARG(system().node()));
state.enable_tcp = system().config().middleman_enable_tcp;
state.enable_udp = system().config().middleman_enable_udp;
if (system().config().middleman_enable_automatic_connections) { if (system().config().middleman_enable_automatic_connections) {
CAF_LOG_INFO("enable automatic connections"); CAF_LOG_INFO("enable automatic connections");
// open a random port and store a record for our peers how to // open a random port and store a record for our peers how to
// connect to this broker directly in the configuration server // connect to this broker directly in the configuration server
//auto port = if (state.enable_tcp) {
auto res = add_tcp_doorman(uint16_t{0}); auto res = add_tcp_doorman(uint16_t{0});
if (res) { if (res) {
auto port = res->second; auto port = res->second;
auto addrs = network::interfaces::list_addresses(false); auto addrs = network::interfaces::list_addresses(false);
auto config_server = system().registry().get(atom("ConfigServ")); auto config_server = system().registry().get(atom("ConfigServ"));
send(actor_cast<actor>(config_server), put_atom::value, send(actor_cast<actor>(config_server), put_atom::value,
"basp.default-connectivity", "basp.default-connectivity-tcp",
make_message(port, std::move(addrs)));
}
}
if (state.enable_udp) {
auto res = add_udp_datagram_servant(uint16_t{0});
if (res) {
auto port = res->second;
auto addrs = network::interfaces::list_addresses(false);
auto config_server = system().registry().get(atom("ConfigServ"));
send(actor_cast<actor>(config_server), put_atom::value,
"basp.default-connectivity-udp",
make_message(port, std::move(addrs))); make_message(port, std::move(addrs)));
state.enable_automatic_connections = true;
} }
} }
state.enable_automatic_connections = true;
}
auto heartbeat_interval = system().config().middleman_heartbeat_interval; auto heartbeat_interval = system().config().middleman_heartbeat_interval;
if (heartbeat_interval > 0) { if (heartbeat_interval > 0) {
CAF_LOG_INFO("enable heartbeat" << CAF_ARG(heartbeat_interval)); CAF_LOG_INFO("enable heartbeat" << CAF_ARG(heartbeat_interval));
...@@ -543,6 +684,43 @@ behavior basp_broker::make_behavior() { ...@@ -543,6 +684,43 @@ behavior basp_broker::make_behavior() {
ctx.cstate = next; ctx.cstate = next;
} }
}, },
// received from auto connect broker for UDP communication
[=](new_datagram_msg& msg, datagram_servant_ptr ptr, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(msg.handle));
auto hdl = ptr->hdl();
move_datagram_servant(ptr);
auto& ctx = state.ctx_udp[hdl];
ctx.hdl = hdl;
ctx.remote_port = port;
ctx.local_port = local_port(hdl);
ctx.requires_ordering = true;
ctx.seq_incoming = 0;
ctx.seq_outgoing = 1; // already sent the client handshake
// Let's not implement this twice
send(this, std::move(msg));
},
// received from underlying broker implementation
[=](new_datagram_msg& msg) {
CAF_LOG_TRACE(CAF_ARG(msg.handle));
state.set_context(msg.handle);
auto& ctx = *state.this_context;
if (ctx.local_port == 0)
ctx.local_port = local_port(msg.handle);
if (!state.instance.handle(context(), msg, ctx)) {
if (ctx.callback) {
CAF_LOG_WARNING("failed to handshake with remote node"
<< CAF_ARG(msg.handle));
ctx.callback->deliver(make_error(sec::disconnect_during_handshake));
}
state.cleanup(msg.handle);
close(msg.handle);
}
},
// received from the underlying broker implementation
[=](datagram_sent_msg& msg) {
if (state.cached_buffers.size() < state.max_buffers)
state.cached_buffers.emplace(std::move(msg.buf));
},
// received from proxy instances // received from proxy instances
[=](forward_atom, strong_actor_ptr& src, [=](forward_atom, strong_actor_ptr& src,
const std::vector<strong_actor_ptr>& fwd_stack, const std::vector<strong_actor_ptr>& fwd_stack,
...@@ -589,8 +767,10 @@ behavior basp_broker::make_behavior() { ...@@ -589,8 +767,10 @@ behavior basp_broker::make_behavior() {
basp::header hdr{basp::message_type::dispatch_message, basp::header hdr{basp::message_type::dispatch_message,
basp::header::named_receiver_flag, basp::header::named_receiver_flag,
0, cme->mid.integer_value(), state.this_node(), 0, cme->mid.integer_value(), state.this_node(),
dest_node, src->id(), invalid_actor_id}; dest_node, src->id(), invalid_actor_id,
state.instance.write(context(), path->wr_buf, hdr, &writer); visit(seq_num_visitor{&state}, path->hdl)};
state.instance.write(context(), state.get_buffer(path->hdl),
hdr, &writer);
state.instance.flush(*path); state.instance.flush(*path);
return delegated<message>(); return delegated<message>();
}, },
...@@ -598,9 +778,9 @@ behavior basp_broker::make_behavior() { ...@@ -598,9 +778,9 @@ behavior basp_broker::make_behavior() {
[=](const new_connection_msg& msg) { [=](const new_connection_msg& msg) {
CAF_LOG_TRACE(CAF_ARG(msg.handle)); CAF_LOG_TRACE(CAF_ARG(msg.handle));
auto& bi = state.instance; auto& bi = state.instance;
bi.write_server_handshake(context(), wr_buf(msg.handle), bi.write_server_handshake(context(), state.get_buffer(msg.handle),
local_port(msg.source)); local_port(msg.source));
flush(msg.handle); state.flush(msg.handle);
configure_read(msg.handle, receive_policy::exactly(basp::header_size)); configure_read(msg.handle, receive_policy::exactly(basp::header_size));
}, },
// received from underlying broker implementation // received from underlying broker implementation
...@@ -632,24 +812,77 @@ behavior basp_broker::make_behavior() { ...@@ -632,24 +812,77 @@ behavior basp_broker::make_behavior() {
auto rp = make_response_promise(); auto rp = make_response_promise();
auto hdl = ptr->hdl(); auto hdl = ptr->hdl();
add_scribe(std::move(ptr)); add_scribe(std::move(ptr));
auto& ctx = state.ctx[hdl]; auto& ctx = state.ctx_tcp[hdl];
ctx.hdl = hdl; ctx.hdl = hdl;
ctx.remote_port = port; ctx.remote_port = port;
ctx.cstate = basp::await_header; ctx.cstate = basp::await_header;
ctx.callback = rp; ctx.callback = rp;
ctx.requires_ordering = false;
// await server handshake // await server handshake
configure_read(hdl, receive_policy::exactly(basp::header_size)); configure_read(hdl, receive_policy::exactly(basp::header_size));
}, },
[=](publish_udp_atom, datagram_servant_ptr& ptr, uint16_t port,
const strong_actor_ptr& whom, std::set<std::string>& sigs) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port)
<< CAF_ARG(whom) << CAF_ARG(sigs));
CAF_ASSERT(ptr != nullptr);
add_datagram_servant(std::move(ptr));
if (whom)
system().registry().put(whom->id(), whom);
state.instance.add_published_actor(port, whom, std::move(sigs));
},
// received from middleman actor (delegated)
[=](contact_atom, datagram_servant_ptr& ptr, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(ptr) << CAF_ARG(port));
auto rp = make_response_promise();
auto hdl = ptr->hdl();
add_datagram_servant(std::move(ptr));
auto& ctx = state.ctx_udp[hdl];
ctx.hdl = hdl;
ctx.remote_port = port;
ctx.local_port = local_port(hdl);
ctx.callback = rp;
ctx.requires_ordering = true;
ctx.seq_incoming = 0;
ctx.seq_outgoing = 0;
auto& bi = state.instance;
bi.write_client_handshake(context(), state.get_buffer(hdl),
none, ctx.seq_outgoing++);
state.flush(hdl);
},
// received from underlying broker implementation
[=](const datagram_servant_closed_msg& msg) {
CAF_LOG_TRACE("");
// since all handles share a port, we can take any of them to query for
// port information
CAF_ASSERT(msg.handles.size() > 0);
auto port = local_port(msg.handles.front());
state.instance.remove_published_actor(port);
},
[=](delete_atom, const node_id& nid, actor_id aid) { [=](delete_atom, const node_id& nid, actor_id aid) {
CAF_LOG_TRACE(CAF_ARG(nid) << ", " << CAF_ARG(aid)); CAF_LOG_TRACE(CAF_ARG(nid) << ", " << CAF_ARG(aid));
state.proxies().erase(nid, aid); state.proxies().erase(nid, aid);
}, },
[=](unpublish_atom, const actor_addr& whom, uint16_t port) -> result<void> { [=](unpublish_atom, const actor_addr& whom, uint16_t port) -> result<void> {
CAF_LOG_TRACE(CAF_ARG(whom) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(whom) << CAF_ARG(port));
auto cb = make_callback([&](const strong_actor_ptr&, uint16_t x) -> error { auto cb = make_callback(
[&](const strong_actor_ptr&, uint16_t x) -> error {
close(hdl_by_port(x)); close(hdl_by_port(x));
return none; return none;
}); }
);
if (state.instance.remove_published_actor(whom, port, &cb) == 0)
return sec::no_actor_published_at_port;
return unit;
},
[=](unpublish_udp_atom, const actor_addr& whom, uint16_t port) -> result<void> {
CAF_LOG_TRACE(CAF_ARG(whom) << CAF_ARG(port));
auto cb = make_callback(
[&](const strong_actor_ptr&, uint16_t x) -> error {
close(datagram_hdl_by_port(x));
return none;
}
);
if (state.instance.remove_published_actor(whom, port, &cb) == 0) if (state.instance.remove_published_actor(whom, port, &cb) == 0)
return sec::no_actor_published_at_port; return sec::no_actor_published_at_port;
return unit; return unit;
...@@ -670,9 +903,9 @@ behavior basp_broker::make_behavior() { ...@@ -670,9 +903,9 @@ behavior basp_broker::make_behavior() {
std::string addr; std::string addr;
uint16_t port = 0; uint16_t port = 0;
auto hdl = state.instance.tbl().lookup_direct(x); auto hdl = state.instance.tbl().lookup_direct(x);
if (hdl != invalid_connection_handle) { if (hdl) {
addr = remote_addr(hdl); addr = visit(addr_visitor{this}, *hdl);
port = remote_port(hdl); port = visit(port_visitor{this}, *hdl);
} }
return std::make_tuple(x, std::move(addr), port); return std::make_tuple(x, std::move(addr), port);
}, },
...@@ -680,6 +913,20 @@ behavior basp_broker::make_behavior() { ...@@ -680,6 +913,20 @@ behavior basp_broker::make_behavior() {
state.instance.handle_heartbeat(context()); state.instance.handle_heartbeat(context());
delayed_send(this, std::chrono::milliseconds{interval}, delayed_send(this, std::chrono::milliseconds{interval},
tick_atom::value, interval); tick_atom::value, interval);
},
[=](pending_atom, datagram_handle hdl, basp::sequence_type seq) {
auto& ep = state.ctx_udp[hdl];
auto itr = ep.pending.find(seq);
if (itr != ep.pending.end()) {
if (seq == ep.seq_incoming ||
basp::instance::is_greater(seq, ep.seq_incoming)) {
// skip missing messages
ep.seq_incoming = seq;
state.deliver_pending(context(), ep);
} else {
state.drop_pending(seq, ep);
}
}
} }
}; };
} }
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include <chrono>
#include "caf/io/basp/instance.hpp"
#include "caf/io/connection_helper.hpp"
namespace caf {
namespace io {
namespace {
auto autoconnect_timeout = std::chrono::minutes(10);
} // namespace <anonymous>
const char* connection_helper_state::name = "connection_helper";
behavior datagram_connection_broker(broker* self, uint16_t port,
network::address_listing addresses,
actor system_broker) {
auto& mx = self->system().middleman().backend();
auto& this_node = self->system().node();
auto& app_id = self->system().config().middleman_app_identifier;
for (auto& kvp : addresses) {
for (auto& addr : kvp.second) {
auto eptr = mx.new_remote_udp_endpoint(addr, port);
if (eptr) {
auto hdl = (*eptr)->hdl();
self->add_datagram_servant(std::move(*eptr));
basp::instance::write_client_handshake(self->context(),
self->wr_buf(hdl),
none, this_node,
app_id);
}
}
}
return {
[=](new_datagram_msg& msg) {
auto hdl = msg.handle;
self->send(system_broker, std::move(msg), self->take(hdl), port);
self->quit();
},
after(autoconnect_timeout) >> [=]() {
CAF_LOG_TRACE(CAF_ARG(""));
// nothing heard in about 10 minutes... just a call it a day, then
CAF_LOG_INFO("aborted direct connection attempt after 10min:"
<< CAF_ARG(nid));
self->quit(exit_reason::user_shutdown);
}
};
}
behavior connection_helper(stateful_actor<connection_helper_state>* self,
actor b) {
CAF_LOG_TRACE(CAF_ARG(s));
self->monitor(b);
self->set_down_handler([=](down_msg& dm) {
CAF_LOG_TRACE(CAF_ARG(dm));
self->quit(std::move(dm.reason));
});
return {
// this config is send from the remote `ConfigServ`
[=](const std::string& item, message& msg) {
CAF_LOG_TRACE(CAF_ARG(item) << CAF_ARG(msg));
CAF_LOG_DEBUG("received requested config:" << CAF_ARG(msg));
// whatever happens, we are done afterwards
self->quit();
msg.apply({
[&](uint16_t port, network::address_listing& addresses) {
if (item == "basp.default-connectivity-tcp") {
auto& mx = self->system().middleman().backend();
for (auto& kvp : addresses) {
for (auto& addr : kvp.second) {
auto hdl = mx.new_tcp_scribe(addr, port);
if (hdl) {
// gotcha! send scribe to our BASP broker
// to initiate handshake etc.
CAF_LOG_INFO("connected directly:" << CAF_ARG(addr));
self->send(b, connect_atom::value, *hdl, port);
return;
}
}
}
CAF_LOG_INFO("could not connect to node directly:" << CAF_ARG(nid));
} else if (item == "basp.default-connectivity-udp") {
// create new broker to try addresses for communication via UDP
if (self->system().config().middleman_detach_utility_actors) {
self->system().middleman().spawn_broker<detached + hidden>(
datagram_connection_broker, port, std::move(addresses), b
);
} else {
self->system().middleman().spawn_broker<hidden>(
datagram_connection_broker, port, std::move(addresses), b
);
}
} else {
CAF_LOG_INFO("aborted direct connection attempt, unknown item: "
<< CAF_ARG(item));
}
}
});
},
after(autoconnect_timeout) >> [=] {
CAF_LOG_TRACE(CAF_ARG(""));
// nothing heard in about 10 minutes... just a call it a day, then
CAF_LOG_INFO("aborted direct connection attempt after 10min:"
<< CAF_ARG(nid));
self->quit(exit_reason::user_shutdown);
}
};
}
} // namespace io
} // namespace caf
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include "caf/io/network/datagram_manager.hpp"
namespace caf {
namespace io {
namespace network {
datagram_manager::~datagram_manager() {
// nop
}
} // namespace network
} // namespace io
} // namespace caf
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include "caf/io/datagram_servant.hpp"
#include "caf/logger.hpp"
namespace caf {
namespace io {
datagram_servant::datagram_servant(datagram_handle hdl)
: datagram_servant_base(hdl) {
// nop
}
datagram_servant::~datagram_servant() {
// nop
}
message datagram_servant::detach_message() {
return make_message(datagram_servant_closed_msg{hdls()});
}
bool datagram_servant::consume(execution_unit* ctx, datagram_handle hdl,
network::receive_buffer& buf) {
CAF_ASSERT(ctx != nullptr);
CAF_LOG_TRACE(CAF_ARG(buf.size()));
if (detached()) {
// we are already disconnected from the broker while the multiplexer
// did not yet remove the socket, this can happen if an I/O event causes
// the broker to call close_all() while the pollset contained
// further activities for the broker
return false;
}
// keep a strong reference to our parent until we leave scope
// to avoid UB when becoming detached during invocation
auto guard = parent_;
msg().handle = hdl;
auto& msg_buf = msg().buf;
msg_buf.swap(buf);
auto result = invoke_mailbox_element(ctx);
// swap buffer back to stream and implicitly flush wr_buf()
msg_buf.swap(buf);
flush();
return result;
}
void datagram_servant::datagram_sent(execution_unit* ctx, datagram_handle hdl,
size_t written, std::vector<char> buffer) {
CAF_LOG_TRACE(CAF_ARG(written));
if (detached())
return;
using sent_t = datagram_sent_msg;
using tmp_t = mailbox_element_vals<datagram_sent_msg>;
tmp_t tmp{strong_actor_ptr{}, message_id::make(),
mailbox_element::forwarding_stack{},
sent_t{hdl, written, std::move(buffer)}};
invoke_mailbox_element_impl(ctx, tmp);
}
void datagram_servant::io_failure(execution_unit* ctx, network::operation op) {
CAF_LOG_TRACE(CAF_ARG(hdl()) << CAF_ARG(op));
// keep compiler happy when compiling w/o logging
static_cast<void>(op);
detach(ctx, true);
}
} // namespace io
} // namespace caf
...@@ -46,8 +46,7 @@ ...@@ -46,8 +46,7 @@
# include <sys/socket.h> # include <sys/socket.h>
# include <netinet/in.h> # include <netinet/in.h>
# include <netinet/tcp.h> # include <netinet/tcp.h>
# include <utility>
#include <utility>
#endif #endif
using std::string; using std::string;
...@@ -56,6 +55,8 @@ using std::string; ...@@ -56,6 +55,8 @@ using std::string;
namespace { namespace {
constexpr size_t receive_buffer_size = std::numeric_limits<uint16_t>::max();
// safe ourselves some typing // safe ourselves some typing
constexpr auto ipv4 = caf::io::network::protocol::ipv4; constexpr auto ipv4 = caf::io::network::protocol::ipv4;
constexpr auto ipv6 = caf::io::network::protocol::ipv6; constexpr auto ipv6 = caf::io::network::protocol::ipv6;
...@@ -276,7 +277,8 @@ namespace network { ...@@ -276,7 +277,8 @@ namespace network {
: multiplexer(sys), : multiplexer(sys),
epollfd_(invalid_native_socket), epollfd_(invalid_native_socket),
shadow_(1), shadow_(1),
pipe_reader_(*this) { pipe_reader_(*this),
servant_ids_(0) {
init(); init();
epollfd_ = epoll_create1(EPOLL_CLOEXEC); epollfd_ = epoll_create1(EPOLL_CLOEXEC);
if (epollfd_ == -1) { if (epollfd_ == -1) {
...@@ -425,7 +427,8 @@ namespace network { ...@@ -425,7 +427,8 @@ namespace network {
default_multiplexer::default_multiplexer(actor_system* sys) default_multiplexer::default_multiplexer(actor_system* sys)
: multiplexer(sys), : multiplexer(sys),
epollfd_(-1), epollfd_(-1),
pipe_reader_(*this) { pipe_reader_(*this),
servant_ids_(0) {
init(); init();
// initial setup // initial setup
pipe_ = create_pipe(); pipe_ = create_pipe();
...@@ -481,7 +484,7 @@ namespace network { ...@@ -481,7 +484,7 @@ namespace network {
CAF_CRITICAL("poll() failed"); CAF_CRITICAL("poll() failed");
} }
} }
continue; // rince and repeat continue; // rinse and repeat
} }
if (presult == 0) if (presult == 0)
return false; return false;
...@@ -507,9 +510,8 @@ namespace network { ...@@ -507,9 +510,8 @@ namespace network {
} }
CAF_LOG_DEBUG(CAF_ARG(events_.size())); CAF_LOG_DEBUG(CAF_ARG(events_.size()));
poll_res.clear(); poll_res.clear();
for (auto& me : events_) { for (auto& me : events_)
handle(me); handle(me);
}
events_.clear(); events_.clear();
return true; return true;
} }
...@@ -676,6 +678,41 @@ rw_state write_some(size_t& result, native_socket fd, const void* buf, ...@@ -676,6 +678,41 @@ rw_state write_some(size_t& result, native_socket fd, const void* buf,
return true; return true;
} }
bool read_datagram(size_t& result, native_socket fd, void* buf, size_t buf_len,
ip_endpoint& ep) {
CAF_LOG_TRACE(CAF_ARG(fd));
memset(ep.address(), 0, sizeof(sockaddr_storage));
socklen_t len = sizeof(sockaddr_storage);
auto sres = ::recvfrom(fd, buf, buf_len, 0, ep.address(), &len);
if (is_error(sres, true)) {
CAF_LOG_ERROR("recvfrom returned" << CAF_ARG(sres));
return false;
}
if (sres == 0)
CAF_LOG_INFO("Received empty datagram");
else if (sres > static_cast<ssize_t>(buf_len))
CAF_LOG_WARNING("recvfrom cut of message, only received " << CAF_ARG(buf_len)
<< " of " << CAF_ARG(sres) << " bytes");
result = (sres > 0) ? static_cast<size_t>(sres) : 0;
*ep.length() = static_cast<size_t>(len);
return true;
}
bool write_datagram(size_t& result, native_socket fd, void* buf, size_t buf_len,
const ip_endpoint& ep) {
CAF_LOG_TRACE(CAF_ARG(fd) << CAF_ARG(buf_len));
socklen_t len = static_cast<socklen_t>(*ep.clength());
auto sres = ::sendto(fd, reinterpret_cast<socket_send_ptr>(buf), buf_len,
0, ep.caddress(),
len);
if (is_error(sres, true)) {
CAF_LOG_ERROR("sendto returned" << CAF_ARG(sres));
return false;
}
result = (sres > 0) ? static_cast<size_t>(sres) : 0;
return true;
}
// -- Policy class for TCP wrapping above free functions ----------------------- // -- Policy class for TCP wrapping above free functions -----------------------
read_some_fun tcp_policy::read_some = network::read_some; read_some_fun tcp_policy::read_some = network::read_some;
...@@ -684,6 +721,12 @@ write_some_fun tcp_policy::write_some = network::write_some; ...@@ -684,6 +721,12 @@ write_some_fun tcp_policy::write_some = network::write_some;
try_accept_fun tcp_policy::try_accept = network::try_accept; try_accept_fun tcp_policy::try_accept = network::try_accept;
// -- Policy class for UDP wrappign above free functions -----------------------
read_datagram_fun udp_policy::read_datagram = network::read_datagram;
write_datagram_fun udp_policy::write_datagram = network::write_datagram;
// -- Platform-independent parts of the default_multiplexer -------------------- // -- Platform-independent parts of the default_multiplexer --------------------
bool default_multiplexer::try_run_once() { bool default_multiplexer::try_run_once() {
...@@ -892,6 +935,43 @@ expected<doorman_ptr> default_multiplexer::new_tcp_doorman(uint16_t port, ...@@ -892,6 +935,43 @@ expected<doorman_ptr> default_multiplexer::new_tcp_doorman(uint16_t port,
return std::move(fd.error()); return std::move(fd.error());
} }
datagram_servant_ptr
default_multiplexer::new_datagram_servant(native_socket fd) {
CAF_LOG_TRACE(CAF_ARG(fd));
CAF_ASSERT(fd != network::invalid_native_socket);
return make_counted<datagram_servant_impl>(*this, fd, next_endpoint_id());
}
datagram_servant_ptr
default_multiplexer::new_datagram_servant_for_endpoint(native_socket fd,
const ip_endpoint& ep) {
CAF_LOG_TRACE(CAF_ARG(ep));
auto ds = new_datagram_servant(fd);
ds->add_endpoint(ep, ds->hdl());
return ds;
};
expected<datagram_servant_ptr>
default_multiplexer::new_remote_udp_endpoint(const std::string& host,
uint16_t port) {
auto res = new_remote_udp_endpoint_impl(host, port);
if (!res)
return std::move(res.error());
return new_datagram_servant_for_endpoint(res->first, res->second);
}
expected<datagram_servant_ptr>
default_multiplexer::new_local_udp_endpoint(uint16_t port, const char* in,
bool reuse_addr) {
auto res = new_local_udp_endpoint_impl(port, in, reuse_addr);
if (res)
return new_datagram_servant((*res).first);
return std::move(res.error());
}
int64_t default_multiplexer::next_endpoint_id() {
return servant_ids_++;
}
event_handler::event_handler(default_multiplexer& dm, native_socket sockfd) event_handler::event_handler(default_multiplexer& dm, native_socket sockfd)
: eventbf_(0), : eventbf_(0),
...@@ -1081,7 +1161,7 @@ acceptor::acceptor(default_multiplexer& backend_ref, native_socket sockfd) ...@@ -1081,7 +1161,7 @@ acceptor::acceptor(default_multiplexer& backend_ref, native_socket sockfd)
} }
void acceptor::start(acceptor_manager* mgr) { void acceptor::start(acceptor_manager* mgr) {
CAF_LOG_TRACE(CAF_ARG(fd())); CAF_LOG_TRACE(CAF_ARG2("fd", fd()));
CAF_ASSERT(mgr != nullptr); CAF_ASSERT(mgr != nullptr);
activate(mgr); activate(mgr);
} }
...@@ -1094,17 +1174,142 @@ void acceptor::activate(acceptor_manager* mgr) { ...@@ -1094,17 +1174,142 @@ void acceptor::activate(acceptor_manager* mgr) {
} }
void acceptor::stop_reading() { void acceptor::stop_reading() {
CAF_LOG_TRACE(CAF_ARG(fd())); CAF_LOG_TRACE(CAF_ARG2("fd", fd()));
close_read_channel(); close_read_channel();
passivate(); passivate();
} }
void acceptor::removed_from_loop(operation op) { void acceptor::removed_from_loop(operation op) {
CAF_LOG_TRACE(CAF_ARG(fd()) << CAF_ARG(op)); CAF_LOG_TRACE(CAF_ARG2("fd", fd()) << CAF_ARG(op));
if (op == operation::read) if (op == operation::read)
mgr_.reset(); mgr_.reset();
} }
datagram_handler::datagram_handler(default_multiplexer& backend_ref,
native_socket sockfd)
: event_handler(backend_ref, sockfd),
max_datagram_size_(receive_buffer_size),
rd_buf_(receive_buffer_size),
send_buffer_size_(0),
ack_writes_(false),
writing_(false) {
auto es = send_buffer_size(sockfd);
if (!es)
CAF_LOG_ERROR("cannot determine socket buffer size");
else
send_buffer_size_ = *es;
}
void datagram_handler::start(datagram_manager* mgr) {
CAF_LOG_TRACE(CAF_ARG2("fd", fd()));
CAF_ASSERT(mgr != nullptr);
activate(mgr);
}
void datagram_handler::activate(datagram_manager* mgr) {
if (!reader_) {
reader_.reset(mgr);
event_handler::activate();
prepare_next_read();
}
}
void datagram_handler::ack_writes(bool x) {
ack_writes_ = x;
}
void datagram_handler::write(datagram_handle hdl, const void* buf,
size_t num_bytes) {
wr_offline_buf_.emplace_back();
wr_offline_buf_.back().first = hdl;
auto cbuf = reinterpret_cast<const char*>(buf);
wr_offline_buf_.back().second.assign(cbuf,
cbuf + static_cast<ptrdiff_t>(num_bytes));
}
void datagram_handler::flush(const manager_ptr& mgr) {
CAF_ASSERT(mgr != nullptr);
CAF_LOG_TRACE(CAF_ARG(wr_offline_buf_.size()));
if (!wr_offline_buf_.empty() && !writing_) {
backend().add(operation::write, fd(), this);
writer_ = mgr;
writing_ = true;
prepare_next_write();
}
}
std::unordered_map<datagram_handle, ip_endpoint>& datagram_handler::endpoints() {
return ep_by_hdl_;
}
const std::unordered_map<datagram_handle, ip_endpoint>&
datagram_handler::endpoints() const {
return ep_by_hdl_;
}
void datagram_handler::add_endpoint(datagram_handle hdl, const ip_endpoint& ep,
const manager_ptr mgr) {
auto itr = hdl_by_ep_.find(ep);
if (itr == hdl_by_ep_.end()) {
hdl_by_ep_[ep] = hdl;
ep_by_hdl_[hdl] = ep;
writer_ = mgr;
} else if (!writer_) {
writer_ = mgr;
} else {
CAF_LOG_ERROR("cannot assign a second servant to the endpoint "
<< to_string(ep));
abort();
}
}
void datagram_handler::remove_endpoint(datagram_handle hdl) {
CAF_LOG_TRACE(CAF_ARG(hdl));
auto itr = ep_by_hdl_.find(hdl);
if (itr != ep_by_hdl_.end()) {
hdl_by_ep_.erase(itr->second);
ep_by_hdl_.erase(itr);
}
}
void datagram_handler::stop_reading() {
CAF_LOG_TRACE("");
close_read_channel();
passivate();
}
void datagram_handler::removed_from_loop(operation op) {
switch (op) {
case operation::read: reader_.reset(); break;
case operation::write: writer_.reset(); break;
case operation::propagate_error: break;
};
}
size_t datagram_handler::max_consecutive_reads() {
return backend().system().config().middleman_max_consecutive_reads;
}
void datagram_handler::prepare_next_read() {
CAF_LOG_TRACE(CAF_ARG(wr_buf_.second.size())
<< CAF_ARG(wr_offline_buf_.size()));
rd_buf_.resize(max_datagram_size_);
}
void datagram_handler::prepare_next_write() {
CAF_LOG_TRACE(CAF_ARG(wr_offline_buf_.size()));
wr_buf_.second.clear();
if (wr_offline_buf_.empty()) {
writing_ = false;
backend().del(operation::write, fd(), this);
} else {
wr_buf_.swap(wr_offline_buf_.front());
wr_offline_buf_.pop_front();
}
}
class socket_guard { class socket_guard {
public: public:
explicit socket_guard(native_socket fd) : fd_(fd) { explicit socket_guard(native_socket fd) : fd_(fd) {
...@@ -1245,12 +1450,29 @@ expected<void> set_inaddr_any(native_socket fd, sockaddr_in6& sa) { ...@@ -1245,12 +1450,29 @@ expected<void> set_inaddr_any(native_socket fd, sockaddr_in6& sa) {
return unit; return unit;
} }
template <int Family> expected<int> send_buffer_size(native_socket fd) {
int size;
socklen_t ret_size = sizeof(size);
CALL_CFUN(res, cc_zero, "getsockopt",
getsockopt(fd, SOL_SOCKET, SO_SNDBUF,
reinterpret_cast<getsockopt_ptr>(&size), &ret_size));
return size;
}
expected<void> send_buffer_size(native_socket fd, int new_value) {
CALL_CFUN(res, cc_zero, "setsockopt",
setsockopt(fd, SOL_SOCKET, SO_SNDBUF,
reinterpret_cast<setsockopt_ptr>(&new_value),
static_cast<socklen_t>(sizeof(int))));
return unit;
}
template <int Family, int SockType = SOCK_STREAM>
expected<native_socket> new_ip_acceptor_impl(uint16_t port, const char* addr, expected<native_socket> new_ip_acceptor_impl(uint16_t port, const char* addr,
bool reuse_addr, bool any) { bool reuse_addr, bool any) {
static_assert(Family == AF_INET || Family == AF_INET6, "invalid family"); static_assert(Family == AF_INET || Family == AF_INET6, "invalid family");
CAF_LOG_TRACE(CAF_ARG(port) << ", addr = " << (addr ? addr : "nullptr")); CAF_LOG_TRACE(CAF_ARG(port) << ", addr = " << (addr ? addr : "nullptr"));
CALL_CFUN(fd, cc_valid_socket, "socket", socket(Family, SOCK_STREAM, 0)); CALL_CFUN(fd, cc_valid_socket, "socket", socket(Family, SockType, 0));
// sguard closes the socket in case of exception // sguard closes the socket in case of exception
socket_guard sguard{fd}; socket_guard sguard{fd};
if (reuse_addr) { if (reuse_addr) {
...@@ -1315,6 +1537,57 @@ expected<native_socket> new_tcp_acceptor_impl(uint16_t port, const char* addr, ...@@ -1315,6 +1537,57 @@ expected<native_socket> new_tcp_acceptor_impl(uint16_t port, const char* addr,
return sguard.release(); return sguard.release();
} }
expected<std::pair<native_socket, ip_endpoint>>
new_remote_udp_endpoint_impl(const std::string& host, uint16_t port,
optional<protocol::network> preferred) {
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port) << CAF_ARG(preferred));
auto lep = new_local_udp_endpoint_impl(0, nullptr, false, preferred);
if (!lep)
return std::move(lep.error());
socket_guard sguard{(*lep).first};
std::pair<native_socket, ip_endpoint> info;
memset(std::get<1>(info).address(), 0, sizeof(sockaddr_storage));
if (!interfaces::get_endpoint(host, port, std::get<1>(info), (*lep).second))
return make_error(sec::cannot_connect_to_node, "no such host", host, port);
get<0>(info) = sguard.release();
return info;
}
expected<std::pair<native_socket, protocol::network>>
new_local_udp_endpoint_impl(uint16_t port, const char* addr, bool reuse,
optional<protocol::network> preferred) {
CAF_LOG_TRACE(CAF_ARG(port) << ", addr = " << (addr ? addr : "nullptr"));
auto addrs = interfaces::server_address(port, addr, preferred);
auto addr_str = std::string{addr == nullptr ? "" : addr};
if (addrs.empty())
return make_error(sec::cannot_open_port, "No local interface available",
addr_str);
bool any = addr_str.empty() || addr_str == "::" || addr_str == "0.0.0.0";
auto fd = invalid_native_socket;
protocol::network proto;
for (auto& elem : addrs) {
auto host = elem.first.c_str();
auto p = elem.second == ipv4
? new_ip_acceptor_impl<AF_INET, SOCK_DGRAM>(port, host, reuse, any)
: new_ip_acceptor_impl<AF_INET6, SOCK_DGRAM>(port, host, reuse, any);
if (!p) {
CAF_LOG_DEBUG(p.error());
continue;
}
fd = *p;
proto = elem.second;
break;
}
if (fd == invalid_native_socket) {
CAF_LOG_WARNING("could not open udp socket on:" << CAF_ARG(port)
<< CAF_ARG(addr_str));
return make_error(sec::cannot_open_port, "udp socket creation failed",
port, addr_str);
}
CAF_LOG_DEBUG(CAF_ARG(fd));
return std::make_pair(fd, proto);
}
expected<std::string> local_addr_of_fd(native_socket fd) { expected<std::string> local_addr_of_fd(native_socket fd) {
sockaddr_storage st; sockaddr_storage st;
socklen_t st_len = sizeof(st); socklen_t st_len = sizeof(st);
...@@ -1496,6 +1769,127 @@ void scribe_impl::remove_from_loop() { ...@@ -1496,6 +1769,127 @@ void scribe_impl::remove_from_loop() {
stream_.passivate(); stream_.passivate();
} }
datagram_servant_impl::datagram_servant_impl(default_multiplexer& mx,
native_socket sockfd, int64_t id)
: datagram_servant(datagram_handle::from_int(id)),
launched_(false),
handler_(mx, sockfd) {
// nop
}
bool datagram_servant_impl::new_endpoint(network::receive_buffer& buf) {
CAF_LOG_TRACE("");
if (detached())
// we are already disconnected from the broker while the multiplexer
// did not yet remove the socket, this can happen if an I/O event
// causes the broker to call close_all() while the pollset contained
// further activities for the broker
return false;
// A datagram that has a source port of zero is valid and never requires a
// reply. In the case of CAF we can simply drop it as nothing but the
// handshake could be communicated which we could not reply to.
// Source: TCP/IP Illustrated, Chapter 10.2
if (network::port(handler_.sending_endpoint()) == 0)
return true;
auto& dm = handler_.backend();
auto hdl = datagram_handle::from_int(dm.next_endpoint_id());
add_endpoint(handler_.sending_endpoint(), hdl);
parent()->add_hdl_for_datagram_servant(this, hdl);
return consume(&dm, hdl, buf);
}
void datagram_servant_impl::ack_writes(bool enable) {
CAF_LOG_TRACE(CAF_ARG(enable));
handler_.ack_writes(enable);
}
std::vector<char>& datagram_servant_impl::wr_buf(datagram_handle hdl) {
return handler_.wr_buf(hdl);
}
void datagram_servant_impl::enqueue_datagram(datagram_handle hdl,
std::vector<char> buffer) {
handler_.enqueue_datagram(hdl, std::move(buffer));
}
network::receive_buffer& datagram_servant_impl::rd_buf() {
return handler_.rd_buf();
}
void datagram_servant_impl::stop_reading() {
CAF_LOG_TRACE("");
handler_.stop_reading();
detach_handles();
detach(&handler_.backend(), false);
}
void datagram_servant_impl::flush() {
CAF_LOG_TRACE("");
handler_.flush(this);
}
std::string datagram_servant_impl::addr() const {
auto x = remote_addr_of_fd(handler_.fd());
if (!x)
return "";
return *x;
}
uint16_t datagram_servant_impl::port(datagram_handle hdl) const {
auto& eps = handler_.endpoints();
auto itr = eps.find(hdl);
if (itr == eps.end())
return 0;
return network::port(itr->second);
}
uint16_t datagram_servant_impl::local_port() const {
auto x = local_port_of_fd(handler_.fd());
if (!x)
return 0;
return *x;
}
std::vector<datagram_handle> datagram_servant_impl::hdls() const {
std::vector<datagram_handle> result;
result.reserve(handler_.endpoints().size());
for (auto& p : handler_.endpoints())
result.push_back(p.first);
return result;
}
void datagram_servant_impl::add_endpoint(const ip_endpoint& ep,
datagram_handle hdl) {
handler_.add_endpoint(hdl, ep, this);
}
void datagram_servant_impl::remove_endpoint(datagram_handle hdl) {
handler_.remove_endpoint(hdl);
}
void datagram_servant_impl::launch() {
CAF_LOG_TRACE("");
CAF_ASSERT(!launched_);
launched_ = true;
handler_.start(this);
}
void datagram_servant_impl::add_to_loop() {
handler_.activate(this);
}
void datagram_servant_impl::remove_from_loop() {
handler_.passivate();
}
void datagram_servant_impl::detach_handles() {
for (auto& p : handler_.endpoints()) {
if (p.first != hdl())
parent()->erase(p.first);
}
}
} // namespace network } // namespace network
} // namespace io } // namespace io
} // namespace caf } // namespace caf
...@@ -44,7 +44,8 @@ std::string to_string(const header &hdr) { ...@@ -44,7 +44,8 @@ std::string to_string(const header &hdr) {
<< to_string(hdr.source_node) << ", " << to_string(hdr.source_node) << ", "
<< to_string(hdr.dest_node) << ", " << to_string(hdr.dest_node) << ", "
<< hdr.source_actor << ", " << hdr.source_actor << ", "
<< hdr.dest_actor << hdr.dest_actor << ", "
<< hdr.sequence_number
<< "}"; << "}";
return oss.str(); return oss.str();
} }
...@@ -57,7 +58,8 @@ bool operator==(const header& lhs, const header& rhs) { ...@@ -57,7 +58,8 @@ bool operator==(const header& lhs, const header& rhs) {
&& lhs.source_node == rhs.source_node && lhs.source_node == rhs.source_node
&& lhs.dest_node == rhs.dest_node && lhs.dest_node == rhs.dest_node
&& lhs.source_actor == rhs.source_actor && lhs.source_actor == rhs.source_actor
&& lhs.dest_actor == rhs.dest_actor; && lhs.dest_actor == rhs.dest_actor
&& lhs.sequence_number == rhs.sequence_number;
} }
namespace { namespace {
...@@ -73,18 +75,15 @@ bool zero(T val) { ...@@ -73,18 +75,15 @@ bool zero(T val) {
bool server_handshake_valid(const header& hdr) { bool server_handshake_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& !valid(hdr.dest_node)
&& zero(hdr.dest_actor) && zero(hdr.dest_actor)
&& !zero(hdr.operation_data); && !zero(hdr.operation_data);
} }
bool client_handshake_valid(const header& hdr) { bool client_handshake_valid(const header& hdr) {
return valid(hdr.source_node) return valid(hdr.source_node)
&& valid(hdr.dest_node)
&& hdr.source_node != hdr.dest_node && hdr.source_node != hdr.dest_node
&& zero(hdr.source_actor) && zero(hdr.source_actor)
&& zero(hdr.dest_actor) && zero(hdr.dest_actor);
&& zero(hdr.operation_data);
} }
bool dispatch_message_valid(const header& hdr) { bool dispatch_message_valid(const header& hdr) {
......
...@@ -30,6 +30,20 @@ namespace caf { ...@@ -30,6 +30,20 @@ namespace caf {
namespace io { namespace io {
namespace basp { namespace basp {
namespace {
struct seq_num_visitor {
using result_type = uint16_t;
seq_num_visitor(instance::callee& c) : cal(c) { }
template <class T>
result_type operator()(const T& hdl) {
return cal.next_sequence_number(hdl);
}
instance::callee& cal;
};
} // namespace <anonymous>
instance::callee::callee(actor_system& sys, proxy_registry::backend& backend) instance::callee::callee(actor_system& sys, proxy_registry::backend& backend)
: namespace_(sys, backend) { : namespace_(sys, backend) {
// nop // nop
...@@ -85,13 +99,13 @@ connection_state instance::handle(execution_unit* ctx, ...@@ -85,13 +99,13 @@ connection_state instance::handle(execution_unit* ctx,
CAF_LOG_DEBUG("forward message"); CAF_LOG_DEBUG("forward message");
auto path = lookup(hdr.dest_node); auto path = lookup(hdr.dest_node);
if (path) { if (path) {
binary_serializer bs{ctx, path->wr_buf}; binary_serializer bs{ctx, callee_.get_buffer(path->hdl)};
auto e = bs(hdr); auto e = bs(hdr);
if (e) if (e)
return err(); return err();
if (payload != nullptr) if (payload != nullptr)
bs.apply_raw(payload->size(), payload->data()); bs.apply_raw(payload->size(), payload->data());
tbl_.flush(*path); flush(*path);
notify<hook::message_forwarded>(hdr, payload); notify<hook::message_forwarded>(hdr, payload);
} else { } else {
CAF_LOG_INFO("cannot forward message, no route to destination"); CAF_LOG_INFO("cannot forward message, no route to destination");
...@@ -110,156 +124,104 @@ connection_state instance::handle(execution_unit* ctx, ...@@ -110,156 +124,104 @@ connection_state instance::handle(execution_unit* ctx,
} }
return await_header; return await_header;
} }
// function object for checking payload validity if (!handle(ctx, dm.handle, hdr, payload, true, none, none))
auto payload_valid = [&]() -> bool {
return payload != nullptr && payload->size() == hdr.payload_len;
};
// handle message to ourselves
switch (hdr.operation) {
case message_type::server_handshake: {
actor_id aid = invalid_actor_id;
std::set<std::string> sigs;
if (payload_valid()) {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return err(); return err();
if (remote_appid != callee_.system().config().middleman_app_identifier) { return await_header;
CAF_LOG_ERROR("app identifier mismatch"); }
bool instance::handle(execution_unit* ctx, new_datagram_msg& dm,
endpoint_context& ep) {
using itr_t = network::receive_buffer::iterator;
// function object providing cleanup code on errors
auto err = [&]() -> bool {
auto cb = make_callback([&](const node_id& nid) -> error {
callee_.purge_state(nid);
return none;
});
tbl_.erase_direct(dm.handle, cb);
return false;
};
// extract payload
std::vector<char> pl_buf{std::move_iterator<itr_t>(std::begin(dm.buf) +
basp::header_size),
std::move_iterator<itr_t>(std::end(dm.buf))};
// resize header
dm.buf.resize(basp::header_size);
// extract header
binary_deserializer bd{ctx, dm.buf};
auto e = bd(ep.hdr);
if (e || !valid(ep.hdr)) {
CAF_LOG_WARNING("received invalid header:" << CAF_ARG(ep.hdr));
return err(); return err();
} }
e = bd(aid, sigs); CAF_LOG_DEBUG(CAF_ARG(ep.hdr));
if (e) std::vector<char>* payload = nullptr;
return err(); if (ep.hdr.payload_len > 0) {
} else { payload = &pl_buf;
CAF_LOG_ERROR("fail to receive the app identifier"); if (payload->size() != ep.hdr.payload_len) {
CAF_LOG_WARNING("received invalid payload");
return err(); return err();
} }
// close self connection after handshake is done
if (hdr.source_node == this_node_) {
CAF_LOG_INFO("close connection to self immediately");
callee_.finalize_handshake(hdr.source_node, aid, sigs);
return close_connection;
} }
// close this connection if we already have a direct connection // Handle FIFO ordering of datagrams
if (tbl_.lookup_direct(hdr.source_node) != invalid_connection_handle) { if (is_greater(ep.hdr.sequence_number, ep.seq_incoming)) {
CAF_LOG_INFO("close connection since we already have a " // Message arrived "early", add to pending messages
"direct connection: " << CAF_ARG(hdr.source_node)); auto s = ep.hdr.sequence_number;
callee_.finalize_handshake(hdr.source_node, aid, sigs); callee_.add_pending(s, ep, std::move(ep.hdr), std::move(pl_buf));
return close_connection; return true;
} else if (ep.hdr.sequence_number != ep.seq_incoming) {
// Message arrived late, drop it!
CAF_LOG_DEBUG("dropping message " << CAF_ARG(dm));
return true;
} }
// add direct route to this node and remove any indirect entry // Message arrived as expected
CAF_LOG_INFO("new direct connection:" << CAF_ARG(hdr.source_node)); ep.seq_incoming += 1;
tbl_.add_direct(dm.handle, hdr.source_node); // TODO: Add optional reliability here (send acks, ...)
auto was_indirect = tbl_.erase_indirect(hdr.source_node); if (!is_handshake(ep.hdr) && !is_heartbeat(ep.hdr)
// write handshake as client in response && ep.hdr.dest_node != this_node_) {
auto path = tbl_.lookup(hdr.source_node); CAF_LOG_DEBUG("forward message");
if (!path) { auto path = lookup(ep.hdr.dest_node);
CAF_LOG_ERROR("no route to host after server handshake"); if (path) {
binary_serializer bs{ctx, callee_.get_buffer(path->hdl)};
auto ex = bs(ep.hdr);
if (ex)
return err(); return err();
} if (payload != nullptr)
write_client_handshake(ctx, path->wr_buf, hdr.source_node); bs.apply_raw(payload->size(), payload->data());
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
callee_.finalize_handshake(hdr.source_node, aid, sigs);
flush(*path); flush(*path);
break; notify<hook::message_forwarded>(ep.hdr, payload);
}
case message_type::client_handshake: {
if (tbl_.lookup_direct(hdr.source_node) != invalid_connection_handle) {
CAF_LOG_INFO("received second client handshake:"
<< CAF_ARG(hdr.source_node));
break;
}
if (payload_valid()) {
binary_deserializer bd{ctx, *payload};
std::string remote_appid;
auto e = bd(remote_appid);
if (e)
return err();
if (remote_appid != callee_.system().config().middleman_app_identifier) {
CAF_LOG_ERROR("app identifier mismatch");
return err();
}
} else { } else {
CAF_LOG_ERROR("fail to receive the app identifier"); CAF_LOG_INFO("cannot forward message, no route to destination");
return err(); if (ep.hdr.source_node != this_node_) {
// TODO: signalize error back to sending node
auto reverse_path = lookup(ep.hdr.source_node);
if (!reverse_path) {
CAF_LOG_WARNING("cannot send error message: no route to source");
} else {
CAF_LOG_WARNING("not implemented yet: signalize forward failure");
} }
// add direct route to this node and remove any indirect entry } else {
CAF_LOG_INFO("new direct connection:" << CAF_ARG(hdr.source_node)); CAF_LOG_WARNING("lost packet with probably spoofed source");
tbl_.add_direct(dm.handle, hdr.source_node);
auto was_indirect = tbl_.erase_indirect(hdr.source_node);
callee_.learned_new_node_directly(hdr.source_node, was_indirect);
break;
} }
case message_type::dispatch_message: { notify<hook::message_forwarding_failed>(ep.hdr, payload);
if (!payload_valid())
return err();
// in case the sender of this message was received via a third node,
// we assume that that node to offers a route to the original source
auto last_hop = tbl_.lookup_direct(dm.handle);
if (hdr.source_node != none
&& hdr.source_node != this_node_
&& last_hop != hdr.source_node
&& tbl_.lookup_direct(hdr.source_node) == invalid_connection_handle
&& tbl_.add_indirect(last_hop, hdr.source_node))
callee_.learned_new_node_indirectly(hdr.source_node);
binary_deserializer bd{ctx, *payload};
auto receiver_name = static_cast<atom_value>(0);
std::vector<strong_actor_ptr> forwarding_stack;
message msg;
if (hdr.has(header::named_receiver_flag)) {
auto e = bd(receiver_name);
if (e)
return err();
} }
auto e = bd(forwarding_stack, msg); return true;
if (e)
return err();
CAF_LOG_DEBUG(CAF_ARG(forwarding_stack) << CAF_ARG(msg));
if (hdr.has(header::named_receiver_flag))
callee_.deliver(hdr.source_node, hdr.source_actor, receiver_name,
message_id::make(hdr.operation_data),
forwarding_stack, msg);
else
callee_.deliver(hdr.source_node, hdr.source_actor, hdr.dest_actor,
message_id::make(hdr.operation_data),
forwarding_stack, msg);
break;
} }
case message_type::announce_proxy: if (!handle(ctx, dm.handle, ep.hdr, payload, false, ep, ep.local_port))
callee_.proxy_announced(hdr.source_node, hdr.dest_actor);
break;
case message_type::kill_proxy: {
if (!payload_valid())
return err(); return err();
binary_deserializer bd{ctx, *payload}; // Look for pending messages
error fail_state; if (!callee_.deliver_pending(ctx, ep))
auto e = bd(fail_state);
if (e)
return err(); return err();
callee_.proxies().erase(hdr.source_node, hdr.source_actor, return true;
std::move(fail_state)); };
break;
}
case message_type::heartbeat: {
CAF_LOG_TRACE("received heartbeat: " << CAF_ARG(hdr.source_node));
callee_.handle_heartbeat(hdr.source_node);
break;
}
default:
CAF_LOG_ERROR("invalid operation");
return err();
}
return await_header;
}
void instance::handle_heartbeat(execution_unit* ctx) { void instance::handle_heartbeat(execution_unit* ctx) {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
for (auto& kvp: tbl_.direct_by_hdl_) { for (auto& kvp: tbl_.direct_by_hdl_) {
CAF_LOG_TRACE(CAF_ARG(kvp.first) << CAF_ARG(kvp.second)); CAF_LOG_TRACE(CAF_ARG(kvp.first) << CAF_ARG(kvp.second));
write_heartbeat(ctx, tbl_.parent_->wr_buf(kvp.first), kvp.second); write_heartbeat(ctx, callee_.get_buffer(kvp.first),
tbl_.parent_->flush(kvp.first); kvp.second, visit(seq_num_visitor{callee_}, kvp.first));
callee_.flush(kvp.first);
} }
} }
...@@ -268,15 +230,15 @@ optional<routing_table::route> instance::lookup(const node_id& target) { ...@@ -268,15 +230,15 @@ optional<routing_table::route> instance::lookup(const node_id& target) {
} }
void instance::flush(const routing_table::route& path) { void instance::flush(const routing_table::route& path) {
tbl_.flush(path); callee_.flush(path.hdl);
} }
void instance::write(execution_unit* ctx, const routing_table::route& r, void instance::write(execution_unit* ctx, const routing_table::route& r,
header& hdr, payload_writer* writer) { header& hdr, payload_writer* writer) {
CAF_LOG_TRACE(CAF_ARG(hdr)); CAF_LOG_TRACE(CAF_ARG(hdr));
CAF_ASSERT(hdr.payload_len == 0 || writer != nullptr); CAF_ASSERT(hdr.payload_len == 0 || writer != nullptr);
write(ctx, r.wr_buf, hdr, writer); write(ctx, callee_.get_buffer(r.hdl), hdr, writer);
tbl_.flush(r); flush(r);
} }
void instance::add_published_actor(uint16_t port, void instance::add_published_actor(uint16_t port,
...@@ -332,6 +294,13 @@ size_t instance::remove_published_actor(const actor_addr& whom, ...@@ -332,6 +294,13 @@ size_t instance::remove_published_actor(const actor_addr& whom,
return result; return result;
} }
bool instance::is_greater(sequence_type lhs, sequence_type rhs,
sequence_type max_distance) {
// distance between lhs and rhs is smaller than max_distance.
return ((lhs > rhs) && (lhs - rhs <= max_distance)) ||
((lhs < rhs) && (rhs - lhs > max_distance));
}
bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
const std::vector<strong_actor_ptr>& forwarding_stack, const std::vector<strong_actor_ptr>& forwarding_stack,
const strong_actor_ptr& receiver, message_id mid, const strong_actor_ptr& receiver, message_id mid,
...@@ -350,8 +319,9 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender, ...@@ -350,8 +319,9 @@ bool instance::dispatch(execution_unit* ctx, const strong_actor_ptr& sender,
}); });
header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(), header hdr{message_type::dispatch_message, 0, 0, mid.integer_value(),
sender ? sender->node() : this_node(), receiver->node(), sender ? sender->node() : this_node(), receiver->node(),
sender ? sender->id() : invalid_actor_id, receiver->id()}; sender ? sender->id() : invalid_actor_id, receiver->id(),
write(ctx, path->wr_buf, hdr, &writer); visit(seq_num_visitor{callee_}, path->hdl)};
write(ctx, callee_.get_buffer(path->hdl), hdr, &writer);
flush(*path); flush(*path);
notify<hook::message_sent>(sender, path->next_hop, receiver, mid, msg); notify<hook::message_sent>(sender, path->next_hop, receiver, mid, msg);
return true; return true;
...@@ -383,7 +353,8 @@ void instance::write(execution_unit* ctx, buffer_type& buf, ...@@ -383,7 +353,8 @@ void instance::write(execution_unit* ctx, buffer_type& buf,
void instance::write_server_handshake(execution_unit* ctx, void instance::write_server_handshake(execution_unit* ctx,
buffer_type& out_buf, buffer_type& out_buf,
optional<uint16_t> port) { optional<uint16_t> port,
uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(port));
using namespace detail; using namespace detail;
published_actor* pa = nullptr; published_actor* pa = nullptr;
...@@ -409,49 +380,66 @@ void instance::write_server_handshake(execution_unit* ctx, ...@@ -409,49 +380,66 @@ void instance::write_server_handshake(execution_unit* ctx,
header hdr{message_type::server_handshake, 0, 0, version, header hdr{message_type::server_handshake, 0, 0, version,
this_node_, none, this_node_, none,
(pa != nullptr) && pa->first ? pa->first->id() : invalid_actor_id, (pa != nullptr) && pa->first ? pa->first->id() : invalid_actor_id,
invalid_actor_id}; invalid_actor_id, sequence_number};
write(ctx, out_buf, hdr, &writer); write(ctx, out_buf, hdr, &writer);
} }
void instance::write_client_handshake(execution_unit* ctx, void instance::write_client_handshake(execution_unit* ctx,
buffer_type& buf, buffer_type& buf,
const node_id& remote_side) { const node_id& remote_side,
const node_id& this_node,
const std::string& app_identifier,
uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(remote_side)); CAF_LOG_TRACE(CAF_ARG(remote_side));
auto writer = make_callback([&](serializer& sink) -> error { auto writer = make_callback([&](serializer& sink) -> error {
auto& str = callee_.system().config().middleman_app_identifier; return sink(const_cast<std::string&>(app_identifier));
return sink(const_cast<std::string&>(str));
}); });
header hdr{message_type::client_handshake, 0, 0, 0, header hdr{message_type::client_handshake, 0, 0, 0,
this_node_, remote_side, invalid_actor_id, invalid_actor_id}; this_node, remote_side, invalid_actor_id, invalid_actor_id,
sequence_number};
write(ctx, buf, hdr, &writer); write(ctx, buf, hdr, &writer);
} }
void instance::write_client_handshake(execution_unit* ctx,
buffer_type& buf,
const node_id& remote_side,
uint16_t sequence_number) {
write_client_handshake(ctx, buf, remote_side, this_node_,
callee_.system().config().middleman_app_identifier,
sequence_number);
}
void instance::write_announce_proxy(execution_unit* ctx, buffer_type& buf, void instance::write_announce_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid) { const node_id& dest_node, actor_id aid,
uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(dest_node) << CAF_ARG(aid)); CAF_LOG_TRACE(CAF_ARG(dest_node) << CAF_ARG(aid));
header hdr{message_type::announce_proxy, 0, 0, 0, header hdr{message_type::announce_proxy, 0, 0, 0,
this_node_, dest_node, invalid_actor_id, aid}; this_node_, dest_node, invalid_actor_id, aid,
sequence_number};
write(ctx, buf, hdr); write(ctx, buf, hdr);
} }
void instance::write_kill_proxy(execution_unit* ctx, buffer_type& buf, void instance::write_kill_proxy(execution_unit* ctx, buffer_type& buf,
const node_id& dest_node, actor_id aid, const node_id& dest_node, actor_id aid,
const error& rsn) { const error& rsn, uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(dest_node) << CAF_ARG(aid) << CAF_ARG(rsn)); CAF_LOG_TRACE(CAF_ARG(dest_node) << CAF_ARG(aid) << CAF_ARG(rsn));
auto writer = make_callback([&](serializer& sink) -> error { auto writer = make_callback([&](serializer& sink) -> error {
return sink(const_cast<error&>(rsn)); return sink(const_cast<error&>(rsn));
}); });
header hdr{message_type::kill_proxy, 0, 0, 0, header hdr{message_type::kill_proxy, 0, 0, 0,
this_node_, dest_node, aid, invalid_actor_id}; this_node_, dest_node, aid, invalid_actor_id,
sequence_number};
write(ctx, buf, hdr, &writer); write(ctx, buf, hdr, &writer);
} }
void instance::write_heartbeat(execution_unit* ctx, void instance::write_heartbeat(execution_unit* ctx,
buffer_type& buf, buffer_type& buf,
const node_id& remote_side) { const node_id& remote_side,
uint16_t sequence_number) {
CAF_LOG_TRACE(CAF_ARG(remote_side)); CAF_LOG_TRACE(CAF_ARG(remote_side));
header hdr{message_type::heartbeat, 0, 0, 0, header hdr{message_type::heartbeat, 0, 0, 0,
this_node_, remote_side, invalid_actor_id, invalid_actor_id}; this_node_, remote_side, invalid_actor_id, invalid_actor_id,
sequence_number};
write(ctx, buf, hdr); write(ctx, buf, hdr);
} }
......
...@@ -50,6 +50,8 @@ ...@@ -50,6 +50,8 @@
#include "caf/detail/get_mac_addresses.hpp" #include "caf/detail/get_mac_addresses.hpp"
#include "caf/io/network/ip_endpoint.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
namespace network { namespace network {
...@@ -276,6 +278,35 @@ interfaces::server_address(uint16_t port, const char* host, ...@@ -276,6 +278,35 @@ interfaces::server_address(uint16_t port, const char* host,
return results; return results;
} }
bool interfaces::get_endpoint(const std::string& host, uint16_t port,
ip_endpoint& ep,
optional<protocol::network> preferred) {
static_assert(sizeof(uint16_t) == sizeof(unsigned short int),
"uint16_t cannot be printed with %hu in snprintf");
addrinfo hint;
// max port is 2^16 which needs 5 characters plus null terminator
char port_hint[6];
sprintf(port_hint, "%hu", port);
memset(&hint, 0, sizeof(hint));
hint.ai_socktype = SOCK_DGRAM;
if (preferred)
hint.ai_family = *preferred == protocol::network::ipv4 ? AF_INET : AF_INET6;
if (hint.ai_family == AF_INET6)
hint.ai_flags = AI_V4MAPPED;
addrinfo* tmp = nullptr;
if (getaddrinfo(host.c_str(), port_hint, &hint, &tmp) != 0)
return false;
std::unique_ptr<addrinfo, decltype(freeaddrinfo)*> addrs{tmp, freeaddrinfo};
for (auto i = addrs.get(); i != nullptr; i = i->ai_next) {
if (i->ai_family != AF_UNSPEC) {
memcpy(ep.address(), i->ai_addr, i->ai_addrlen);
*ep.length() = i->ai_addrlen;
return true;
}
}
return false;
}
} // namespace network } // namespace network
} // namespace io } // namespace io
} // namespace caf } // namespace caf
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/sec.hpp"
#include "caf/logger.hpp"
#ifdef CAF_WINDOWS
# include <windows.h>
# include <winsock2.h>
# include <ws2tcpip.h>
# include <ws2ipdef.h>
#else
# include <unistd.h>
# include <cerrno>
# include <arpa/inet.h>
# include <sys/socket.h>
# include <netinet/in.h>
# include <netinet/ip.h>
#endif
namespace {
template <class SizeType = size_t>
struct hash_conf {
template <class T = SizeType>
static constexpr caf::detail::enable_if_t<(sizeof(T) == 4), size_t> basis() {
return 2166136261u;
}
template <class T = SizeType>
static constexpr caf::detail::enable_if_t<(sizeof(T) == 4), size_t> prime() {
return 16777619u;
}
template <class T = SizeType>
static constexpr caf::detail::enable_if_t<(sizeof(T) == 8), size_t> basis() {
return 14695981039346656037u;
}
template <class T = SizeType>
static constexpr caf::detail::enable_if_t<(sizeof(T) == 8), size_t> prime() {
return 1099511628211u;
}
};
constexpr uint8_t static_bytes[] = {
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0xFF, 0xFF
};
constexpr size_t prehash(int i = 11) {
return (i > 0)
? (prehash(i - 1) * hash_conf<>::prime()) ^ static_bytes[i]
: (hash_conf<>::basis() * hash_conf<>::prime()) ^ static_bytes[i];
}
} // namespace <anonymous>
namespace caf {
namespace io {
namespace network {
struct ip_endpoint::impl {
sockaddr_storage addr;
size_t len;
};
ip_endpoint::ip_endpoint() : ptr_(new ip_endpoint::impl) {
// nop
}
ip_endpoint::ip_endpoint(const ip_endpoint& other) {
ptr_.reset(new ip_endpoint::impl);
memcpy(address(), other.caddress(), sizeof(sockaddr_storage));
*length() = *other.clength();
}
ip_endpoint& ip_endpoint::operator=(const ip_endpoint& other) {
ptr_.reset(new ip_endpoint::impl);
memcpy(address(), other.caddress(), sizeof(sockaddr_storage));
*length() = *other.clength();
return *this;
}
sockaddr* ip_endpoint::address() {
return reinterpret_cast<struct sockaddr*>(&ptr_->addr);
}
const sockaddr* ip_endpoint::caddress() const {
return reinterpret_cast<const struct sockaddr*>(&ptr_->addr);
}
size_t* ip_endpoint::length() {
return &ptr_->len;
}
const size_t* ip_endpoint::clength() const {
return &ptr_->len;
}
void ip_endpoint::clear() {
memset(&ptr_->addr, 0, sizeof(sockaddr_storage));
ptr_->len = 0;
}
void ip_endpoint::impl_deleter::operator()(ip_endpoint::impl *ptr) const {
delete ptr;
}
ep_hash::ep_hash() {
// nop
}
size_t ep_hash::operator()(const sockaddr& sa) const noexcept {
switch (sa.sa_family) {
case AF_INET:
return hash(reinterpret_cast<const struct sockaddr_in*>(&sa));
case AF_INET6:
return hash(reinterpret_cast<const struct sockaddr_in6*>(&sa));
default:
CAF_LOG_ERROR("Only IPv4 and IPv6 are supported.");
return 0;
}
}
size_t ep_hash::hash(const sockaddr_in* sa) const noexcept {
auto& addr = sa->sin_addr;
size_t res = prehash();
// the first loop was replaces with `constexpr size_t prehash()`
for (int i = 0; i < 4; ++i) {
res = res * hash_conf<>::prime();
res = res ^ ((addr.s_addr >> i) & 0xFF);
}
res = res * hash_conf<>::prime();
res = res ^ (sa->sin_port >> 1);
res = res * hash_conf<>::prime();
res = res ^ (sa->sin_port & 0xFF);
return res;
}
size_t ep_hash::hash(const sockaddr_in6* sa) const noexcept {
auto& addr = sa->sin6_addr;
size_t res = hash_conf<>::basis();
for (int i = 0; i < 16; ++i) {
res = res * hash_conf<>::prime();
res = res ^ addr.s6_addr[i];
}
res = res * hash_conf<>::prime();
res = res ^ (sa->sin6_port >> 1);
res = res * hash_conf<>::prime();
res = res ^ (sa->sin6_port & 0xFF);
return res;
}
bool operator==(const ip_endpoint& lhs, const ip_endpoint& rhs) {
auto same = false;
if (*lhs.clength() == *rhs.clength() &&
lhs.caddress()->sa_family == rhs.caddress()->sa_family) {
switch (lhs.caddress()->sa_family) {
case AF_INET: {
auto* la = reinterpret_cast<const sockaddr_in*>(lhs.caddress());
auto* ra = reinterpret_cast<const sockaddr_in*>(rhs.caddress());
same = (0 == memcmp(&la->sin_addr, &ra->sin_addr, sizeof(in_addr)))
&& (la->sin_port == ra->sin_port);
break;
}
case AF_INET6: {
auto* la = reinterpret_cast<const sockaddr_in6*>(lhs.caddress());
auto* ra = reinterpret_cast<const sockaddr_in6*>(rhs.caddress());
same = (0 == memcmp(&la->sin6_addr, &ra->sin6_addr, sizeof(in6_addr)))
&& (la->sin6_port == ra->sin6_port);
break;
}
default:
break;
}
}
return same;
}
std::string to_string(const ip_endpoint& ep) {
return host(ep) + ":" + std::to_string(port(ep));
}
std::string host(const ip_endpoint& ep) {
char addr[INET6_ADDRSTRLEN];
if (*ep.clength() == 0)
return "";
switch(ep.caddress()->sa_family) {
case AF_INET:
inet_ntop(AF_INET,
&reinterpret_cast<const sockaddr_in*>(ep.caddress())->sin_addr,
addr, static_cast<socklen_t>(*ep.clength()));
break;
case AF_INET6:
inet_ntop(AF_INET6,
&reinterpret_cast<const sockaddr_in6*>(ep.caddress())->sin6_addr,
addr, static_cast<socklen_t>(*ep.clength()));
break;
default:
addr[0] = '\0';
break;
}
return std::string(addr);
}
uint16_t port(const ip_endpoint& ep) {
uint16_t port = 0;
if (*ep.clength() == 0)
return 0;
switch(ep.caddress()->sa_family) {
case AF_INET:
port = ntohs(reinterpret_cast<const sockaddr_in*>(ep.caddress())->sin_port);
break;
case AF_INET6:
port = ntohs(reinterpret_cast<const sockaddr_in6*>(ep.caddress())->sin6_port);
break;
default:
// nop
break;
}
return port;
}
uint32_t family(const ip_endpoint& ep) {
if (*ep.clength() == 0)
return 0;
return ep.caddress()->sa_family;
}
error load_endpoint(ip_endpoint& ep, uint32_t& f, std::string& h,
uint16_t& p, size_t& l) {
ep.clear();
if (l > 0) {
*ep.length() = l;
switch(f) {
case AF_INET: {
auto* addr = reinterpret_cast<sockaddr_in*>(ep.address());
inet_pton(AF_INET, h.c_str(), &addr->sin_addr);
addr->sin_port = htons(p);
addr->sin_family = static_cast<sa_family_t>(f);
break;
}
case AF_INET6: {
auto* addr = reinterpret_cast<sockaddr_in6*>(ep.address());
inet_pton(AF_INET6, h.c_str(), &addr->sin6_addr);
addr->sin6_port = htons(p);
addr->sin6_family = static_cast<sa_family_t>(f);
break;
}
default:
return sec::invalid_argument;
}
}
return none;
}
error save_endpoint(ip_endpoint& ep, uint32_t& f, std::string& h,
uint16_t& p, size_t& l) {
if (*ep.length() > 0) {
f = family(ep);
h = host(ep);
p = port(ep);
l = *ep.length();
} else {
f = 0;
h = "";
p = 0;
l = 0;
}
return none;
}
} // namespace network
} // namespace io
} // namespace caf
...@@ -150,6 +150,21 @@ expected<uint16_t> middleman::publish(const strong_actor_ptr& whom, ...@@ -150,6 +150,21 @@ expected<uint16_t> middleman::publish(const strong_actor_ptr& whom,
return f(publish_atom::value, port, std::move(whom), std::move(sigs), in, ru); return f(publish_atom::value, port, std::move(whom), std::move(sigs), in, ru);
} }
expected<uint16_t> middleman::publish_udp(const strong_actor_ptr& whom,
std::set<std::string> sigs,
uint16_t port, const char* cstr,
bool ru) {
CAF_LOG_TRACE(CAF_ARG(whom) << CAF_ARG(sigs) << CAF_ARG(port));
if (!whom)
return sec::cannot_publish_invalid_actor;
std::string in;
if (cstr != nullptr)
in = cstr;
auto f = make_function_view(actor_handle());
return f(publish_udp_atom::value, port, std::move(whom),
std::move(sigs), in, ru);
}
expected<uint16_t> middleman::publish_local_groups(uint16_t port, expected<uint16_t> middleman::publish_local_groups(uint16_t port,
const char* in, bool reuse) { const char* in, bool reuse) {
CAF_LOG_TRACE(CAF_ARG(port) << CAF_ARG(in)); CAF_LOG_TRACE(CAF_ARG(port) << CAF_ARG(in));
...@@ -176,6 +191,12 @@ expected<void> middleman::unpublish(const actor_addr& whom, uint16_t port) { ...@@ -176,6 +191,12 @@ expected<void> middleman::unpublish(const actor_addr& whom, uint16_t port) {
return f(unpublish_atom::value, whom, port); return f(unpublish_atom::value, whom, port);
} }
expected<void> middleman::unpublish_udp(const actor_addr& whom, uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(whom) << CAF_ARG(port));
auto f = make_function_view(actor_handle());
return f(unpublish_udp_atom::value, whom, port);
}
expected<strong_actor_ptr> middleman::remote_actor(std::set<std::string> ifs, expected<strong_actor_ptr> middleman::remote_actor(std::set<std::string> ifs,
std::string host, std::string host,
uint16_t port) { uint16_t port) {
...@@ -193,6 +214,23 @@ expected<strong_actor_ptr> middleman::remote_actor(std::set<std::string> ifs, ...@@ -193,6 +214,23 @@ expected<strong_actor_ptr> middleman::remote_actor(std::set<std::string> ifs,
return ptr; return ptr;
} }
expected<strong_actor_ptr>
middleman::remote_actor_udp(std::set<std::string> ifs, std::string host,
uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(ifs) << CAF_ARG(host) << CAF_ARG(port));
auto f = make_function_view(actor_handle());
auto res = f(contact_atom::value, std::move(host), port);
if (!res)
return std::move(res.error());
strong_actor_ptr ptr = std::move(std::get<1>(*res));
if (!ptr)
return make_error(sec::no_actor_published_at_port, port);
if (!system().assignable(std::get<2>(*res), ifs))
return make_error(sec::unexpected_actor_messaging_interface, std::move(ifs),
std::move(std::get<2>(*res)));
return ptr;
}
expected<group> middleman::remote_group(const std::string& group_uri) { expected<group> middleman::remote_group(const std::string& group_uri) {
CAF_LOG_TRACE(CAF_ARG(group_uri)); CAF_LOG_TRACE(CAF_ARG(group_uri));
// format of group_identifier is group@host:port // format of group_identifier is group@host:port
...@@ -359,6 +397,7 @@ void middleman::init(actor_system_config& cfg) { ...@@ -359,6 +397,7 @@ void middleman::init(actor_system_config& cfg) {
// add I/O-related types to config // add I/O-related types to config
cfg.add_message_type<network::protocol>("@protocol") cfg.add_message_type<network::protocol>("@protocol")
.add_message_type<network::address_listing>("@address_listing") .add_message_type<network::address_listing>("@address_listing")
.add_message_type<network::receive_buffer>("@receive_buffer")
.add_message_type<new_data_msg>("@new_data_msg") .add_message_type<new_data_msg>("@new_data_msg")
.add_message_type<new_connection_msg>("@new_connection_msg") .add_message_type<new_connection_msg>("@new_connection_msg")
.add_message_type<acceptor_closed_msg>("@acceptor_closed_msg") .add_message_type<acceptor_closed_msg>("@acceptor_closed_msg")
......
...@@ -31,7 +31,6 @@ ...@@ -31,7 +31,6 @@
#include "caf/actor_proxy.hpp" #include "caf/actor_proxy.hpp"
#include "caf/actor_system_config.hpp" #include "caf/actor_system_config.hpp"
#include "caf/typed_event_based_actor.hpp" #include "caf/typed_event_based_actor.hpp"
#include "caf/typed_event_based_actor.hpp"
#include "caf/io/basp_broker.hpp" #include "caf/io/basp_broker.hpp"
#include "caf/io/system_messages.hpp" #include "caf/io/system_messages.hpp"
...@@ -47,11 +46,19 @@ middleman_actor_impl::middleman_actor_impl(actor_config& cfg, ...@@ -47,11 +46,19 @@ middleman_actor_impl::middleman_actor_impl(actor_config& cfg,
: middleman_actor::base(cfg), : middleman_actor::base(cfg),
broker_(std::move(default_broker)) { broker_(std::move(default_broker)) {
set_down_handler([=](down_msg& dm) { set_down_handler([=](down_msg& dm) {
auto i = cached_.begin(); auto i = cached_tcp_.begin();
auto e = cached_.end(); auto e = cached_tcp_.end();
while (i != e) {
if (get<1>(i->second) == dm.source)
i = cached_tcp_.erase(i);
else
++i;
}
i = cached_udp_.begin();
e = cached_udp_.end();
while (i != e) { while (i != e) {
if (get<1>(i->second) == dm.source) if (get<1>(i->second) == dm.source)
i = cached_.erase(i); i = cached_udp_.erase(i);
else else
++i; ++i;
} }
...@@ -77,37 +84,43 @@ auto middleman_actor_impl::make_behavior() -> behavior_type { ...@@ -77,37 +84,43 @@ auto middleman_actor_impl::make_behavior() -> behavior_type {
[=](publish_atom, uint16_t port, strong_actor_ptr& whom, mpi_set& sigs, [=](publish_atom, uint16_t port, strong_actor_ptr& whom, mpi_set& sigs,
std::string& addr, bool reuse) -> put_res { std::string& addr, bool reuse) -> put_res {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
if (!system().config().middleman_enable_tcp)
return make_error(sec::feature_disabled);
return put(port, whom, sigs, addr.c_str(), reuse); return put(port, whom, sigs, addr.c_str(), reuse);
}, },
[=](open_atom, uint16_t port, std::string& addr, bool reuse) -> put_res { [=](open_atom, uint16_t port, std::string& addr, bool reuse) -> put_res {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
if (!system().config().middleman_enable_tcp)
return make_error(sec::feature_disabled);
strong_actor_ptr whom; strong_actor_ptr whom;
mpi_set sigs; mpi_set sigs;
return put(port, whom, sigs, addr.c_str(), reuse); return put(port, whom, sigs, addr.c_str(), reuse);
}, },
[=](connect_atom, std::string& hostname, uint16_t port) -> get_res { [=](connect_atom, std::string& hostname, uint16_t port) -> get_res {
CAF_LOG_TRACE(CAF_ARG(hostname) << CAF_ARG(port)); CAF_LOG_TRACE(CAF_ARG(hostname) << CAF_ARG(port));
if (!system().config().middleman_enable_tcp)
return make_error(sec::feature_disabled);
auto rp = make_response_promise(); auto rp = make_response_promise();
endpoint key{std::move(hostname), port}; endpoint key{std::move(hostname), port};
// respond immediately if endpoint is cached // respond immediately if endpoint is cached
auto x = cached(key); auto x = cached_tcp(key);
if (x) { if (x) {
CAF_LOG_DEBUG("found cached entry" << CAF_ARG(*x)); CAF_LOG_DEBUG("found cached entry" << CAF_ARG(*x));
rp.deliver(get<0>(*x), get<1>(*x), get<2>(*x)); rp.deliver(get<0>(*x), get<1>(*x), get<2>(*x));
return {}; return get_delegated{};
} }
// attach this promise to a pending request if possible // attach this promise to a pending request if possible
auto rps = pending(key); auto rps = pending(key);
if (rps) { if (rps) {
CAF_LOG_DEBUG("attach to pending request"); CAF_LOG_DEBUG("attach to pending request");
rps->emplace_back(std::move(rp)); rps->emplace_back(std::move(rp));
return {}; return get_delegated{};
} }
// connect to endpoint and initiate handhsake etc. // connect to endpoint and initiate handhsake etc.
auto r = connect(key.first, port); auto r = connect(key.first, port);
if (!r) { if (!r) {
rp.deliver(std::move(r.error())); rp.deliver(std::move(r.error()));
return {}; return get_delegated{};
} }
auto& ptr = *r; auto& ptr = *r;
std::vector<response_promise> tmp{std::move(rp)}; std::vector<response_promise> tmp{std::move(rp)};
...@@ -120,7 +133,7 @@ auto middleman_actor_impl::make_behavior() -> behavior_type { ...@@ -120,7 +133,7 @@ auto middleman_actor_impl::make_behavior() -> behavior_type {
return; return;
if (nid && addr) { if (nid && addr) {
monitor(addr); monitor(addr);
cached_.emplace(key, std::make_tuple(nid, addr, sigs)); cached_tcp_.emplace(key, std::make_tuple(nid, addr, sigs));
} }
auto res = make_message(std::move(nid), std::move(addr), auto res = make_message(std::move(nid), std::move(addr),
std::move(sigs)); std::move(sigs));
...@@ -136,13 +149,80 @@ auto middleman_actor_impl::make_behavior() -> behavior_type { ...@@ -136,13 +149,80 @@ auto middleman_actor_impl::make_behavior() -> behavior_type {
promise.deliver(err); promise.deliver(err);
pending_.erase(i); pending_.erase(i);
}); });
return {}; return get_delegated{};
},
[=](publish_udp_atom, uint16_t port, strong_actor_ptr& whom,
mpi_set& sigs, std::string& addr, bool reuse) -> put_res {
CAF_LOG_TRACE("");
if (!system().config().middleman_enable_udp)
return make_error(sec::feature_disabled);
return put_udp(port, whom, sigs, addr.c_str(), reuse);
},
[=](contact_atom, std::string& hostname, uint16_t port) -> get_res {
CAF_LOG_TRACE(CAF_ARG(hostname) << CAF_ARG(port));
if (!system().config().middleman_enable_udp)
return make_error(sec::feature_disabled);
auto rp = make_response_promise();
endpoint key{std::move(hostname), port};
// respond immediately if endpoint is cached
auto x = cached_udp(key);
if (x) {
CAF_LOG_DEBUG("found cached entry" << CAF_ARG(*x));
rp.deliver(get<0>(*x), get<1>(*x), get<2>(*x));
return get_delegated{};
}
// attach this promise to a pending request if possible
auto rps = pending(key);
if (rps) {
CAF_LOG_DEBUG("attach to pending request");
rps->emplace_back(std::move(rp));
return get_delegated{};
}
// connect to endpoint and initiate handshake etc.
auto r = contact(key.first, port);
if (!r) {
rp.deliver(std::move(r.error()));
return get_delegated{};
}
auto& ptr = *r;
std::vector<response_promise> tmp{std::move(rp)};
pending_.emplace(key, std::move(tmp));
request(broker_, infinite, contact_atom::value, std::move(ptr), port)
.then(
[=](node_id& nid, strong_actor_ptr& addr, mpi_set& sigs) {
auto i = pending_.find(key);
if (i == pending_.end())
return;
if (nid && addr) {
monitor(addr);
cached_udp_.emplace(key, std::make_tuple(nid, addr, sigs));
}
auto res = make_message(std::move(nid), std::move(addr),
std::move(sigs));
for (auto& promise : i->second)
promise.deliver(res);
pending_.erase(i);
},
[=](error& err) {
auto i = pending_.find(key);
if (i == pending_.end())
return;
for (auto& promise : i->second)
promise.deliver(err);
pending_.erase(i);
});
return get_delegated{};
}, },
[=](unpublish_atom atm, actor_addr addr, uint16_t p) -> del_res { [=](unpublish_atom atm, actor_addr addr, uint16_t p) -> del_res {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
delegate(broker_, atm, std::move(addr), p); delegate(broker_, atm, std::move(addr), p);
return {}; return {};
}, },
[=](unpublish_udp_atom atm, actor_addr addr, uint16_t p) -> del_res {
CAF_LOG_TRACE("");
delegate(broker_, atm, std::move(addr), p);
return {};
},
[=](close_atom atm, uint16_t p) -> del_res { [=](close_atom atm, uint16_t p) -> del_res {
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
delegate(broker_, atm, p); delegate(broker_, atm, p);
...@@ -184,10 +264,37 @@ middleman_actor_impl::put(uint16_t port, strong_actor_ptr& whom, mpi_set& sigs, ...@@ -184,10 +264,37 @@ middleman_actor_impl::put(uint16_t port, strong_actor_ptr& whom, mpi_set& sigs,
return actual_port; return actual_port;
} }
middleman_actor_impl::put_res
middleman_actor_impl::put_udp(uint16_t port, strong_actor_ptr& whom,
mpi_set& sigs, const char* in, bool reuse_addr) {
CAF_LOG_TRACE(CAF_ARG(port) << CAF_ARG(whom) << CAF_ARG(sigs)
<< CAF_ARG(in) << CAF_ARG(reuse_addr));
uint16_t actual_port;
// treat empty strings like nullptr
if (in != nullptr && in[0] == '\0')
in = nullptr;
auto res = open_udp(port, in, reuse_addr);
if (!res)
return std::move(res.error());
auto& ptr = *res;
actual_port = ptr->local_port();
anon_send(broker_, publish_udp_atom::value, std::move(ptr), actual_port,
std::move(whom), std::move(sigs));
return actual_port;
}
optional<middleman_actor_impl::endpoint_data&>
middleman_actor_impl::cached_tcp(const endpoint& ep) {
auto i = cached_tcp_.find(ep);
if (i != cached_tcp_.end())
return i->second;
return none;
}
optional<middleman_actor_impl::endpoint_data&> optional<middleman_actor_impl::endpoint_data&>
middleman_actor_impl::cached(const endpoint& ep) { middleman_actor_impl::cached_udp(const endpoint& ep) {
auto i = cached_.find(ep); auto i = cached_udp_.find(ep);
if (i != cached_.end()) if (i != cached_udp_.end())
return i->second; return i->second;
return none; return none;
} }
...@@ -205,10 +312,21 @@ expected<scribe_ptr> middleman_actor_impl::connect(const std::string& host, ...@@ -205,10 +312,21 @@ expected<scribe_ptr> middleman_actor_impl::connect(const std::string& host,
return system().middleman().backend().new_tcp_scribe(host, port); return system().middleman().backend().new_tcp_scribe(host, port);
} }
expected<datagram_servant_ptr>
middleman_actor_impl::contact(const std::string& host, uint16_t port) {
return system().middleman().backend().new_remote_udp_endpoint(host, port);
}
expected<doorman_ptr> expected<doorman_ptr>
middleman_actor_impl::open(uint16_t port, const char* addr, bool reuse) { middleman_actor_impl::open(uint16_t port, const char* addr, bool reuse) {
return system().middleman().backend().new_tcp_doorman(port, addr, reuse); return system().middleman().backend().new_tcp_doorman(port, addr, reuse);
} }
expected<datagram_servant_ptr>
middleman_actor_impl::open_udp(uint16_t port, const char* addr, bool reuse) {
return system().middleman().backend().new_local_udp_endpoint(port, addr,
reuse);
}
} // namespace io } // namespace io
} // namespace caf } // namespace caf
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include <algorithm>
#include "caf/config.hpp"
#include "caf/io/network/receive_buffer.hpp"
namespace {
constexpr size_t min_size = 1;
} // namespace anonymous
namespace caf {
namespace io {
namespace network {
receive_buffer::receive_buffer()
: buffer_(nullptr),
capacity_(0),
size_(0) {
// nop
}
receive_buffer::receive_buffer(size_type size)
: buffer_(new value_type[size]),
capacity_(size),
size_(0) {
// nop
}
receive_buffer::receive_buffer(receive_buffer&& other) noexcept
: capacity_(std::move(other.capacity_)),
size_(std::move(other.size_)) {
buffer_ = std::move(other.buffer_);
other.size_ = 0;
other.capacity_ = 0;
other.buffer_.reset();
}
receive_buffer::receive_buffer(const receive_buffer& other)
: capacity_(other.capacity_),
size_(other.size_) {
if (other.size_ == 0) {
buffer_.reset();
} else {
buffer_.reset(new value_type[other.size_]);
std::copy(other.cbegin(), other.cend(), buffer_.get());
}
}
receive_buffer& receive_buffer::operator=(receive_buffer&& other) noexcept {
size_ = std::move(other.size_);
capacity_ = std::move(other.capacity_);
buffer_ = std::move(other.buffer_);
other.clear();
return *this;
}
receive_buffer& receive_buffer::operator=(const receive_buffer& other) {
size_ = other.size_;
capacity_ = other.capacity_;
if (other.size_ == 0) {
buffer_.reset();
} else {
buffer_.reset(new value_type[other.size_]);
std::copy(other.cbegin(), other.cend(), buffer_.get());
}
return *this;
}
void receive_buffer::resize(size_type new_size) {
if (new_size > capacity_)
increase_by(new_size - capacity_);
size_ = new_size;
}
void receive_buffer::reserve(size_type new_size) {
if (new_size > capacity_)
increase_by(new_size - capacity_);
}
void receive_buffer::shrink_to_fit() {
if (capacity_ > size_)
shrink_by(capacity_ - size_);
}
void receive_buffer::clear() {
size_ = 0;
buffer_ptr new_buffer{new value_type[capacity_]};
std::swap(buffer_, new_buffer);
}
void receive_buffer::swap(receive_buffer& other) noexcept {
std::swap(capacity_, other.capacity_);
std::swap(size_, other.size_);
std::swap(buffer_, other.buffer_);
}
void receive_buffer::push_back(value_type value) {
if (size_ == capacity_)
increase_by(std::max(capacity_, min_size));
buffer_.get()[size_] = value;
++size_;
}
void receive_buffer::increase_by(size_t bytes) {
if (bytes == 0)
return;
if (!buffer_) {
buffer_.reset(new value_type[bytes]);
} else {
buffer_ptr new_buffer{new value_type[capacity_ + bytes]};
std::copy(begin(), end(), new_buffer.get());
std::swap(buffer_, new_buffer);
}
capacity_ += bytes;
}
void receive_buffer::shrink_by(size_t bytes) {
CAF_ASSERT(bytes <= capacity_);
size_t new_size = capacity_ - bytes;
if (new_size == 0) {
buffer_.reset();
} else {
buffer_ptr new_buffer{new value_type[new_size]};
std::copy(begin(), begin() + new_size, new_buffer.get());
std::swap(buffer_, new_buffer);
}
capacity_ = new_size;
}
receive_buffer::iterator receive_buffer::insert(iterator pos, value_type value) {
if (size_ == capacity_) {
auto dist = (pos == nullptr) ? 0 : std::distance(begin(), pos);
increase_by(std::max(capacity_, min_size));
pos = begin() + dist;
}
std::copy(pos, end(), pos + 1);
*pos = value;
++size_;
return pos;
}
} // namepsace network
} // namespace io
} // namespace caf
...@@ -25,7 +25,8 @@ namespace caf { ...@@ -25,7 +25,8 @@ namespace caf {
namespace io { namespace io {
namespace basp { namespace basp {
routing_table::routing_table(abstract_broker* parent) : parent_(parent) { routing_table::routing_table(abstract_broker* parent)
: parent_(parent) {
// nop // nop
} }
...@@ -35,8 +36,8 @@ routing_table::~routing_table() { ...@@ -35,8 +36,8 @@ routing_table::~routing_table() {
optional<routing_table::route> routing_table::lookup(const node_id& target) { optional<routing_table::route> routing_table::lookup(const node_id& target) {
auto hdl = lookup_direct(target); auto hdl = lookup_direct(target);
if (hdl != invalid_connection_handle) if (hdl)
return route{parent_->wr_buf(hdl), target, hdl}; return route{target, *hdl};
// pick first available indirect route // pick first available indirect route
auto i = indirect_.find(target); auto i = indirect_.find(target);
if (i != indirect_.end()) { if (i != indirect_.end()) {
...@@ -44,24 +45,24 @@ optional<routing_table::route> routing_table::lookup(const node_id& target) { ...@@ -44,24 +45,24 @@ optional<routing_table::route> routing_table::lookup(const node_id& target) {
while (!hops.empty()) { while (!hops.empty()) {
auto& hop = *hops.begin(); auto& hop = *hops.begin();
hdl = lookup_direct(hop); hdl = lookup_direct(hop);
if (hdl != invalid_connection_handle) if (hdl)
return route{parent_->wr_buf(hdl), hop, hdl}; return route{hop, *hdl};
hops.erase(hops.begin()); hops.erase(hops.begin());
} }
} }
return none; return none;
} }
void routing_table::flush(const route& r) { node_id routing_table::lookup_direct(const endpoint_handle& hdl) const {
parent_->flush(r.hdl);
}
node_id routing_table::lookup_direct(const connection_handle& hdl) const {
return get_opt(direct_by_hdl_, hdl, none); return get_opt(direct_by_hdl_, hdl, none);
} }
connection_handle routing_table::lookup_direct(const node_id& nid) const { optional<routing_table::endpoint_handle>
return get_opt(direct_by_nid_, nid, invalid_connection_handle); routing_table::lookup_direct(const node_id& nid) const {
auto i = direct_by_nid_.find(nid);
if (i != direct_by_nid_.end())
return i->second;
return none;
} }
node_id routing_table::lookup_indirect(const node_id& nid) const { node_id routing_table::lookup_indirect(const node_id& nid) const {
...@@ -83,7 +84,7 @@ void routing_table::blacklist(const node_id& hop, const node_id& dest) { ...@@ -83,7 +84,7 @@ void routing_table::blacklist(const node_id& hop, const node_id& dest) {
indirect_.erase(i); indirect_.erase(i);
} }
void routing_table::erase_direct(const connection_handle& hdl, void routing_table::erase_direct(const endpoint_handle& hdl,
erase_callback& cb) { erase_callback& cb) {
auto i = direct_by_hdl_.find(hdl); auto i = direct_by_hdl_.find(hdl);
if (i == direct_by_hdl_.end()) if (i == direct_by_hdl_.end())
...@@ -91,7 +92,7 @@ void routing_table::erase_direct(const connection_handle& hdl, ...@@ -91,7 +92,7 @@ void routing_table::erase_direct(const connection_handle& hdl,
cb(i->second); cb(i->second);
parent_->parent().notify<hook::connection_lost>(i->second); parent_->parent().notify<hook::connection_lost>(i->second);
direct_by_nid_.erase(i->second); direct_by_nid_.erase(i->second);
direct_by_hdl_.erase(i); direct_by_hdl_.erase(i->first);
} }
bool routing_table::erase_indirect(const node_id& dest) { bool routing_table::erase_indirect(const node_id& dest) {
...@@ -105,7 +106,7 @@ bool routing_table::erase_indirect(const node_id& dest) { ...@@ -105,7 +106,7 @@ bool routing_table::erase_indirect(const node_id& dest) {
return true; return true;
} }
void routing_table::add_direct(const connection_handle& hdl, void routing_table::add_direct(const endpoint_handle& hdl,
const node_id& nid) { const node_id& nid) {
CAF_ASSERT(direct_by_hdl_.count(hdl) == 0); CAF_ASSERT(direct_by_hdl_.count(hdl) == 0);
CAF_ASSERT(direct_by_nid_.count(nid) == 0); CAF_ASSERT(direct_by_nid_.count(nid) == 0);
...@@ -143,8 +144,8 @@ size_t routing_table::erase(const node_id& dest, erase_callback& cb) { ...@@ -143,8 +144,8 @@ size_t routing_table::erase(const node_id& dest, erase_callback& cb) {
indirect_.erase(i); indirect_.erase(i);
} }
auto hdl = lookup_direct(dest); auto hdl = lookup_direct(dest);
if (hdl != invalid_connection_handle) { if (hdl) {
direct_by_hdl_.erase(hdl); direct_by_hdl_.erase(*hdl);
direct_by_nid_.erase(dest); direct_by_nid_.erase(dest);
parent_->parent().notify<hook::connection_lost>(dest); parent_->parent().notify<hook::connection_lost>(dest);
++res; ++res;
......
...@@ -23,11 +23,18 @@ ...@@ -23,11 +23,18 @@
#include "caf/io/scribe.hpp" #include "caf/io/scribe.hpp"
#include "caf/io/doorman.hpp" #include "caf/io/doorman.hpp"
#include "caf/io/datagram_servant.hpp"
namespace caf { namespace caf {
namespace io { namespace io {
namespace network { namespace network {
namespace {
constexpr size_t receive_buffer_size = std::numeric_limits<uint16_t>::max();
} // namespace anonymous
test_multiplexer::scribe_data::scribe_data(shared_buffer_type input, test_multiplexer::scribe_data::scribe_data(shared_buffer_type input,
shared_buffer_type output) shared_buffer_type output)
: vn_buf_ptr(std::move(input)), : vn_buf_ptr(std::move(input)),
...@@ -47,10 +54,28 @@ test_multiplexer::doorman_data::doorman_data() ...@@ -47,10 +54,28 @@ test_multiplexer::doorman_data::doorman_data()
// nop // nop
} }
test_multiplexer::datagram_data::
datagram_data(shared_job_queue_type input,
shared_job_queue_type output)
: vn_buf_ptr(std::move(input)),
wr_buf_ptr(std::move(output)),
vn_buf(*vn_buf_ptr),
wr_buf(*wr_buf_ptr),
rd_buf(datagram_handle::from_int(0), receive_buffer_size),
stopped_reading(false),
passive_mode(false),
ack_writes(false),
port(0),
local_port(0),
datagram_size(receive_buffer_size) {
// nop
}
test_multiplexer::test_multiplexer(actor_system* sys) test_multiplexer::test_multiplexer(actor_system* sys)
: multiplexer(sys), : multiplexer(sys),
tid_(std::this_thread::get_id()), tid_(std::this_thread::get_id()),
inline_runnables_(0) { inline_runnables_(0),
servant_ids_(0) {
CAF_ASSERT(sys != nullptr); CAF_ASSERT(sys != nullptr);
} }
...@@ -62,9 +87,7 @@ test_multiplexer::~test_multiplexer() { ...@@ -62,9 +87,7 @@ test_multiplexer::~test_multiplexer() {
scribe_ptr test_multiplexer::new_scribe(native_socket) { scribe_ptr test_multiplexer::new_scribe(native_socket) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
std::cerr << "test_multiplexer::add_tcp_scribe called with native socket" CAF_CRITICAL("test_multiplexer::add_tcp_scribe called with native socket");
<< std::endl;
abort();
} }
scribe_ptr test_multiplexer::new_scribe(connection_handle hdl) { scribe_ptr test_multiplexer::new_scribe(connection_handle hdl) {
...@@ -137,9 +160,7 @@ expected<scribe_ptr> test_multiplexer::new_tcp_scribe(const std::string& host, ...@@ -137,9 +160,7 @@ expected<scribe_ptr> test_multiplexer::new_tcp_scribe(const std::string& host,
doorman_ptr test_multiplexer::new_doorman(native_socket) { doorman_ptr test_multiplexer::new_doorman(native_socket) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
std::cerr << "test_multiplexer::add_tcp_doorman called with native socket" CAF_CRITICAL("test_multiplexer::add_tcp_doorman called with native socket");
<< std::endl;
abort();
} }
doorman_ptr test_multiplexer::new_doorman(accept_handle hdl, uint16_t port) { doorman_ptr test_multiplexer::new_doorman(accept_handle hdl, uint16_t port) {
...@@ -230,12 +251,210 @@ expected<doorman_ptr> test_multiplexer::new_tcp_doorman(uint16_t desired_port, ...@@ -230,12 +251,210 @@ expected<doorman_ptr> test_multiplexer::new_tcp_doorman(uint16_t desired_port,
return new_doorman(hdl, port); return new_doorman(hdl, port);
} }
datagram_servant_ptr test_multiplexer::new_datagram_servant(native_socket) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_CRITICAL("test_multiplexer::new_datagram_servant called with native socket");
}
datagram_servant_ptr
test_multiplexer::new_datagram_servant_for_endpoint(native_socket,
const ip_endpoint&) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_CRITICAL("test_multiplexer::new_datagram_servant_for_endpoint called with "
"native socket");
}
expected<datagram_servant_ptr>
test_multiplexer::new_remote_udp_endpoint(const std::string& host,
uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(port));
datagram_handle hdl;
{ // lifetime scope of guard
guard_type guard{mx_};
auto i = remote_endpoints_.find(std::make_pair(host, port));
if (i != remote_endpoints_.end()) {
hdl = i->second;
remote_endpoints_.erase(i);
} else {
return sec::cannot_connect_to_node;
}
}
auto ptr = new_datagram_servant(hdl, port);
// Set state in the struct to enable direct communication?
{ // lifetime scope of guard
guard_type guard{mx_};
auto data = data_for_hdl(hdl);
data->servants.emplace(hdl);
local_port(hdl) = data->local_port;
}
return ptr;
}
expected<datagram_servant_ptr>
test_multiplexer::new_local_udp_endpoint(uint16_t desired_port,
const char*, bool) {
CAF_LOG_TRACE(CAF_ARG(desired_port));
datagram_handle hdl;
uint16_t port = 0;
{ // Lifetime scope of guard.
guard_type guard{mx_};
if (desired_port == 0) {
// Start with largest possible port and reverse iterate until we find a
// port that's not assigned to a known doorman.
port = std::numeric_limits<uint16_t>::max();
while (is_known_port(port))
--port;
// Do the same for finding a local dgram handle
auto y = std::numeric_limits<int64_t>::max();
while (is_known_handle(datagram_handle::from_int(y)))
--y;
hdl = datagram_handle::from_int(y);
} else {
auto i = local_endpoints_.find(desired_port);
if (i != local_endpoints_.end()) {
hdl = i->second;
local_endpoints_.erase(i);
port = desired_port;
} else {
return sec::cannot_open_port;
}
}
}
return new_datagram_servant(hdl, port);
}
datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle hdl,
uint16_t port) {
CAF_LOG_TRACE(CAF_ARG(hdl));
class impl : public datagram_servant {
public:
impl(datagram_handle dh, test_multiplexer* mpx)
: datagram_servant(dh), mpx_(mpx) {
// nop
}
bool new_endpoint(network::receive_buffer& buf) override {
datagram_handle dhdl;
{ // Try to get a connection handle of a pending connect.
guard_type guard{mpx_->mx_};
auto& pe = mpx_->pending_endpoints();
auto i = pe.find(hdl().id());
if (i == pe.end())
return false;
dhdl = i->second;
pe.erase(i);
}
auto data = mpx_->data_for_hdl(hdl());
data->servants.emplace(dhdl);
mpx_->datagram_data_.emplace(dhdl, data);
parent()->add_hdl_for_datagram_servant(this, dhdl);
return consume(mpx_, dhdl, buf);
}
void ack_writes(bool enable) override {
mpx_->ack_writes(hdl()) = enable;
}
std::vector<char>& wr_buf(datagram_handle dh) override {
auto& buf = mpx_->output_buffer(dh);
buf.first = dh;
return buf.second;
}
void enqueue_datagram(datagram_handle dh,
std::vector<char> buf) override {
auto& q = mpx_->output_queue(dh);
q.emplace_back(dh, std::move(buf));
}
network::receive_buffer& rd_buf() override {
auto& buf = mpx_->input_buffer(hdl());
return buf.second;
}
void stop_reading() override {
mpx_->stopped_reading(hdl()) = true;
detach_handles();
detach(mpx_, false);
}
void launch() override {
// nop
}
void flush() override {
// nop
}
std::string addr() const override {
return "test";
}
uint16_t port(datagram_handle dh) const override {
return static_cast<uint16_t>(dh.id());
}
uint16_t local_port() const override {
guard_type guard{mpx_->mx_};
return mpx_->local_port(hdl());
}
std::vector<datagram_handle> hdls() const override {
auto data = mpx_->data_for_hdl(hdl());
std::vector<datagram_handle> result(data->servants.begin(),
data->servants.end());
return result;
}
void add_to_loop() override {
mpx_->passive_mode(hdl()) = false;
}
void remove_from_loop() override {
mpx_->passive_mode(hdl()) = true;
}
void add_endpoint(const ip_endpoint&, datagram_handle) override {
CAF_CRITICAL("datagram_servant impl::add_endpoint called with ip_endpoint");
}
void remove_endpoint(datagram_handle dh) override {
auto data = mpx_->data_for_hdl(hdl());
{ // lifetime scope of guard
guard_type guard{mpx_->mx_};
auto itr = std::find(data->servants.begin(), data->servants.end(), dh);
if (itr != data->servants.end())
data->servants.erase(itr);
}
}
void detach_handles() override {
auto data = mpx_->data_for_hdl(hdl());
for (auto& p : data->servants)
if (p != hdl())
parent()->erase(p);
data->servants.clear();
data->servants.emplace(hdl());
}
private:
test_multiplexer* mpx_;
};
auto dptr = make_counted<impl>(hdl, this);
CAF_LOG_INFO("new datagram servant" << hdl);
auto data = data_for_hdl(hdl);
{ // lifetime scope of guard
guard_type guard{mx_};
data->ptr = dptr;
data->port = port;
data->servants.emplace(hdl);
}
return dptr;
}
datagram_servant_ptr test_multiplexer::new_datagram_servant(datagram_handle,
const std::string&,
uint16_t) {
CAF_CRITICAL("This has no implementation in the test multiplexer");
}
int64_t test_multiplexer::next_endpoint_id() {
return servant_ids_++;
}
bool test_multiplexer::is_known_port(uint16_t x) const { bool test_multiplexer::is_known_port(uint16_t x) const {
auto pred = [&](const doorman_data_map::value_type& y) { auto pred1 = [&](const doorman_data_map::value_type& y) {
return x == y.second.port; return x == y.second.port;
}; };
return doormen_.count(x) > 0 auto pred2 = [&](const datagram_data_map::value_type& y) {
|| std::any_of(doorman_data_.begin(), doorman_data_.end(), pred); return x == y.second->port;
};
return (doormen_.count(x) + local_endpoints_.count(x)) > 0
|| std::any_of(doorman_data_.begin(), doorman_data_.end(), pred1)
|| std::any_of(datagram_data_.begin(), datagram_data_.end(), pred2);
} }
bool test_multiplexer::is_known_handle(accept_handle x) const { bool test_multiplexer::is_known_handle(accept_handle x) const {
...@@ -246,6 +465,18 @@ bool test_multiplexer::is_known_handle(accept_handle x) const { ...@@ -246,6 +465,18 @@ bool test_multiplexer::is_known_handle(accept_handle x) const {
|| std::any_of(doormen_.begin(), doormen_.end(), pred); || std::any_of(doormen_.begin(), doormen_.end(), pred);
} }
bool test_multiplexer::is_known_handle(datagram_handle x) const {
auto pred1 = [&](const pending_local_datagram_endpoints_map::value_type& y) {
return x == y.second;
};
auto pred2 = [&](const pending_remote_datagram_endpoints_map::value_type& y) {
return x == y.second;
};
return datagram_data_.count(x) > 0
|| std::any_of(local_endpoints_.begin(), local_endpoints_.end(), pred1)
|| std::any_of(remote_endpoints_.begin(), remote_endpoints_.end(), pred2);
}
auto test_multiplexer::make_supervisor() -> supervisor_ptr { auto test_multiplexer::make_supervisor() -> supervisor_ptr {
// not needed // not needed
return nullptr; return nullptr;
...@@ -278,6 +509,25 @@ void test_multiplexer::provide_acceptor(uint16_t desired_port, ...@@ -278,6 +509,25 @@ void test_multiplexer::provide_acceptor(uint16_t desired_port,
doorman_data_[hdl].port = desired_port; doorman_data_[hdl].port = desired_port;
} }
void test_multiplexer::provide_datagram_servant(uint16_t desired_port,
datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(desired_port) << CAF_ARG(hdl));
guard_type guard{mx_};
local_endpoints_.emplace(desired_port, hdl);
auto data = data_for_hdl(hdl);
data->local_port = desired_port;
}
void test_multiplexer::provide_datagram_servant(std::string host,
uint16_t desired_port,
datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(host) << CAF_ARG(desired_port) << CAF_ARG(hdl));
guard_type guard{mx_};
remote_endpoints_.emplace(std::make_pair(std::move(host), desired_port), hdl);
}
/// The external input buffer should be filled by /// The external input buffer should be filled by
/// the test program. /// the test program.
test_multiplexer::buffer_type& test_multiplexer::buffer_type&
...@@ -286,6 +536,12 @@ test_multiplexer::virtual_network_buffer(connection_handle hdl) { ...@@ -286,6 +536,12 @@ test_multiplexer::virtual_network_buffer(connection_handle hdl) {
return scribe_data_[hdl].vn_buf; return scribe_data_[hdl].vn_buf;
} }
test_multiplexer::write_job_queue_type&
test_multiplexer::virtual_network_buffer(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->vn_buf;
}
test_multiplexer::buffer_type& test_multiplexer::buffer_type&
test_multiplexer::output_buffer(connection_handle hdl) { test_multiplexer::output_buffer(connection_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
...@@ -298,6 +554,26 @@ test_multiplexer::input_buffer(connection_handle hdl) { ...@@ -298,6 +554,26 @@ test_multiplexer::input_buffer(connection_handle hdl) {
return scribe_data_[hdl].rd_buf; return scribe_data_[hdl].rd_buf;
} }
test_multiplexer::write_job_type&
test_multiplexer::output_buffer(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
auto& buf = data_for_hdl(hdl)->wr_buf;
buf.emplace_back();
return buf.back();
}
test_multiplexer::write_job_queue_type&
test_multiplexer::output_queue(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->wr_buf;
}
test_multiplexer::read_job_type&
test_multiplexer::input_buffer(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->rd_buf;
}
receive_policy::config& test_multiplexer::read_config(connection_handle hdl) { receive_policy::config& test_multiplexer::read_config(connection_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return scribe_data_[hdl].recv_conf; return scribe_data_[hdl].recv_conf;
...@@ -308,16 +584,31 @@ bool& test_multiplexer::ack_writes(connection_handle hdl) { ...@@ -308,16 +584,31 @@ bool& test_multiplexer::ack_writes(connection_handle hdl) {
return scribe_data_[hdl].ack_writes; return scribe_data_[hdl].ack_writes;
} }
bool& test_multiplexer::ack_writes(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->ack_writes;
}
bool& test_multiplexer::stopped_reading(connection_handle hdl) { bool& test_multiplexer::stopped_reading(connection_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return scribe_data_[hdl].stopped_reading; return scribe_data_[hdl].stopped_reading;
} }
bool& test_multiplexer::stopped_reading(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->stopped_reading;
}
bool& test_multiplexer::passive_mode(connection_handle hdl) { bool& test_multiplexer::passive_mode(connection_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return scribe_data_[hdl].passive_mode; return scribe_data_[hdl].passive_mode;
} }
bool& test_multiplexer::passive_mode(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return data_for_hdl(hdl)->passive_mode;
}
scribe_ptr& test_multiplexer::impl_ptr(connection_handle hdl) { scribe_ptr& test_multiplexer::impl_ptr(connection_handle hdl) {
return scribe_data_[hdl].ptr; return scribe_data_[hdl].ptr;
} }
...@@ -326,6 +617,22 @@ uint16_t& test_multiplexer::port(accept_handle hdl) { ...@@ -326,6 +617,22 @@ uint16_t& test_multiplexer::port(accept_handle hdl) {
return doorman_data_[hdl].port; return doorman_data_[hdl].port;
} }
uint16_t& test_multiplexer::port(datagram_handle hdl) {
return data_for_hdl(hdl)->port;
}
uint16_t& test_multiplexer::local_port(datagram_handle hdl) {
return data_for_hdl(hdl)->local_port;
}
datagram_servant_ptr& test_multiplexer::impl_ptr(datagram_handle hdl) {
return data_for_hdl(hdl)->ptr;
}
std::set<datagram_handle>& test_multiplexer::servants(datagram_handle hdl) {
return data_for_hdl(hdl)->servants;
}
bool& test_multiplexer::stopped_reading(accept_handle hdl) { bool& test_multiplexer::stopped_reading(accept_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return doorman_data_[hdl].stopped_reading; return doorman_data_[hdl].stopped_reading;
...@@ -346,6 +653,16 @@ void test_multiplexer::add_pending_connect(accept_handle src, ...@@ -346,6 +653,16 @@ void test_multiplexer::add_pending_connect(accept_handle src,
pending_connects_.emplace(src, hdl); pending_connects_.emplace(src, hdl);
} }
std::shared_ptr<test_multiplexer::datagram_data>
test_multiplexer::data_for_hdl(datagram_handle hdl) {
auto itr = datagram_data_.find(hdl);
if (itr != datagram_data_.end())
return itr->second;
// if it does not exist, create a new entry
datagram_data_.emplace(hdl, std::make_shared<datagram_data>());
return datagram_data_[hdl];
}
void test_multiplexer::prepare_connection(accept_handle src, void test_multiplexer::prepare_connection(accept_handle src,
connection_handle hdl, connection_handle hdl,
test_multiplexer& peer, test_multiplexer& peer,
...@@ -374,17 +691,35 @@ void test_multiplexer::prepare_connection(accept_handle src, ...@@ -374,17 +691,35 @@ void test_multiplexer::prepare_connection(accept_handle src,
peer.provide_scribe(std::move(host), port, peer_hdl); peer.provide_scribe(std::move(host), port, peer_hdl);
} }
void test_multiplexer::add_pending_endpoint(datagram_handle src,
datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
pending_endpoints_.emplace(src.id(), hdl);
}
test_multiplexer::pending_connects_map& test_multiplexer::pending_connects() { test_multiplexer::pending_connects_map& test_multiplexer::pending_connects() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
return pending_connects_; return pending_connects_;
} }
test_multiplexer::pending_endpoints_map& test_multiplexer::pending_endpoints() {
CAF_ASSERT(std::this_thread::get_id() == tid_);
return pending_endpoints_;
}
bool test_multiplexer::has_pending_scribe(std::string x, uint16_t y) { bool test_multiplexer::has_pending_scribe(std::string x, uint16_t y) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
guard_type guard{mx_}; guard_type guard{mx_};
return scribes_.count(std::make_pair(std::move(x), y)) > 0; return scribes_.count(std::make_pair(std::move(x), y)) > 0;
} }
bool test_multiplexer::has_pending_remote_endpoint(std::string x,
uint16_t y) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
guard_type guard{mx_};
return remote_endpoints_.count(std::make_pair(std::move(x), y)) > 0;
}
void test_multiplexer::accept_connection(accept_handle hdl) { void test_multiplexer::accept_connection(accept_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(hdl)); CAF_LOG_TRACE(CAF_ARG(hdl));
...@@ -464,7 +799,8 @@ bool test_multiplexer::try_read_data(connection_handle hdl) { ...@@ -464,7 +799,8 @@ bool test_multiplexer::try_read_data(connection_handle hdl) {
sd.rd_buf.clear(); sd.rd_buf.clear();
auto xbuf_size = static_cast<ptrdiff_t>(sd.vn_buf.size()); auto xbuf_size = static_cast<ptrdiff_t>(sd.vn_buf.size());
auto first = sd.vn_buf.begin(); auto first = sd.vn_buf.begin();
auto last = (max_bytes < xbuf_size) ? first + max_bytes : sd.vn_buf.end(); auto last = (max_bytes < xbuf_size) ? first + max_bytes
: sd.vn_buf.end();
sd.rd_buf.insert(sd.rd_buf.end(), first, last); sd.rd_buf.insert(sd.rd_buf.end(), first, last);
sd.vn_buf.erase(first, last); sd.vn_buf.erase(first, last);
if (!sd.ptr->consume(this, sd.rd_buf.data(), sd.rd_buf.size())) if (!sd.ptr->consume(this, sd.rd_buf.data(), sd.rd_buf.size()))
...@@ -538,7 +874,8 @@ bool test_multiplexer::read_data(connection_handle hdl) { ...@@ -538,7 +874,8 @@ bool test_multiplexer::read_data(connection_handle hdl) {
sd.rd_buf.clear(); sd.rd_buf.clear();
auto xbuf_size = static_cast<ptrdiff_t>(sd.vn_buf.size()); auto xbuf_size = static_cast<ptrdiff_t>(sd.vn_buf.size());
auto first = sd.vn_buf.begin(); auto first = sd.vn_buf.begin();
auto last = (max_bytes < xbuf_size) ? first + max_bytes : sd.vn_buf.end(); auto last = (max_bytes < xbuf_size) ? first + max_bytes
: sd.vn_buf.end();
sd.rd_buf.insert(sd.rd_buf.end(), first, last); sd.rd_buf.insert(sd.rd_buf.end(), first, last);
sd.vn_buf.erase(first, last); sd.vn_buf.erase(first, last);
if (!sd.ptr->consume(this, sd.rd_buf.data(), sd.rd_buf.size())) if (!sd.ptr->consume(this, sd.rd_buf.data(), sd.rd_buf.size()))
...@@ -550,6 +887,39 @@ bool test_multiplexer::read_data(connection_handle hdl) { ...@@ -550,6 +887,39 @@ bool test_multiplexer::read_data(connection_handle hdl) {
} }
} }
bool test_multiplexer::read_data(datagram_handle hdl) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(hdl));
flush_runnables();
if (passive_mode(hdl))
return false;
auto ditr = datagram_data_.find(hdl);
if (ditr == datagram_data_.end() || ditr->second->ptr->parent() == nullptr
|| !ditr->second->ptr->parent()->getf(abstract_actor::is_initialized_flag))
return false;
auto& data = ditr->second;
if (data->vn_buf.back().second.empty())
return false;
// Since we can't swap std::vector and caf::io::network::receive_buffer
// just copy over the data. This is for testing and not performance critical.
auto& from = data->vn_buf.front();
auto& to = data->rd_buf;
to.first = from.first;
CAF_ASSERT(to.second.capacity() > from.second.size());
to.second.resize(from.second.size());
std::copy(from.second.begin(), from.second.end(), to.second.begin());
data->vn_buf.pop_front();
auto sitr = datagram_data_.find(data->rd_buf.first);
if (sitr == datagram_data_.end()) {
if (!data->ptr->new_endpoint(data->rd_buf.second))
passive_mode(hdl) = true;
} else {
if (!data->ptr->consume(this, data->rd_buf.first, data->rd_buf.second))
passive_mode(hdl) = true;
}
return true;
}
void test_multiplexer::virtual_send(connection_handle hdl, void test_multiplexer::virtual_send(connection_handle hdl,
const buffer_type& buf) { const buffer_type& buf) {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
...@@ -559,6 +929,15 @@ void test_multiplexer::virtual_send(connection_handle hdl, ...@@ -559,6 +929,15 @@ void test_multiplexer::virtual_send(connection_handle hdl,
read_data(hdl); read_data(hdl);
} }
void test_multiplexer::virtual_send(datagram_handle dst, datagram_handle ep,
const buffer_type& buf) {
CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(CAF_ARG(hdl));
auto& vb = virtual_network_buffer(dst);
vb.emplace_back(ep, buf);
read_data(dst);
}
void test_multiplexer::exec_runnable() { void test_multiplexer::exec_runnable() {
CAF_ASSERT(std::this_thread::get_id() == tid_); CAF_ASSERT(std::this_thread::get_id() == tid_);
CAF_LOG_TRACE(""); CAF_LOG_TRACE("");
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
#include "caf/config.hpp" #include "caf/config.hpp"
#define CAF_SUITE io_basp #define CAF_SUITE io_basp_tcp
#include "caf/test/dsl.hpp" #include "caf/test/dsl.hpp"
#include <array> #include <array>
...@@ -482,7 +482,7 @@ CAF_TEST_FIXTURE_SCOPE(basp_tests, fixture) ...@@ -482,7 +482,7 @@ CAF_TEST_FIXTURE_SCOPE(basp_tests, fixture)
CAF_TEST(empty_server_handshake) { CAF_TEST(empty_server_handshake) {
// test whether basp instance correctly sends a // test whether basp instance correctly sends a
// server handshake whene there's no actor published // server handshake when there's no actor published
buffer buf; buffer buf;
instance().write_server_handshake(mpx(), buf, none); instance().write_server_handshake(mpx(), buf, none);
basp::header hdr; basp::header hdr;
...@@ -786,6 +786,12 @@ CAF_TEST(automatic_connection) { ...@@ -786,6 +786,12 @@ CAF_TEST(automatic_connection) {
// jupiter [remote hdl 0] -> mars [remote hdl 1] -> earth [this_node] // jupiter [remote hdl 0] -> mars [remote hdl 1] -> earth [this_node]
// (this node receives a message from jupiter via mars and responds via mars, // (this node receives a message from jupiter via mars and responds via mars,
// but then also establishes a connection to jupiter directly) // but then also establishes a connection to jupiter directly)
auto check_node_in_tbl = [&](node& n) {
io::id_visitor id_vis;
auto hdl = tbl().lookup_direct(n.id);
CAF_REQUIRE(hdl);
CAF_CHECK_EQUAL(visit(id_vis, *hdl), n.connection.id());
};
mpx()->provide_scribe("jupiter", 8080, jupiter().connection); mpx()->provide_scribe("jupiter", 8080, jupiter().connection);
CAF_CHECK(mpx()->has_pending_scribe("jupiter", 8080)); CAF_CHECK(mpx()->has_pending_scribe("jupiter", 8080));
CAF_MESSAGE("self: " << to_string(self()->address())); CAF_MESSAGE("self: " << to_string(self()->address()));
...@@ -795,7 +801,8 @@ CAF_TEST(automatic_connection) { ...@@ -795,7 +801,8 @@ CAF_TEST(automatic_connection) {
mpx()->flush_runnables(); // process publish message in basp_broker mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to mars"); CAF_MESSAGE("connect to mars");
connect_node(mars(), ax, self()->id()); connect_node(mars(), ax, self()->id());
CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id()); //CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id());
check_node_in_tbl(mars());
CAF_MESSAGE("simulate that an actor from jupiter " CAF_MESSAGE("simulate that an actor from jupiter "
"sends a message to us via mars"); "sends a message to us via mars");
mock(mars().connection, mock(mars().connection,
...@@ -819,7 +826,7 @@ CAF_TEST(automatic_connection) { ...@@ -819,7 +826,7 @@ CAF_TEST(automatic_connection) {
invalid_actor_id, invalid_actor_id,
config_serv_atom, config_serv_atom,
std::vector<actor_id>{}, std::vector<actor_id>{},
make_message(get_atom::value, "basp.default-connectivity")) make_message(get_atom::value, "basp.default-connectivity-tcp"))
.receive(mars().connection, .receive(mars().connection,
basp::message_type::announce_proxy, no_flags, no_payload, basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id, no_operation_data, this_node(), jupiter().id,
...@@ -827,7 +834,7 @@ CAF_TEST(automatic_connection) { ...@@ -827,7 +834,7 @@ CAF_TEST(automatic_connection) {
CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u); CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u);
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), mars().id); CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), mars().id);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none); CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
auto connection_helper = sys.latest_actor_id(); auto connection_helper_actor = sys.latest_actor_id();
CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u); CAF_CHECK_EQUAL(mpx()->output_buffer(mars().connection).size(), 0u);
// create a dummy config server and respond to the name lookup // create a dummy config server and respond to the name lookup
CAF_MESSAGE("receive ConfigServ of jupiter"); CAF_MESSAGE("receive ConfigServ of jupiter");
...@@ -836,9 +843,9 @@ CAF_TEST(automatic_connection) { ...@@ -836,9 +843,9 @@ CAF_TEST(automatic_connection) {
mock(mars().connection, mock(mars().connection,
{basp::message_type::dispatch_message, 0, 0, 0, {basp::message_type::dispatch_message, 0, 0, 0,
this_node(), this_node(), this_node(), this_node(),
invalid_actor_id, connection_helper}, invalid_actor_id, connection_helper_actor},
std::vector<actor_id>{}, std::vector<actor_id>{},
make_message("basp.default-connectivity", make_message("basp.default-connectivity-tcp",
make_message(uint16_t{8080}, std::move(res)))); make_message(uint16_t{8080}, std::move(res))));
// our connection helper should now connect to jupiter and // our connection helper should now connect to jupiter and
// send the scribe handle over to the BASP broker // send the scribe handle over to the BASP broker
...@@ -861,9 +868,8 @@ CAF_TEST(automatic_connection) { ...@@ -861,9 +868,8 @@ CAF_TEST(automatic_connection) {
invalid_actor_id, invalid_actor_id, std::string{}); invalid_actor_id, invalid_actor_id, std::string{});
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), none); CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), none);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none); CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
CAF_CHECK_EQUAL(tbl().lookup_direct(jupiter().id).id(), check_node_in_tbl(jupiter());
jupiter().connection.id()); check_node_in_tbl(mars());
CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id());
CAF_MESSAGE("receive message from jupiter"); CAF_MESSAGE("receive message from jupiter");
self()->receive( self()->receive(
[](const std::string& str) -> std::string { [](const std::string& str) -> std::string {
......
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include "caf/config.hpp"
#define CAF_SUITE io_basp_udp
#include "caf/test/dsl.hpp"
#include <array>
#include <mutex>
#include <memory>
#include <limits>
#include <vector>
#include <iostream>
#include <condition_variable>
#include "caf/all.hpp"
#include "caf/io/all.hpp"
#include "caf/io/datagram_handle.hpp"
#include "caf/io/datagram_servant.hpp"
#include "caf/deep_to_string.hpp"
#include "caf/io/network/interfaces.hpp"
#include "caf/io/network/ip_endpoint.hpp"
#include "caf/io/network/test_multiplexer.hpp"
namespace {
struct anything { };
anything any_vals;
template <class T>
using maybe = caf::variant<anything, T>;
constexpr uint8_t no_flags = 0;
constexpr uint32_t no_payload = 0;
constexpr uint64_t no_operation_data = 0;
constexpr auto basp_atom = caf::atom("BASP");
constexpr auto spawn_serv_atom = caf::atom("SpawnServ");
constexpr auto config_serv_atom = caf::atom("ConfigServ");
} // namespace <anonymous>
namespace caf {
template <class T, class U>
bool operator==(const maybe<T>& x, const U& y) {
return get_if<anything>(&x) != nullptr || get<T>(x) == y;
}
template <class T, class U>
bool operator==(const T& x, const maybe<U>& y) {
return (y == x);
}
template <class T>
std::string to_string(const maybe<T>& x) {
return !get_if<anything>(&x) ? std::string{"*"} : deep_to_string(get<T>(x));
}
} // namespace caf
using namespace std;
using namespace caf;
using namespace caf::io;
namespace {
constexpr uint32_t num_remote_nodes = 2;
using buffer = std::vector<char>;
std::string hexstr(const buffer& buf) {
return deep_to_string(meta::hex_formatted(), buf);
}
struct node {
std::string name;
node_id id;
datagram_handle endpoint;
union { scoped_actor dummy_actor; };
node() {
// nop
}
~node() {
// nop
}
};
class fixture {
public:
fixture(bool autoconn = false)
: sys(cfg.load<io::middleman, network::test_multiplexer>()
.set("middleman.enable-automatic-connections", autoconn)
.set("middleman.enable-udp", true)
.set("middleman.enable-tcp", false)
.set("scheduler.policy", autoconn ? caf::atom("testing")
: caf::atom("stealing"))
.set("middleman.detach-utility-actors", !autoconn)) {
auto& mm = sys.middleman();
mpx_ = dynamic_cast<network::test_multiplexer*>(&mm.backend());
CAF_REQUIRE(mpx_ != nullptr);
CAF_REQUIRE(&sys == &mpx_->system());
auto hdl = mm.named_broker<basp_broker>(basp_atom);
aut_ = static_cast<basp_broker*>(actor_cast<abstract_actor*>(hdl));
this_node_ = sys.node();
self_.reset(new scoped_actor{sys});
dhdl_ = datagram_handle::from_int(1);
aut_->add_datagram_servant(mpx_->new_datagram_servant(dhdl_, 1u));
registry_ = &sys.registry();
registry_->put((*self_)->id(), actor_cast<strong_actor_ptr>(*self_));
// first remote node is everything of this_node + 1, then +2, etc.
for (uint32_t i = 0; i < num_remote_nodes; ++i) {
auto& n = nodes_[i];
node_id::host_id_type tmp = this_node_.host_id();
for (auto& c : tmp)
c = static_cast<uint8_t>(c + i + 1);
n.id = node_id{this_node_.process_id() + i + 1, tmp};
n.endpoint = datagram_handle::from_int(i + 2);
new (&n.dummy_actor) scoped_actor(sys);
// register all pseudo remote actors in the registry
registry_->put(n.dummy_actor->id(),
actor_cast<strong_actor_ptr>(n.dummy_actor));
}
// make sure all init messages are handled properly
mpx_->flush_runnables();
nodes_[0].name = "Jupiter";
nodes_[1].name = "Mars";
CAF_REQUIRE_NOT_EQUAL(jupiter().endpoint, mars().endpoint);
CAF_MESSAGE("Earth: " << to_string(this_node_) << ", ID = "
<< endpoint_handle().id());
CAF_MESSAGE("Jupiter: " << to_string(jupiter().id) << ", ID = "
<< jupiter().endpoint.id());
CAF_MESSAGE("Mars: " << to_string(mars().id) << ", ID = "
<< mars().endpoint.id());
CAF_REQUIRE_NOT_EQUAL(this_node_, jupiter().id);
CAF_REQUIRE_NOT_EQUAL(jupiter().id, mars().id);
}
~fixture() {
this_node_ = none;
self_.reset();
for (auto& n : nodes_) {
n.id = none;
n.dummy_actor.~scoped_actor();
}
}
uint32_t serialized_size(const message& msg) {
buffer buf;
binary_serializer bs{mpx_, buf};
auto e = bs(const_cast<message&>(msg));
CAF_REQUIRE(!e);
return static_cast<uint32_t>(buf.size());
}
node& jupiter() {
return nodes_[0];
}
node& mars() {
return nodes_[1];
}
// our "virtual communication backend"
network::test_multiplexer* mpx() {
return mpx_;
}
// actor-under-test
basp_broker* aut() {
return aut_;
}
// our node ID
node_id& this_node() {
return this_node_;
}
// an actor reference representing a local actor
scoped_actor& self() {
return *self_;
}
datagram_handle endpoint_handle() {
return dhdl_;
}
// implementation of the Binary Actor System Protocol
basp::instance& instance() {
return aut()->state.instance;
}
// our routing table (filled by BASP)
basp::routing_table& tbl() {
return aut()->state.instance.tbl();
}
// access to proxy instances
proxy_registry& proxies() {
return aut()->state.proxies();
}
// stores the singleton pointer for convenience
actor_registry* registry() {
return registry_;
}
using payload_writer = basp::instance::payload_writer;
template <class... Ts>
void to_payload(binary_serializer& bs, const Ts&... xs) {
bs(const_cast<Ts&>(xs)...);
}
template <class... Ts>
void to_payload(buffer& buf, const Ts&... xs) {
binary_serializer bs{mpx_, buf};
to_payload(bs, xs...);
}
void to_buf(buffer& buf, basp::header& hdr, payload_writer* writer) {
instance().write(mpx_, buf, hdr, writer);
}
template <class T, class... Ts>
void to_buf(buffer& buf, basp::header& hdr, payload_writer* writer,
const T& x, const Ts&... xs) {
auto pw = make_callback([&](serializer& sink) -> error {
if (writer)
return error::eval([&] { return (*writer)(sink); },
[&] { return sink(const_cast<T&>(x)); });
return sink(const_cast<T&>(x));
});
to_buf(buf, hdr, &pw, xs...);
}
std::pair<basp::header, buffer> from_buf(const buffer& buf) {
basp::header hdr;
binary_deserializer bd{mpx_, buf};
auto e = bd(hdr);
CAF_REQUIRE(!e);
buffer payload;
if (hdr.payload_len > 0) {
std::copy(buf.begin() + basp::header_size, buf.end(),
std::back_inserter(payload));
}
return {hdr, std::move(payload)};
}
void establish_communication(node& n,
optional<datagram_handle> dx = none,
actor_id published_actor_id = invalid_actor_id,
const set<string>& published_actor_ifs
= std::set<std::string>{}) {
auto src = dx ? *dx : dhdl_;
CAF_MESSAGE("establish communication on node " << n.name
<< ", delegated servant ID = " << n.endpoint.id()
<< ", initial servant ID = " << src.id());
// send the client handshake and receive the server handshake
// and a dispatch_message as answers
auto hdl = n.endpoint;
mpx_->add_pending_endpoint(src, hdl);
CAF_MESSAGE("Send client handshake");
mock(src, hdl,
{basp::message_type::client_handshake, 0, 0, 0,
n.id, this_node(),
invalid_actor_id, invalid_actor_id}, std::string{})
// upon receiving the client handshake, the server should answer
// with the server handshake and send the dispatch_message blow
.receive(hdl,
basp::message_type::server_handshake, no_flags,
any_vals, basp::version, this_node(), node_id{none},
published_actor_id, invalid_actor_id, std::string{},
published_actor_id, published_actor_ifs)
// upon receiving our client handshake, BASP will check
// whether there is a SpawnServ actor on this node
.receive(hdl,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data,
this_node(), n.id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_addr>{},
make_message(sys_atom::value, get_atom::value, "info"));
// test whether basp instance correctly updates the
// routing table upon receiving client handshakes
auto path = tbl().lookup(n.id);
CAF_REQUIRE(path);
CAF_CHECK_EQUAL(path->hdl, n.endpoint);
CAF_CHECK_EQUAL(path->next_hop, n.id);
}
std::pair<basp::header, buffer> read_from_out_buf(datagram_handle hdl) {
CAF_MESSAGE("read from output buffer for endpoint " << hdl.id());
auto& que = mpx_->output_queue(hdl);
while (que.empty())
mpx()->exec_runnable();
auto result = from_buf(que.front().second);
que.pop_front();
return result;
}
void dispatch_out_buf(datagram_handle hdl) {
basp::header hdr;
buffer buf;
std::tie(hdr, buf) = read_from_out_buf(hdl);
CAF_MESSAGE("dispatch output buffer for endpoint " << hdl.id());
CAF_REQUIRE(hdr.operation == basp::message_type::dispatch_message);
binary_deserializer source{mpx_, buf};
std::vector<strong_actor_ptr> stages;
message msg;
auto e = source(stages, msg);
CAF_REQUIRE(!e);
auto src = actor_cast<strong_actor_ptr>(registry_->get(hdr.source_actor));
auto dest = registry_->get(hdr.dest_actor);
CAF_REQUIRE(dest);
dest->enqueue(make_mailbox_element(src, message_id::make(),
std::move(stages), std::move(msg)),
nullptr);
}
class mock_t {
public:
mock_t(fixture* thisptr) : this_(thisptr) {
// nop
}
mock_t(mock_t&&) = default;
~mock_t() {
if (num > 1)
CAF_MESSAGE("implementation under test responded with "
<< (num - 1) << " BASP message" << (num > 2 ? "s" : ""));
}
template <class... Ts>
mock_t& receive(datagram_handle hdl,
maybe<basp::message_type> operation,
maybe<uint8_t> flags,
maybe<uint32_t> payload_len,
maybe<uint64_t> operation_data,
maybe<node_id> source_node,
maybe<node_id> dest_node,
maybe<actor_id> source_actor,
maybe<actor_id> dest_actor,
const Ts&... xs) {
CAF_MESSAGE("expect #" << num << " on endpoint ID = " << hdl.id());
buffer buf;
this_->to_payload(buf, xs...);
auto& oq = this_->mpx()->output_queue(hdl);
while (oq.empty())
this_->mpx()->exec_runnable();
CAF_MESSAGE("output queue has " << oq.size() << " messages");
auto dh = oq.front().first;
buffer& ob = oq.front().second;
CAF_MESSAGE("next datagram has " << ob.size()
<< " bytes, servant ID = " << dh.id());
CAF_CHECK_EQUAL(dh.id(), hdl.id());
basp::header hdr;
{ // lifetime scope of source
binary_deserializer source{this_->mpx(), ob};
auto e = source(hdr);
CAF_REQUIRE_EQUAL(e, none);
}
buffer payload;
if (hdr.payload_len > 0) {
CAF_REQUIRE(ob.size() >= (basp::header_size + hdr.payload_len));
auto first = ob.begin() + basp::header_size;
auto end = first + hdr.payload_len;
payload.assign(first, end);
}
CAF_MESSAGE("erase message from output queue");
oq.pop_front();
CAF_CHECK_EQUAL(operation, hdr.operation);
CAF_CHECK_EQUAL(flags, static_cast<size_t>(hdr.flags));
CAF_CHECK_EQUAL(payload_len, hdr.payload_len);
CAF_CHECK_EQUAL(operation_data, hdr.operation_data);
CAF_CHECK_EQUAL(source_node, hdr.source_node);
CAF_CHECK_EQUAL(dest_node, hdr.dest_node);
CAF_CHECK_EQUAL(source_actor, hdr.source_actor);
CAF_CHECK_EQUAL(dest_actor, hdr.dest_actor);
CAF_REQUIRE_EQUAL(buf.size(), payload.size());
CAF_REQUIRE_EQUAL(hexstr(buf), hexstr(payload));
++num;
return *this;
}
template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, datagram_handle ep,
basp::header hdr, const Ts&... xs) {
buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size)
<< " bytes payload to back of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_back(ep, buf);
return *this;
}
template <class... Ts>
mock_t& enqueue_back(datagram_handle hdl, basp::header hdr,
Ts&&... xs) {
return enqueue_back(hdl, hdl, hdr, std::forward<Ts>(xs)...);
}
template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, datagram_handle ep,
basp::header hdr, const Ts&... xs) {
buffer buf;
this_->to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("adding msg " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size)
<< " bytes payload to front of queue");
this_->mpx()->virtual_network_buffer(hdl).emplace_front(ep, buf);
return *this;
}
template <class... Ts>
mock_t& enqueue_front(datagram_handle hdl, basp::header hdr,
Ts&&... xs) {
return enqueue_front(hdl, hdl, hdr, std::forward<Ts>(xs)...);
}
mock_t& deliver(datagram_handle hdl, size_t num_messages = 1) {
for (size_t i = 0; i < num_messages; ++i)
this_->mpx()->read_data(hdl);
return *this;
}
private:
fixture* this_;
size_t num = 1;
};
template <class... Ts>
mock_t mock(datagram_handle hdl, basp::header hdr,
const Ts&... xs) {
buffer buf;
to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size)
<< " bytes payload");
mpx()->virtual_send(hdl, hdl, buf);
return {this};
}
template <class... Ts>
mock_t mock(datagram_handle hdl, datagram_handle ep, basp::header hdr,
const Ts&... xs) {
buffer buf;
to_buf(buf, hdr, nullptr, xs...);
CAF_MESSAGE("virtually send " << to_string(hdr.operation)
<< " with " << (buf.size() - basp::header_size)
<< " bytes payload");
mpx()->virtual_send(hdl, ep, buf);
return {this};
}
mock_t mock() {
return {this};
}
actor_system_config cfg;
actor_system sys;
private:
basp_broker* aut_;
datagram_handle dhdl_;
network::test_multiplexer* mpx_;
node_id this_node_;
unique_ptr<scoped_actor> self_;
array<node, num_remote_nodes> nodes_;
/*
array<node_id, num_remote_nodes> remote_node_;
array<connection_handle, num_remote_nodes> remote_hdl_;
array<unique_ptr<scoped_actor>, num_remote_nodes> pseudo_remote_;
*/
actor_registry* registry_;
};
class autoconn_enabled_fixture : public fixture {
public:
using scheduler_type = caf::scheduler::test_coordinator;
scheduler_type& sched;
middleman_actor mma;
autoconn_enabled_fixture()
: fixture(true),
sched(dynamic_cast<scheduler_type&>(sys.scheduler())),
mma(sys.middleman().actor_handle()) {
// nop
}
void publish(const actor& whom, uint16_t port, bool is_udp = false) {
using sig_t = std::set<std::string>;
scoped_actor tmp{sys};
sig_t sigs;
CAF_MESSAGE("publish whom on port " << port);
if (is_udp)
tmp->send(mma, publish_udp_atom::value, port,
actor_cast<strong_actor_ptr>(whom), std::move(sigs), "", false);
else
tmp->send(mma, publish_atom::value, port,
actor_cast<strong_actor_ptr>(whom), std::move(sigs), "", false);
CAF_MESSAGE("publish from tmp to mma with port _");
expect((atom_value, uint16_t, strong_actor_ptr, sig_t, std::string, bool),
from(tmp).to(mma).with(_));
CAF_MESSAGE("publish: from mma to tmp with port " << port);
expect((uint16_t), from(mma).to(tmp).with(port));
}
};
} // namespace <anonymous>
CAF_TEST_FIXTURE_SCOPE(basp_udp_tests, fixture)
CAF_TEST(empty_server_handshake_udp) {
// test whether basp instance correctly sends a
// client handshake when there's no actor published
buffer buf;
instance().write_server_handshake(mpx(), buf, none);
basp::header hdr;
buffer payload;
std::tie(hdr, payload) = from_buf(buf);
basp::header expected{basp::message_type::server_handshake, 0,
static_cast<uint32_t>(payload.size()),
basp::version,
this_node(), none,
invalid_actor_id, invalid_actor_id,
0};
CAF_CHECK(basp::valid(hdr));
CAF_CHECK(basp::is_handshake(hdr));
CAF_CHECK_EQUAL(to_string(hdr), to_string(expected));
}
CAF_TEST(empty_client_handshake_udp) {
// test whether basp instance correctly sends a
// client handshake when there's no actor published
buffer buf;
instance().write_client_handshake(mpx(), buf, none);
basp::header hdr;
buffer payload;
std::tie(hdr, payload) = from_buf(buf);
basp::header expected{basp::message_type::client_handshake, 0,
static_cast<uint32_t>(payload.size()), 0,
this_node(), none,
invalid_actor_id, invalid_actor_id,
0};
CAF_CHECK(basp::valid(hdr));
CAF_CHECK(basp::is_handshake(hdr));
CAF_MESSAGE("got : " << to_string(hdr));
CAF_MESSAGE("expecting: " << to_string(expected));
CAF_CHECK_EQUAL(to_string(hdr), to_string(expected));
}
CAF_TEST(non_empty_server_handshake_udp) {
// test whether basp instance correctly sends a
// server handshake with published actors
buffer buf;
instance().add_published_actor(4242, actor_cast<strong_actor_ptr>(self()),
{"caf::replies_to<@u16>::with<@u16>"});
instance().write_server_handshake(mpx(), buf, uint16_t{4242});
buffer expected_buf;
basp::header expected{basp::message_type::server_handshake, 0, 0,
basp::version, this_node(), none,
self()->id(), invalid_actor_id, 0};
to_buf(expected_buf, expected, nullptr, std::string{},
self()->id(), set<string>{"caf::replies_to<@u16>::with<@u16>"});
CAF_CHECK_EQUAL(hexstr(buf), hexstr(expected_buf));
}
CAF_TEST(remote_address_and_port_udp) {
CAF_MESSAGE("connect to Mars");
establish_communication(mars());
auto mm = sys.middleman().actor_handle();
CAF_MESSAGE("ask MM about node ID of Mars");
self()->send(mm, get_atom::value, mars().id);
do {
mpx()->exec_runnable();
} while (!self()->has_next_message());
CAF_MESSAGE("receive result of MM");
self()->receive(
[&](const node_id& nid, const std::string& addr, uint16_t port) {
CAF_CHECK_EQUAL(nid, mars().id);
// all test nodes have address "test" and connection handle ID as port
CAF_CHECK_EQUAL(addr, "test");
CAF_CHECK_EQUAL(port, mars().endpoint.id());
}
);
}
CAF_TEST(client_handshake_and_dispatch_udp) {
CAF_MESSAGE("establish communication with Jupiter");
establish_communication(jupiter());
CAF_MESSAGE("send dispatch message");
// send a message via `dispatch` from node 0 on endpoint 1
mock(endpoint_handle(), jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(), jupiter().dummy_actor->id(), self()->id(),
1}, // increment sequence number
std::vector<actor_addr>{},
make_message(1, 2, 3))
.receive(jupiter().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
// must've created a proxy for our remote actor
CAF_REQUIRE(proxies().count_proxies(jupiter().id) == 1);
// must've send remote node a message that this proxy is monitored now
// receive the message
self()->receive(
[](int a, int b, int c) {
CAF_CHECK_EQUAL(a, 1);
CAF_CHECK_EQUAL(b, 2);
CAF_CHECK_EQUAL(c, 3);
return a + b + c;
}
);
CAF_MESSAGE("exec message of forwarding proxy");
mpx()->exec_runnable();
// deserialize and send message from out buf
dispatch_out_buf(jupiter().endpoint);
jupiter().dummy_actor->receive(
[](int i) {
CAF_CHECK_EQUAL(i, 6);
}
);
}
CAF_TEST(message_forwarding_udp) {
// connect two remote nodes
CAF_MESSAGE("establish communication with Jupiter");
establish_communication(jupiter());
CAF_MESSAGE("establish communication with Mars");
establish_communication(mars());
CAF_MESSAGE("send dispatch message to ... ");
auto msg = make_message(1, 2, 3);
// send a message from node 0 to node 1, forwarded by this node
mock(endpoint_handle(), jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id(),
1}, // increment sequence number
msg)
.receive(mars().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, jupiter().id, mars().id,
invalid_actor_id, mars().dummy_actor->id(),
msg);
}
CAF_TEST(publish_and_connect_udp) {
auto dx = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx);
auto res = sys.middleman().publish_udp(self(), 4242);
CAF_REQUIRE(res == 4242);
mpx()->flush_runnables(); // process publish message in basp_broker
establish_communication(jupiter(), dx, self()->id());
}
CAF_TEST(remote_actor_and_send_udp) {
constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint);
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle();
actor result;
auto f = self()->request(mm1, infinite, contact_atom::value,
lo, uint16_t{4242});
// wait until BASP broker has received and processed the connect message
while (!aut()->valid(jupiter().endpoint))
mpx()->exec_runnable();
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors();
mock()
.receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u,
no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{});
mock(jupiter().endpoint,
{basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id,
0}, // sequence number, first message
std::string{},
jupiter().dummy_actor->id(),
uint32_t{0})
.receive(jupiter().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"))
.receive(jupiter().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("BASP broker should've send the proxy");
f.receive(
[&](node_id nid, strong_actor_ptr res, std::set<std::string> ifs) {
CAF_REQUIRE(res);
auto aptr = actor_cast<abstract_actor*>(res);
CAF_REQUIRE(dynamic_cast<forwarding_actor_proxy*>(aptr) != nullptr);
CAF_CHECK_EQUAL(proxies().count_proxies(jupiter().id), 1u);
CAF_CHECK_EQUAL(nid, jupiter().id);
CAF_CHECK_EQUAL(res->node(), jupiter().id);
CAF_CHECK_EQUAL(res->id(), jupiter().dummy_actor->id());
CAF_CHECK(ifs.empty());
auto proxy = proxies().get(jupiter().id, jupiter().dummy_actor->id());
CAF_REQUIRE(proxy != nullptr);
CAF_REQUIRE(proxy == res);
result = actor_cast<actor>(res);
},
[&](error& err) {
CAF_FAIL("error: " << sys.render(err));
}
);
CAF_MESSAGE("send message to proxy");
anon_send(actor_cast<actor>(result), 42);
mpx()->flush_runnables();
mock()
.receive(jupiter().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message(42));
auto msg = make_message("hi there!");
CAF_MESSAGE("send message via BASP (from proxy)");
mock(jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id(),
1}, // sequence number, second message
std::vector<actor_id>{},
make_message("hi there!"));
self()->receive(
[&](const string& str) {
CAF_CHECK_EQUAL(to_string(self()->current_sender()), to_string(result));
CAF_CHECK_EQUAL(self()->current_sender(), result.address());
CAF_CHECK_EQUAL(str, "hi there!");
}
);
}
CAF_TEST(actor_serialize_and_deserialize_udp) {
auto testee_impl = [](event_based_actor* testee_self) -> behavior {
testee_self->set_default_handler(reflect_and_quit);
return {
[] {
// nop
}
};
};
establish_communication(jupiter());
auto prx = proxies().get_or_put(jupiter().id, jupiter().dummy_actor->id());
mock()
.receive(jupiter().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), prx->node(),
invalid_actor_id, prx->id());
CAF_CHECK_EQUAL(prx->node(), jupiter().id);
CAF_CHECK_EQUAL(prx->id(), jupiter().dummy_actor->id());
auto testee = sys.spawn(testee_impl);
registry()->put(testee->id(), actor_cast<strong_actor_ptr>(testee));
CAF_MESSAGE("send message via BASP (from proxy)");
auto msg = make_message(actor_cast<actor_addr>(prx));
mock(endpoint_handle(), jupiter().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
prx->node(), this_node(),
prx->id(), testee->id(),
1}, // sequence number, first message after handshake
std::vector<actor_id>{},
msg);
// testee must've responded (process forwarded message in BASP broker)
CAF_MESSAGE("wait until BASP broker writes to its output buffer");
while (mpx()->output_queue(jupiter().endpoint).empty())
mpx()->exec_runnable(); // process forwarded message in basp_broker
// output buffer must contain the reflected message
mock()
.receive(jupiter().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), prx->node(), testee->id(), prx->id(),
std::vector<actor_id>{}, msg);
}
CAF_TEST(indirect_connections_udp) {
// this node receives a message from jupiter via mars and responds via mars
// and any ad-hoc automatic connection requests are ignored
CAF_MESSAGE("self: " << to_string(self()->address()));
auto dx = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx);
sys.middleman().publish_udp(self(), 4242);
mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to Mars");
establish_communication(mars(), dx, self()->id());
CAF_MESSAGE("actor from Jupiter sends a message to us via Mars");
auto mx = mock(dx, mars().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id(),
1}, // sequence number
std::vector<actor_id>{},
make_message("hello from jupiter!"));
CAF_MESSAGE("expect ('sys', 'get', \"info\") from Earth to Jupiter at Mars");
// this asks Jupiter if it has a 'SpawnServ'
mx.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"));
CAF_MESSAGE("expect announce_proxy message at Mars from Earth to Jupiter");
mx.receive(mars().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("receive message from jupiter");
self()->receive(
[](const std::string& str) -> std::string {
CAF_CHECK_EQUAL(str, "hello from jupiter!");
return "hello from earth!";
}
);
mpx()->exec_runnable(); // process forwarded message in basp_broker
mock()
.receive(mars().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
}
CAF_TEST(out_of_order_delivery_udp) {
constexpr const char* lo = "localhost";
CAF_MESSAGE("self: " << to_string(self()->address()));
mpx()->provide_datagram_servant(lo, 4242, jupiter().endpoint);
CAF_REQUIRE(mpx()->has_pending_remote_endpoint(lo, 4242));
auto mm1 = sys.middleman().actor_handle();
actor result;
auto f = self()->request(mm1, infinite, contact_atom::value,
lo, uint16_t{4242});
// wait until BASP broker has received and processed the connect message
while (!aut()->valid(jupiter().endpoint))
mpx()->exec_runnable();
CAF_REQUIRE(!mpx()->has_pending_scribe(lo, 4242));
// build a fake server handshake containing the id of our first pseudo actor
CAF_MESSAGE("client handshake => server handshake => proxy announcement");
auto na = registry()->named_actors();
mock()
.receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u,
no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{});
mock(jupiter().endpoint, jupiter().endpoint,
{basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id,
0}, // sequence number, first message
std::string{},
jupiter().dummy_actor->id(),
uint32_t{0})
.receive(jupiter().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"))
.receive(jupiter().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_MESSAGE("BASP broker should've send the proxy");
f.receive(
[&](node_id nid, strong_actor_ptr res, std::set<std::string> ifs) {
CAF_REQUIRE(res);
auto aptr = actor_cast<abstract_actor*>(res);
CAF_REQUIRE(dynamic_cast<forwarding_actor_proxy*>(aptr) != nullptr);
CAF_CHECK_EQUAL(proxies().count_proxies(jupiter().id), 1u);
CAF_CHECK_EQUAL(nid, jupiter().id);
CAF_CHECK_EQUAL(res->node(), jupiter().id);
CAF_CHECK_EQUAL(res->id(), jupiter().dummy_actor->id());
CAF_CHECK(ifs.empty());
auto proxy = proxies().get(jupiter().id, jupiter().dummy_actor->id());
CAF_REQUIRE(proxy != nullptr);
CAF_REQUIRE(proxy == res);
result = actor_cast<actor>(res);
},
[&](error& err) {
CAF_FAIL("error: " << sys.render(err));
}
);
CAF_MESSAGE("send message to proxy");
anon_send(actor_cast<actor>(result), 42);
mpx()->flush_runnables();
mock()
.receive(jupiter().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message(42));
auto header_with_seq = [&](uint16_t seq) -> basp::header {
return {basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id(),
seq};
};
mock()
.enqueue_back(jupiter().endpoint, header_with_seq(1),
std::vector<actor_id>{}, make_message(0))
.enqueue_back(jupiter().endpoint, header_with_seq(2),
std::vector<actor_id>{}, make_message(1))
.enqueue_front(jupiter().endpoint, header_with_seq(3),
std::vector<actor_id>{}, make_message(2))
.enqueue_back(jupiter().endpoint, header_with_seq(4),
std::vector<actor_id>{}, make_message(3))
.enqueue_back(jupiter().endpoint, header_with_seq(5),
std::vector<actor_id>{}, make_message(4))
.enqueue_back(jupiter().endpoint, header_with_seq(6),
std::vector<actor_id>{}, make_message(5))
.enqueue_front(jupiter().endpoint, header_with_seq(7),
std::vector<actor_id>{}, make_message(6))
.enqueue_back(jupiter().endpoint, header_with_seq(8),
std::vector<actor_id>{}, make_message(7))
.enqueue_back(jupiter().endpoint, header_with_seq(9),
std::vector<actor_id>{}, make_message(8))
.enqueue_front(jupiter().endpoint, header_with_seq(10),
std::vector<actor_id>{}, make_message(9))
.deliver(jupiter().endpoint, 10);
int expected_next = 0;
self()->receive_while([&] { return expected_next < 10; }) (
[&](int val) {
CAF_CHECK_EQUAL(to_string(self()->current_sender()), to_string(result));
CAF_CHECK_EQUAL(self()->current_sender(), result.address());
CAF_CHECK_EQUAL(expected_next, val);
++expected_next;
}
);
}
CAF_TEST_FIXTURE_SCOPE_END()
CAF_TEST_FIXTURE_SCOPE(basp_udp_tests_with_autoconn, autoconn_enabled_fixture)
CAF_TEST(automatic_connection_udp) {
// this tells our BASP broker to enable the automatic connection feature
//anon_send(aut(), ok_atom::value,
// "middleman.enable-automatic-connections", make_message(true));
//mpx()->exec_runnable(); // process publish message in basp_broker
// jupiter [remote hdl 0] -> mars [remote hdl 1] -> earth [this_node]
// (this node receives a message from jupiter via mars and responds via mars,
// but then also establishes a connection to jupiter directly)
auto check_node_in_tbl = [&](node& n) {
io::id_visitor id_vis;
auto hdl = tbl().lookup_direct(n.id);
CAF_REQUIRE(hdl);
CAF_CHECK_EQUAL(visit(id_vis, *hdl), n.endpoint.id());
};
mpx()->provide_datagram_servant("jupiter", 8080, jupiter().endpoint);
CAF_CHECK(mpx()->has_pending_remote_endpoint("jupiter", 8080));
CAF_MESSAGE("self: " << to_string(self()->address()));
auto dx = datagram_handle::from_int(4242);
mpx()->provide_datagram_servant(4242, dx);
CAF_MESSAGE("A");
publish(self(), 4242, true);
CAF_MESSAGE("B");
mpx()->flush_runnables(); // process publish message in basp_broker
CAF_MESSAGE("connect to mars");
establish_communication(mars(), dx, self()->id());
//CAF_CHECK_EQUAL(tbl().lookup_direct(mars().id).id(), mars().connection.id());
check_node_in_tbl(mars());
CAF_MESSAGE("simulate that an actor from jupiter "
"sends a message to us via mars");
mock(dx, mars().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
jupiter().id, this_node(),
jupiter().dummy_actor->id(), self()->id(),
1}, // sequence number
std::vector<actor_id>{},
make_message("hello from jupiter!"))
.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals, no_operation_data,
this_node(), jupiter().id, any_vals, invalid_actor_id,
spawn_serv_atom,
std::vector<actor_id>{},
make_message(sys_atom::value, get_atom::value, "info"))
// if we do not disable TCP, we would receive a second message here asking
// for the tcp related addresses
.receive(mars().endpoint,
basp::message_type::dispatch_message,
basp::header::named_receiver_flag, any_vals,
no_operation_data, this_node(), jupiter().id,
any_vals, // actor ID of an actor spawned by the BASP broker
invalid_actor_id,
config_serv_atom,
std::vector<actor_id>{},
make_message(get_atom::value, "basp.default-connectivity-udp"))
.receive(mars().endpoint,
basp::message_type::announce_proxy, no_flags, no_payload,
no_operation_data, this_node(), jupiter().id,
invalid_actor_id, jupiter().dummy_actor->id());
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), mars().id);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
auto connection_helper_actor = sys.latest_actor_id();
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
// create a dummy config server and respond to the name lookup
CAF_MESSAGE("receive ConfigServ of jupiter");
network::address_listing res;
res[network::protocol::ipv4].emplace_back("jupiter");
mock(dx, mars().endpoint,
{basp::message_type::dispatch_message, 0, 0, 0,
this_node(), this_node(),
invalid_actor_id, connection_helper_actor,
2}, // sequence number
std::vector<actor_id>{},
make_message("basp.default-connectivity-udp",
make_message(uint16_t{8080}, std::move(res))));
// our connection helper should now connect to jupiter and
// send the scribe handle over to the BASP broker
while (mpx()->has_pending_remote_endpoint("jupiter", 8080)) {
sched.run();
mpx()->flush_runnables();
}
CAF_REQUIRE(mpx()->output_queue(mars().endpoint).empty());
CAF_MESSAGE("Let's do the handshake.");
mock()
.receive(jupiter().endpoint,
basp::message_type::client_handshake, no_flags, 1u,
no_operation_data, this_node(), node_id(),
invalid_actor_id, invalid_actor_id, std::string{});
CAF_MESSAGE("Received client handshake.");
// send handshake from jupiter
mock(jupiter().endpoint, jupiter().endpoint,
{basp::message_type::server_handshake, 0, 0, basp::version,
jupiter().id, none,
jupiter().dummy_actor->id(), invalid_actor_id},
std::string{},
jupiter().dummy_actor->id(),
uint32_t{0});
mpx()->exec_runnable(); // Receive message from connection helper
mpx()->exec_runnable(); // Process BASP server handshake
CAF_CHECK_EQUAL(tbl().lookup_indirect(jupiter().id), none);
CAF_CHECK_EQUAL(tbl().lookup_indirect(mars().id), none);
check_node_in_tbl(jupiter());
check_node_in_tbl(mars());
CAF_MESSAGE("receive message from jupiter");
self()->receive(
[](const std::string& str) -> std::string {
CAF_CHECK_EQUAL(str, "hello from jupiter!");
return "hello from earth!";
}
);
mpx()->exec_runnable(); // process forwarded message in basp_broker
CAF_MESSAGE("response message must take direct route now");
mock()
.receive(jupiter().endpoint,
basp::message_type::dispatch_message, no_flags, any_vals,
no_operation_data, this_node(), jupiter().id,
self()->id(), jupiter().dummy_actor->id(),
std::vector<actor_id>{},
make_message("hello from earth!"));
CAF_CHECK_EQUAL(mpx()->output_queue(mars().endpoint).size(), 0u);
}
CAF_TEST_FIXTURE_SCOPE_END()
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include <vector>
#include "caf/config.hpp"
#define CAF_SUITE io_ip_endpoint
#include "caf/test/unit_test.hpp"
#include "caf/actor_system.hpp"
#include "caf/actor_system_config.hpp"
#include "caf/binary_serializer.hpp"
#include "caf/binary_deserializer.hpp"
#include "caf/io/network/interfaces.hpp"
#include "caf/io/network/ip_endpoint.hpp"
using namespace caf;
using namespace caf::io;
namespace {
struct fixture {
template <class T, class... Ts>
std::vector<char> serialize(T& x, Ts&... xs) {
std::vector<char> buf;
binary_serializer bs{&context, buf};
bs(x, xs...);
return buf;
}
template <class T, class... Ts>
void deserialize(const std::vector<char>& buf, T& x, Ts&... xs) {
binary_deserializer bd{&context, buf};
bd(x, xs...);
}
fixture() : system(cfg), context(&system) {
}
actor_system_config cfg;
actor_system system;
scoped_execution_unit context;
};
} // namespace anonymous
CAF_TEST_FIXTURE_SCOPE(ep_endpoint_tests, fixture)
CAF_TEST(ip_endpoint) {
// create an empty endpoint
network::ip_endpoint ep;
ep.clear();
CAF_CHECK_EQUAL("", network::host(ep));
CAF_CHECK_EQUAL(uint16_t{0}, network::port(ep));
CAF_CHECK_EQUAL(size_t{0}, *ep.length());
// fill it with data from a local endpoint
network::interfaces::get_endpoint("localhost", 12345, ep);
// save the data
auto h = network::host(ep);
auto p = network::port(ep);
auto l = *ep.length();
CAF_CHECK("localhost" == h || "127.0.0.1" == h || "::1" == h);
CAF_CHECK_EQUAL(12345, p);
CAF_CHECK(0 < l);
// serialize the endpoint and clear it
std::vector<char> buf = serialize(ep);
auto save = ep;
ep.clear();
CAF_CHECK_EQUAL("", network::host(ep));
CAF_CHECK_EQUAL(uint16_t{0}, network::port(ep));
CAF_CHECK_EQUAL(size_t{0}, *ep.length());
// deserialize the data and check if it was load successfully
deserialize(buf, ep);
CAF_CHECK_EQUAL(h, network::host(ep));
CAF_CHECK_EQUAL(uint16_t{p}, network::port(ep));
CAF_CHECK_EQUAL(size_t{l}, *ep.length());
CAF_CHECK_EQUAL(save, ep);
}
CAF_TEST_FIXTURE_SCOPE_END()
/******************************************************************************
* ____ _ _____ *
* / ___| / \ | ___| C++ *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
* *
* Copyright (C) 2011 - 2017 *
* Dominik Charousset <dominik.charousset (at) haw-hamburg.de> *
* *
* Distributed under the terms and conditions of the BSD 3-Clause License or *
* (at your option) under the terms and conditions of the Boost Software *
* License 1.0. See accompanying files LICENSE and LICENSE_ALTERNATIVE. *
* *
* If you did not receive a copy of the license files, see *
* http://opensource.org/licenses/BSD-3-Clause and *
* http://www.boost.org/LICENSE_1_0.txt. *
******************************************************************************/
#include "caf/config.hpp"
#define CAF_SUITE io_receive_buffer
#include "caf/test/unit_test.hpp"
#include <algorithm>
#include "caf/io/network/receive_buffer.hpp"
using namespace caf;
using caf::io::network::receive_buffer;
struct fixture {
receive_buffer a;
receive_buffer b;
std::vector<char> vec;
fixture()
: a{},
b{1024ul},
vec{'h', 'a', 'l', 'l', 'o'} {
// nop
}
};
CAF_TEST_FIXTURE_SCOPE(receive_buffer_tests, fixture)
CAF_TEST(constuctors) {
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 0ul);
CAF_CHECK_EQUAL(a.data(), nullptr);
CAF_CHECK(a.empty());
CAF_CHECK_EQUAL(b.size(), 0ul);
CAF_CHECK_EQUAL(b.capacity(), 1024ul);
CAF_CHECK(b.data() != nullptr);
CAF_CHECK(b.empty());
receive_buffer other{std::move(b)};
CAF_CHECK_EQUAL(other.size(), 0ul);
CAF_CHECK_EQUAL(other.capacity(), 1024ul);
CAF_CHECK(other.data() != nullptr);
CAF_CHECK(other.empty());
}
CAF_TEST(reserve) {
a.reserve(0);
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 0ul);
CAF_CHECK(a.data() == nullptr);
CAF_CHECK(a.empty());
a.reserve(1024);
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 1024ul);
CAF_CHECK(a.data() != nullptr);
CAF_CHECK_EQUAL(a.begin(), a.end());
CAF_CHECK(a.empty());
a.reserve(512);
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 1024ul);
CAF_CHECK(a.data() != nullptr);
CAF_CHECK_EQUAL(a.begin(), a.end());
CAF_CHECK(a.empty());
}
CAF_TEST(resize) {
a.resize(512);
CAF_CHECK_EQUAL(a.size(), 512ul);
CAF_CHECK_EQUAL(a.capacity(), 512ul);
CAF_CHECK(a.data() != nullptr);
CAF_CHECK(!a.empty());
b.resize(512);
CAF_CHECK_EQUAL(b.size(), 512ul);
CAF_CHECK_EQUAL(b.capacity(), 1024ul);
CAF_CHECK(b.data() != nullptr);
CAF_CHECK(!b.empty());
a.resize(1024);
std::fill(a.begin(), a.end(), 'a');
auto cnt = 0;
CAF_CHECK(std::all_of(a.begin(), a.end(),
[&](char c) {
++cnt;
return c == 'a';
}));
CAF_CHECK_EQUAL(cnt, 1024);
a.resize(10);
cnt = 0;
CAF_CHECK(std::all_of(a.begin(), a.end(),
[&](char c) {
++cnt;
return c == 'a';
}));
CAF_CHECK_EQUAL(cnt, 10);
a.resize(1024);
cnt = 0;
CAF_CHECK(std::all_of(a.begin(), a.end(),
[&](char c) {
++cnt;
return c == 'a';
}));
CAF_CHECK_EQUAL(cnt, 1024);
}
CAF_TEST(push_back) {
for (auto c : vec)
a.push_back(c);
CAF_CHECK_EQUAL(vec.size(), a.size());
CAF_CHECK_EQUAL(a.capacity(), 8ul);
CAF_CHECK(a.data() != nullptr);
CAF_CHECK(!a.empty());
CAF_CHECK(std::equal(vec.begin(), vec.end(), a.begin()));
}
CAF_TEST(insert) {
for (auto c: vec)
a.insert(a.end(), c);
CAF_CHECK_EQUAL(*(a.data() + 0), 'h');
CAF_CHECK_EQUAL(*(a.data() + 1), 'a');
CAF_CHECK_EQUAL(*(a.data() + 2), 'l');
CAF_CHECK_EQUAL(*(a.data() + 3), 'l');
CAF_CHECK_EQUAL(*(a.data() + 4), 'o');
a.insert(a.begin(), '!');
CAF_CHECK_EQUAL(*(a.data() + 0), '!');
CAF_CHECK_EQUAL(*(a.data() + 1), 'h');
CAF_CHECK_EQUAL(*(a.data() + 2), 'a');
CAF_CHECK_EQUAL(*(a.data() + 3), 'l');
CAF_CHECK_EQUAL(*(a.data() + 4), 'l');
CAF_CHECK_EQUAL(*(a.data() + 5), 'o');
a.insert(a.begin() + 4, '-');
CAF_CHECK_EQUAL(*(a.data() + 0), '!');
CAF_CHECK_EQUAL(*(a.data() + 1), 'h');
CAF_CHECK_EQUAL(*(a.data() + 2), 'a');
CAF_CHECK_EQUAL(*(a.data() + 3), 'l');
CAF_CHECK_EQUAL(*(a.data() + 4), '-');
CAF_CHECK_EQUAL(*(a.data() + 5), 'l');
CAF_CHECK_EQUAL(*(a.data() + 6), 'o');
}
CAF_TEST(shrink_to_fit) {
a.shrink_to_fit();
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 0ul);
CAF_CHECK(a.data() == nullptr);
CAF_CHECK(a.empty());
}
CAF_TEST(swap) {
for (auto c : vec)
a.push_back(c);
std::swap(a, b);
CAF_CHECK_EQUAL(a.size(), 0ul);
CAF_CHECK_EQUAL(a.capacity(), 1024ul);
CAF_CHECK(a.data() != nullptr);
CAF_CHECK_EQUAL(b.size(), vec.size());
CAF_CHECK_EQUAL(std::distance(b.begin(), b.end()),
static_cast<receive_buffer::difference_type>(vec.size()));
CAF_CHECK_EQUAL(b.capacity(), 8ul);
CAF_CHECK(b.data() != nullptr);
CAF_CHECK_EQUAL(*(b.data() + 0), 'h');
CAF_CHECK_EQUAL(*(b.data() + 1), 'a');
CAF_CHECK_EQUAL(*(b.data() + 2), 'l');
CAF_CHECK_EQUAL(*(b.data() + 3), 'l');
CAF_CHECK_EQUAL(*(b.data() + 4), 'o');
}
CAF_TEST_FIXTURE_SCOPE_END()
...@@ -40,6 +40,7 @@ class config : public actor_system_config { ...@@ -40,6 +40,7 @@ class config : public actor_system_config {
public: public:
config() { config() {
load<io::middleman>(); load<io::middleman>();
set("middleman.enable-udp", true);
add_message_type<std::vector<int>>("std::vector<int>"); add_message_type<std::vector<int>>("std::vector<int>");
actor_system_config::parse(test::engine::argc(), actor_system_config::parse(test::engine::argc(),
test::engine::argv()); test::engine::argv());
...@@ -132,7 +133,7 @@ behavior linking_actor(event_based_actor* self, const actor& buddy) { ...@@ -132,7 +133,7 @@ behavior linking_actor(event_based_actor* self, const actor& buddy) {
CAF_TEST_FIXTURE_SCOPE(dynamic_remote_actor_tests, fixture) CAF_TEST_FIXTURE_SCOPE(dynamic_remote_actor_tests, fixture)
CAF_TEST(identity_semantics) { CAF_TEST(identity_semantics_tcp) {
// server side // server side
auto server = server_side.spawn(make_pong_behavior); auto server = server_side.spawn(make_pong_behavior);
CAF_EXP_THROW(port1, server_side_mm.publish(server, 0, local_host)); CAF_EXP_THROW(port1, server_side_mm.publish(server, 0, local_host));
...@@ -148,7 +149,7 @@ CAF_TEST(identity_semantics) { ...@@ -148,7 +149,7 @@ CAF_TEST(identity_semantics) {
anon_send_exit(server, exit_reason::user_shutdown); anon_send_exit(server, exit_reason::user_shutdown);
} }
CAF_TEST(ping_pong) { CAF_TEST(ping_pong_tcp) {
// server side // server side
CAF_EXP_THROW(port, CAF_EXP_THROW(port,
server_side_mm.publish(server_side.spawn(make_pong_behavior), server_side_mm.publish(server_side.spawn(make_pong_behavior),
...@@ -158,7 +159,7 @@ CAF_TEST(ping_pong) { ...@@ -158,7 +159,7 @@ CAF_TEST(ping_pong) {
client_side.spawn(make_ping_behavior, pong); client_side.spawn(make_ping_behavior, pong);
} }
CAF_TEST(custom_message_type) { CAF_TEST(custom_message_type_tcp) {
// server side // server side
CAF_EXP_THROW(port, server_side_mm.publish(server_side.spawn(make_sort_behavior), CAF_EXP_THROW(port, server_side_mm.publish(server_side.spawn(make_sort_behavior),
0, local_host)); 0, local_host));
...@@ -167,7 +168,7 @@ CAF_TEST(custom_message_type) { ...@@ -167,7 +168,7 @@ CAF_TEST(custom_message_type) {
client_side.spawn(make_sort_requester_behavior, sorter); client_side.spawn(make_sort_requester_behavior, sorter);
} }
CAF_TEST(remote_link) { CAF_TEST(remote_link_tcp) {
// server side // server side
CAF_EXP_THROW(port, server_side_mm.publish(server_side.spawn(fragile_mirror), CAF_EXP_THROW(port, server_side_mm.publish(server_side.spawn(fragile_mirror),
0, local_host)); 0, local_host));
...@@ -181,4 +182,112 @@ CAF_TEST(remote_link) { ...@@ -181,4 +182,112 @@ CAF_TEST(remote_link) {
CAF_MESSAGE("mirror exited"); CAF_MESSAGE("mirror exited");
} }
// same tests using UDP instead of TCP
CAF_TEST(identity_semantics_udp) {
// server side
auto server = server_side.spawn(make_pong_behavior);
CAF_EXP_THROW(port1, server_side_mm.publish_udp(server, 0, local_host));
CAF_EXP_THROW(port2, server_side_mm.publish_udp(server, 0, local_host));
CAF_REQUIRE_NOT_EQUAL(port1, port2);
CAF_EXP_THROW(same_server, server_side_mm.remote_actor_udp(local_host, port2));
CAF_REQUIRE_EQUAL(same_server, server);
CAF_CHECK_EQUAL(same_server->node(), server_side.node());
CAF_EXP_THROW(server1, client_side_mm.remote_actor_udp(local_host, port1));
CAF_EXP_THROW(server2, client_side_mm.remote_actor_udp(local_host, port2));
CAF_CHECK_EQUAL(server1, client_side_mm.remote_actor_udp(local_host, port1));
CAF_CHECK_EQUAL(server2, client_side_mm.remote_actor_udp(local_host, port2));
anon_send_exit(server, exit_reason::user_shutdown);
}
CAF_TEST(ping_pong_udp) {
// server side
CAF_EXP_THROW(port,
server_side_mm.publish_udp(server_side.spawn(make_pong_behavior),
0, local_host));
// client side
CAF_EXP_THROW(pong, client_side_mm.remote_actor_udp(local_host, port));
client_side.spawn(make_ping_behavior, pong);
}
CAF_TEST(custom_message_type_udp) {
// server side
CAF_EXP_THROW(port,
server_side_mm.publish_udp(server_side.spawn(make_sort_behavior),
0, local_host));
// client side
CAF_EXP_THROW(sorter, client_side_mm.remote_actor_udp(local_host, port));
client_side.spawn(make_sort_requester_behavior, sorter);
}
CAF_TEST(remote_link_udp) {
// server side
CAF_EXP_THROW(port,
server_side_mm.publish_udp(server_side.spawn(fragile_mirror),
0, local_host));
// client side
CAF_EXP_THROW(mirror, client_side_mm.remote_actor_udp(local_host, port));
auto linker = client_side.spawn(linking_actor, mirror);
scoped_actor self{client_side};
self->wait_for(linker);
CAF_MESSAGE("linker exited");
self->wait_for(mirror);
CAF_MESSAGE("mirror exited");
}
CAF_TEST(multiple_endpoints_udp) {
config cfg;
// setup server
CAF_MESSAGE("creating server");
actor_system server_sys{cfg};
auto mirror = server_sys.spawn([]() -> behavior {
return {
[] (std::string str) {
std::reverse(begin(str), end(str));
return str;
}
};
});
server_sys.middleman().publish_udp(mirror, 12345);
auto client_fun = [](event_based_actor* self) -> behavior {
return {
[=](actor s) {
self->send(s, "hellow, world");
},
[=](const std::string& str) {
CAF_CHECK_EQUAL(str, "dlrow ,wolleh");
self->quit();
CAF_MESSAGE("done");
}
};
};
// setup client a
CAF_MESSAGE("creating first client");
config client_cfg;
actor_system client_sys{client_cfg};
auto client = client_sys.spawn(client_fun);
// acquire remote actor from the server
auto client_srv = client_sys.middleman().remote_actor_udp("localhost", 12345);
CAF_REQUIRE(client_srv);
// setup other clients
for (int i = 0; i < 5; ++i) {
config other_cfg;
actor_system other_sys{other_cfg};
CAF_MESSAGE("creating new client");
auto other = other_sys.spawn(client_fun);
// acquire remote actor from the server
auto other_srv = other_sys.middleman().remote_actor_udp("localhost", 12345);
CAF_REQUIRE(other_srv);
// establish communication and exit
CAF_MESSAGE("client contacts server and exits");
anon_send(other, *other_srv);
other_sys.await_all_actors_done();
}
// establish communication and exit
CAF_MESSAGE("first client contacts server and exits");
anon_send(client, *client_srv);
client_sys.await_all_actors_done();
anon_send_exit(mirror, exit_reason::user_shutdown);
}
CAF_TEST_FIXTURE_SCOPE_END() CAF_TEST_FIXTURE_SCOPE_END()
...@@ -116,7 +116,7 @@ CAF_TEST(unpublishing) { ...@@ -116,7 +116,7 @@ CAF_TEST(unpublishing) {
CAF_MESSAGE("check whether testee is still available via cache"); CAF_MESSAGE("check whether testee is still available via cache");
auto x1 = remote_actor("127.0.0.1", port); auto x1 = remote_actor("127.0.0.1", port);
CAF_CHECK_EQUAL(x1, testee); CAF_CHECK_EQUAL(x1, testee);
CAF_MESSAGE("fake dead of testee and check if testee becomes unavailable"); CAF_MESSAGE("fake death of testee and check if testee becomes unavailable");
anon_send(actor_cast<actor>(system.middleman().actor_handle()), anon_send(actor_cast<actor>(system.middleman().actor_handle()),
down_msg{testee.address(), exit_reason::normal}); down_msg{testee.address(), exit_reason::normal});
// must fail now // must fail now
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment