Commit 303e1841 authored by Dominik Charousset's avatar Dominik Charousset

ipv4 networking bugfixes and improvements

parent de0c81ce
...@@ -39,22 +39,21 @@ ...@@ -39,22 +39,21 @@
namespace cppa { namespace detail { namespace cppa { namespace detail {
template<size_t ChunkSize, size_t MaxBufferSize, typename DataType = char> template<size_t ChunkSize, size_t MaxBufferSize>
class buffer { class buffer {
DataType* m_data; char* m_data;
size_t m_written; size_t m_written;
size_t m_allocated; size_t m_allocated;
size_t m_final_size; size_t m_final_size;
public: public:
buffer() : m_data(nullptr), m_written(0), m_allocated(0), m_final_size(0) { buffer() : m_data(nullptr), m_written(0), m_allocated(0), m_final_size(0) {}
}
buffer(buffer&& other) buffer(buffer&& other)
: m_data(other.m_data), m_written(other.m_written) : m_data(other.m_data), m_written(other.m_written)
, m_allocated(other.m_allocated), m_final_size(other.m_final_size) { , m_allocated(other.m_allocated), m_final_size(other.m_final_size) {
other.m_data = nullptr; other.m_data = nullptr;
other.m_written = other.m_allocated = other.m_final_size = 0; other.m_written = other.m_allocated = other.m_final_size = 0;
} }
...@@ -68,12 +67,17 @@ class buffer { ...@@ -68,12 +67,17 @@ class buffer {
} }
void reset(size_t new_final_size = 0) { void reset(size_t new_final_size = 0) {
if (new_final_size > MaxBufferSize) {
m_written = 0;
m_allocated = 0;
m_final_size = 0;
delete[] m_data;
m_data = nullptr;
throw std::ios_base::failure("maximum buffer size exceeded");
}
m_written = 0; m_written = 0;
m_final_size = new_final_size; m_final_size = new_final_size;
if (new_final_size > m_allocated) { if (new_final_size > m_allocated) {
if (new_final_size > MaxBufferSize) {
throw std::ios_base::failure("maximum buffer size exceeded");
}
auto remainder = (new_final_size % ChunkSize); auto remainder = (new_final_size % ChunkSize);
if (remainder == 0) { if (remainder == 0) {
m_allocated = new_final_size; m_allocated = new_final_size;
...@@ -82,16 +86,12 @@ class buffer { ...@@ -82,16 +86,12 @@ class buffer {
m_allocated = (new_final_size - remainder) + ChunkSize; m_allocated = (new_final_size - remainder) + ChunkSize;
} }
delete[] m_data; delete[] m_data;
m_data = new DataType[m_allocated]; m_data = new char[m_allocated];
} }
} }
bool ready() {
return m_written == m_final_size;
}
// pointer to the current write position // pointer to the current write position
DataType* wr_ptr() { char* wr_ptr() {
return m_data + m_written; return m_data + m_written;
} }
...@@ -111,7 +111,7 @@ class buffer { ...@@ -111,7 +111,7 @@ class buffer {
m_written += value; m_written += value;
} }
DataType* data() { char* data() {
return m_data; return m_data;
} }
...@@ -119,13 +119,12 @@ class buffer { ...@@ -119,13 +119,12 @@ class buffer {
return remaining() == 0; return remaining() == 0;
} }
bool append_from(util::input_stream* istream) { void append_from(util::input_stream* istream) {
CPPA_REQUIRE(remaining() > 0);
auto num_bytes = istream->read_some(wr_ptr(), remaining()); auto num_bytes = istream->read_some(wr_ptr(), remaining());
if (num_bytes > 0) { if (num_bytes > 0) {
inc_written(num_bytes); inc_written(num_bytes);
return true;
} }
return false;
} }
}; };
......
...@@ -42,7 +42,7 @@ class ipv4_io_stream : public util::io_stream { ...@@ -42,7 +42,7 @@ class ipv4_io_stream : public util::io_stream {
static util::io_stream_ptr connect_to(const char* host, std::uint16_t port); static util::io_stream_ptr connect_to(const char* host, std::uint16_t port);
ipv4_io_stream(native_socket_type fd); static util::io_stream_ptr from_native_socket(native_socket_type fd);
native_socket_type read_file_handle() const; native_socket_type read_file_handle() const;
...@@ -58,6 +58,8 @@ class ipv4_io_stream : public util::io_stream { ...@@ -58,6 +58,8 @@ class ipv4_io_stream : public util::io_stream {
private: private:
ipv4_io_stream(native_socket_type fd);
native_socket_type m_fd; native_socket_type m_fd;
}; };
......
...@@ -34,8 +34,13 @@ ...@@ -34,8 +34,13 @@
#include "cppa/any_tuple.hpp" #include "cppa/any_tuple.hpp"
#include "cppa/actor_proxy.hpp" #include "cppa/actor_proxy.hpp"
#include "cppa/process_information.hpp" #include "cppa/process_information.hpp"
#include "cppa/detail/addressed_message.hpp"
#include "cppa/util/acceptor.hpp" #include "cppa/util/acceptor.hpp"
#include "cppa/detail/network_manager.hpp"
#include "cppa/detail/singleton_manager.hpp"
#include "cppa/detail/addressed_message.hpp"
#include "cppa/intrusive/single_reader_queue.hpp" #include "cppa/intrusive/single_reader_queue.hpp"
namespace cppa { namespace detail { namespace cppa { namespace detail {
...@@ -63,7 +68,23 @@ struct mm_message { ...@@ -63,7 +68,23 @@ struct mm_message {
} }
}; };
void mailman_loop(); void mailman_loop(intrusive::single_reader_queue<mm_message>& q);
template<typename... Args>
inline void send2mm(Args&&... args) {
auto nm = singleton_manager::get_network_manager();
nm->send_to_mailman(mm_message::create(std::forward<Args>(args)...));
}
inline void mailman_enqueue(process_information_ptr peer,
addressed_message outgoing_message) {
send2mm(std::move(peer), std::move(outgoing_message));
}
inline void mailman_add_peer(util::io_stream_ptr_pair peer_streams,
process_information_ptr peer_ptr ) {
send2mm(std::move(peer_streams), std::move(peer_ptr));
}
}} // namespace cppa::detail }} // namespace cppa::detail
......
...@@ -76,12 +76,14 @@ struct po_message { ...@@ -76,12 +76,14 @@ struct po_message {
} }
}; };
void post_office_loop(int input_fd, intrusive::single_reader_queue<po_message>&); typedef intrusive::single_reader_queue<po_message> po_message_queue;
void post_office_loop(int input_fd, po_message_queue&);
template<typename... Args> template<typename... Args>
inline void send2po(Args&&... args) { inline void send2po(Args&&... args) {
auto nm = singleton_manager::get_network_manager(); auto nm = singleton_manager::get_network_manager();
nm->send_to_post_office(std::unique_ptr<po_message>(new po_message(std::forward<Args>(args)...))); nm->send_to_post_office(po_message::create(std::forward<Args>(args)...));
} }
inline void post_office_add_peer(util::io_stream_ptr_pair peer_streams, inline void post_office_add_peer(util::io_stream_ptr_pair peer_streams,
......
...@@ -97,7 +97,7 @@ bool accept_impl(util::io_stream_ptr_pair& result, native_socket_type fd, bool n ...@@ -97,7 +97,7 @@ bool accept_impl(util::io_stream_ptr_pair& result, native_socket_type fd, bool n
} }
int flags = 1; int flags = 1;
setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int)); setsockopt(sfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int));
util::io_stream_ptr ptr(new ipv4_io_stream(sfd)); util::io_stream_ptr ptr(ipv4_io_stream::from_native_socket(sfd));
result.first = ptr; result.first = ptr;
result.second = ptr; result.second = ptr;
return true; return true;
......
...@@ -53,10 +53,12 @@ namespace { ...@@ -53,10 +53,12 @@ namespace {
template<typename T> template<typename T>
void handle_syscall_result(T result, size_t num_bytes, bool nonblocking) { void handle_syscall_result(T result, size_t num_bytes, bool nonblocking) {
if (result < 0) { if (result < 0) {
if (!nonblocking || errno != EAGAIN) { if (!nonblocking || (errno != EAGAIN && errno != EWOULDBLOCK)) {
char* cstr = strerror(errno); char* cstr = strerror(errno);
std::string errmsg = cstr; std::string errmsg = cstr;
free(cstr); errmsg += " [errno = ";
errmsg += std::to_string(errno);
errmsg += "]";
throw std::ios_base::failure(std::move(errmsg)); throw std::ios_base::failure(std::move(errmsg));
} }
} }
...@@ -68,6 +70,23 @@ void handle_syscall_result(T result, size_t num_bytes, bool nonblocking) { ...@@ -68,6 +70,23 @@ void handle_syscall_result(T result, size_t num_bytes, bool nonblocking) {
} }
} }
int rd_flags(native_socket_type fd) {
auto flags = fcntl(fd, F_GETFL, 0);
if (flags == -1) {
throw network_error("unable to read socket flags");
}
return flags;
}
void set_nonblocking(native_socket_type fd) {
auto flags = rd_flags(fd);
if ((flags & O_NONBLOCK) != 0) {
if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) {
throw network_error("unable to set socket to nonblock");
}
}
}
} // namespace <anonymous> } // namespace <anonymous>
ipv4_io_stream::ipv4_io_stream(native_socket_type fd) : m_fd(fd) { } ipv4_io_stream::ipv4_io_stream(native_socket_type fd) : m_fd(fd) { }
...@@ -81,25 +100,45 @@ native_socket_type ipv4_io_stream::write_file_handle() const { ...@@ -81,25 +100,45 @@ native_socket_type ipv4_io_stream::write_file_handle() const {
} }
void ipv4_io_stream::read(void* buf, size_t len) { void ipv4_io_stream::read(void* buf, size_t len) {
handle_syscall_result(::recv(m_fd, buf, len, 0), len, false); handle_syscall_result(::recv(m_fd, buf, len, MSG_WAITALL), len, false);
} }
size_t ipv4_io_stream::read_some(void* buf, size_t len) { size_t ipv4_io_stream::read_some(void* buf, size_t len) {
auto recv_result = ::recv(m_fd, buf, len, MSG_DONTWAIT); auto recv_result = ::recv(m_fd, buf, len, 0);
handle_syscall_result(recv_result, len, true); handle_syscall_result(recv_result, len, true);
return static_cast<size_t>(recv_result); return (recv_result > 0) ? static_cast<size_t>(recv_result) : 0;
} }
void ipv4_io_stream::write(const void* buf, size_t len) { void ipv4_io_stream::write(const void* vbuf, size_t len) {
handle_syscall_result(::send(m_fd, buf, len, 0), len, false); auto buf = reinterpret_cast<const char*>(vbuf);
size_t written = 0;
while (written < len) {
auto send_result = ::send(m_fd, buf + written, len - written, 0);
handle_syscall_result(send_result, len - written, true);
written += static_cast<size_t>(send_result);
if (written < len) {
// block until socked is writable again
fd_set writeset;
FD_ZERO(&writeset);
FD_SET(m_fd, &writeset);
if (select(m_fd + 1, nullptr, &writeset, nullptr, nullptr) < 0) {
throw network_error("select() failed");
}
}
}
} }
size_t ipv4_io_stream::write_some(const void* buf, size_t len) { size_t ipv4_io_stream::write_some(const void* buf, size_t len) {
auto send_result = ::send(m_fd, buf, len, MSG_DONTWAIT); auto send_result = ::send(m_fd, buf, len, 0);
handle_syscall_result(send_result, len, true); handle_syscall_result(send_result, len, true);
return static_cast<size_t>(send_result); return static_cast<size_t>(send_result);
} }
util::io_stream_ptr ipv4_io_stream::from_native_socket(native_socket_type fd) {
set_nonblocking(fd);
return new ipv4_io_stream(fd);
}
util::io_stream_ptr ipv4_io_stream::connect_to(const char* host, util::io_stream_ptr ipv4_io_stream::connect_to(const char* host,
std::uint16_t port) { std::uint16_t port) {
native_socket_type sockfd; native_socket_type sockfd;
...@@ -124,6 +163,7 @@ util::io_stream_ptr ipv4_io_stream::connect_to(const char* host, ...@@ -124,6 +163,7 @@ util::io_stream_ptr ipv4_io_stream::connect_to(const char* host,
} }
int flags = 1; int flags = 1;
setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int)); setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int));
set_nonblocking(sockfd);
return new ipv4_io_stream(sockfd); return new ipv4_io_stream(sockfd);
} }
......
...@@ -103,66 +103,68 @@ mm_message::~mm_message() { ...@@ -103,66 +103,68 @@ mm_message::~mm_message() {
} }
} }
void mailman_loop() { void mailman_loop(intrusive::single_reader_queue<mm_message>& q) {
bool done = false; bool done = false;
// serializes outgoing messages // serializes outgoing messages
binary_serializer bs; binary_serializer bs;
// connected tcp peers // connected tcp peers
std::map<process_information, native_socket_type> peers; std::map<process_information, util::io_stream_ptr_pair> peers;
do_receive ( std::unique_ptr<mm_message> msg;
on_arg_match >> [&](process_information_ptr target_peer, addressed_message msg) { auto fetch_next = [&] { msg.reset(q.pop()); };
auto i = peers.find(*target_peer); for (fetch_next(); !done; fetch_next()) {
if (i != peers.end()) { switch (msg->type) {
bool disconnect_peer = false; case mm_message_type::outgoing_message: {
auto peer_fd = i->second; auto& target_peer = msg->out_msg.first;
try { auto& out_msg = msg->out_msg.second;
bs << msg; CPPA_REQUIRE(target_peer != nullptr);
DEBUG("--> " << to_string(msg)); auto i = peers.find(*target_peer);
auto sent = ::send(peer_fd, bs.sendable_data(), bs.sendable_size(), 0); if (i != peers.end()) {
if (sent < 0 || static_cast<size_t>(sent) != bs.sendable_size()) { bool disconnect_peer = false;
try {
bs << out_msg;
DEBUG("--> " << to_string(out_msg));
DEBUG("outgoing message size: " << bs.size());
i->second.second->write(bs.sendable_data(),
bs.sendable_size());
}
// something went wrong; close connection to this peer
catch (std::exception& e) {
DEBUG(to_uniform_name(typeid(e)) << ": " << e.what());
disconnect_peer = true; disconnect_peer = true;
DEBUG("too few bytes written");
} }
if (disconnect_peer) {
DEBUG("peer disconnected (error during send)");
//closesocket(peer);
//post_office_close_socket(peer_fd);
peers.erase(i);
}
bs.reset();
} }
// something went wrong; close connection to this peer else {
catch (std::exception& e) { DEBUG("message to an unknown peer: " << to_string(out_msg));
DEBUG(to_uniform_name(typeid(e)) << ": " << e.what());
disconnect_peer = true;
}
if (disconnect_peer) {
DEBUG("peer disconnected (error during send)");
//closesocket(peer);
//post_office_close_socket(peer_fd);
peers.erase(i);
} }
bs.reset(); break;
}
else {
DEBUG("message to an unknown peer: " << to_string(msg));
} }
}, case mm_message_type::add_peer: {
on_arg_match >> [&](native_socket_type sockfd, process_information_ptr pinfo) { DEBUG("mailman: add_peer");
DEBUG("mailman: add_peer"); auto& iopair = msg->peer.first;
auto i = peers.find(*pinfo); auto& pinfo = msg->peer.second;
if (i == peers.end()) { auto i = peers.find(*pinfo);
//cout << "mailman added " << pjob.pinfo->process_id() << "@" if (i == peers.end()) {
// << to_string(pjob.pinfo->node_id()) << endl; //cout << "mailman added " << pjob.pinfo->process_id() << "@"
peers.insert(std::make_pair(*pinfo, sockfd)); // << to_string(pjob.pinfo->node_id()) << endl;
peers.insert(std::make_pair(*pinfo, iopair));
}
else {
DEBUG("add_peer failed: peer already known");
}
break;
} }
else { case mm_message_type::shutdown: {
DEBUG("add_peer_job failed: peer already known"); done = true;
} }
},
on(atom("DONE")) >> [&]() {
done = true;
},
others() >> [&]() {
std::string str = "unexpected message in post_office: ";
str += to_string(self->last_dequeued());
CPPA_CRITICAL(str.c_str());
} }
) }
.until(gref(done));
} }
} } // namespace cppa::detail } } // namespace cppa::detail
...@@ -67,19 +67,14 @@ struct network_manager_impl : network_manager { ...@@ -67,19 +67,14 @@ struct network_manager_impl : network_manager {
if (pipe(pipe_fd) != 0) { if (pipe(pipe_fd) != 0) {
CPPA_CRITICAL("cannot create pipe"); CPPA_CRITICAL("cannot create pipe");
} }
// create actors // store pipe read handle in local variables for lambda expression
//m_post_office.reset(new thread_mapped_actor);
//m_mailman.reset(new thread_mapped_actor);
// store some data in local variables for lambdas
int pipe_fd0 = pipe_fd[0]; int pipe_fd0 = pipe_fd[0];
// start threads // start threads
m_post_office_thread = std::thread([this, pipe_fd0] { m_post_office_thread = std::thread([this, pipe_fd0] {
//scoped_self_setter sss{po_ptr.get()};
post_office_loop(pipe_fd0, this->m_post_office_queue); post_office_loop(pipe_fd0, this->m_post_office_queue);
}); });
m_mailman_thread = std::thread([] { m_mailman_thread = std::thread([this] {
//scoped_self_setter sss{mm_ptr.get()}; mailman_loop(this->m_mailman_queue);
mailman_loop();
}); });
} }
......
This diff is collapsed.
...@@ -65,142 +65,42 @@ using std::endl; ...@@ -65,142 +65,42 @@ using std::endl;
namespace cppa { namespace cppa {
/* void publish(actor_ptr whom, std::unique_ptr<util::acceptor> acceptor) {
namespace { if (!whom && !acceptor) return;
void read_from_socket(native_socket_type sfd, void* buf, size_t buf_size) {
char* cbuf = reinterpret_cast<char*>(buf);
size_t read_bytes = 0;
size_t left = buf_size;
int rres = 0;
size_t urres = 0;
do {
rres = ::recv(sfd, cbuf + read_bytes, left, 0);
if (rres <= 0) {
throw std::ios_base::failure("cannot read from closed socket");
}
urres = static_cast<size_t>(rres);
read_bytes += urres;
left -= urres;
}
while (urres < left);
}
} // namespace <anonmyous>
struct socket_guard {
bool m_released;
native_socket_type m_socket;
public:
socket_guard(native_socket_type sfd) : m_released(false), m_socket(sfd) {
}
~socket_guard() {
if (!m_released) detail::closesocket(m_socket);
}
void release() {
m_released = true;
}
};
*/
void publish(actor_ptr whom, std::uint16_t port) {
if (!whom) return;
// throws on error
auto ptr = detail::ipv4_acceptor::create(port);
detail::singleton_manager::get_actor_registry()->put(whom->id(), whom); detail::singleton_manager::get_actor_registry()->put(whom->id(), whom);
detail::post_office_publish(std::move(ptr), whom); detail::post_office_publish(std::move(acceptor), whom);
/*
native_socket_type sockfd;
struct sockaddr_in serv_addr;
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == detail::invalid_socket) {
throw network_error("could not create server socket");
}
// sguard closes the socket if an exception occurs
socket_guard sguard(sockfd);
memset((char*) &serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(port);
if (bind(sockfd, (struct sockaddr*) &serv_addr, sizeof(serv_addr)) < 0) {
throw bind_failure(errno);
}
if (listen(sockfd, 10) != 0) {
throw network_error("listen() failed");
}
int flags = fcntl(sockfd, F_GETFL, 0);
if (flags == -1) {
throw network_error("unable to get socket flags");
}
if (fcntl(sockfd, F_SETFL, flags | O_NONBLOCK) < 0) {
throw network_error("unable to set socket to nonblock");
}
flags = 1;
setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int));
// ok, no exceptions
sguard.release();
detail::post_office_publish(sockfd, whom);
*/
} }
actor_ptr remote_actor(const char* host, std::uint16_t port) { actor_ptr remote_actor(util::io_stream_ptr_pair peer) {
/*
native_socket_type sockfd;
struct sockaddr_in serv_addr;
struct hostent* server;
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd == detail::invalid_socket) {
throw network_error("socket creation failed");
}
server = gethostbyname(host);
if (!server) {
std::string errstr = "no such host: ";
errstr += host;
throw network_error(std::move(errstr));
}
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
memmove(&serv_addr.sin_addr.s_addr, server->h_addr, server->h_length);
serv_addr.sin_port = htons(port);
if (connect(sockfd, (const sockaddr*) &serv_addr, sizeof(serv_addr)) != 0) {
throw network_error("could not connect to host");
}
*/
auto pinf = process_information::get(); auto pinf = process_information::get();
std::uint32_t process_id = pinf->process_id(); std::uint32_t process_id = pinf->process_id();
/*int flags = 1;
setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int));
*/
// throws on error // throws on error
util::io_stream_ptr peer = detail::ipv4_io_stream::connect_to(host, port); peer.second->write(&process_id, sizeof(std::uint32_t));
peer->write(&process_id, sizeof(std::uint32_t)); peer.second->write(pinf->node_id().data(), pinf->node_id().size());
peer->write(pinf->node_id().data(), pinf->node_id().size());
std::uint32_t remote_actor_id; std::uint32_t remote_actor_id;
std::uint32_t peer_pid; std::uint32_t peer_pid;
process_information::node_id_type peer_node_id; process_information::node_id_type peer_node_id;
peer->read(&remote_actor_id, sizeof(remote_actor_id)); peer.first->read(&remote_actor_id, sizeof(remote_actor_id));
peer->read(&peer_pid, sizeof(std::uint32_t)); peer.first->read(&peer_pid, sizeof(std::uint32_t));
peer->read(peer_node_id.data(), peer_node_id.size()); peer.first->read(peer_node_id.data(), peer_node_id.size());
auto peer_pinf = new process_information(peer_pid, peer_node_id); process_information_ptr pinfptr(new process_information(peer_pid, peer_node_id));
process_information_ptr pinfptr(peer_pinf);
//auto key = std::make_tuple(remote_actor_id, pinfptr->process_id(), pinfptr->node_id()); //auto key = std::make_tuple(remote_actor_id, pinfptr->process_id(), pinfptr->node_id());
util::io_stream_ptr_pair io_ptrs(peer, peer); detail::mailman_add_peer(peer, pinfptr);
//detail::singleton_manager::get_network_manager() detail::post_office_add_peer(peer, pinfptr);
//->send_to_mailman(make_any_tuple(util::io_stream_ptr_pair(peer, peer),
// pinfptr));
detail::post_office_add_peer(io_ptrs, pinfptr);
return detail::get_actor_proxy_cache().get(remote_actor_id, return detail::get_actor_proxy_cache().get(remote_actor_id,
pinfptr->process_id(), pinfptr->process_id(),
pinfptr->node_id()); pinfptr->node_id());
//auto ptr = get_scheduler()->register_hidden_context(); }
void publish(actor_ptr whom, std::uint16_t port) {
if (whom) publish(whom, detail::ipv4_acceptor::create(port));
}
actor_ptr remote_actor(const char* host, std::uint16_t port) {
// throws on error
util::io_stream_ptr peer = detail::ipv4_io_stream::connect_to(host, port);
util::io_stream_ptr_pair ptrpair(peer, peer);
return remote_actor(ptrpair);
} }
} // namespace cppa } // namespace cppa
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment