Commit a2418241 authored by Matthias Vallentin's avatar Matthias Vallentin

Write out sequences in variable byte encoding

This optimization pays is a common one in binary serialization
protocols. We lifted the implementation directly from VAST's
serialization framework. The optimization pays off when a sequence has
less than 2^21 elements, which is the common case.

Moreover, with this change we treat strings as orindary sequences.
Consequently, strings are now binary-compatible with byte vectors, in
case that ever turns out to be important.
parent b23455db
...@@ -42,13 +42,9 @@ namespace caf { ...@@ -42,13 +42,9 @@ namespace caf {
/// Implements the deserializer interface with a binary serialization protocol. /// Implements the deserializer interface with a binary serialization protocol.
template <class Streambuf> template <class Streambuf>
class stream_deserializer : public deserializer { class stream_deserializer : public deserializer {
static_assert( using streambuf_type = typename std::remove_reference<Streambuf>::type;
std::is_base_of< static_assert(std::is_base_of<std::streambuf, streambuf_type>::value,
std::streambuf, "Streambuf must inherit from std::streambuf");
typename std::remove_reference<Streambuf>::type
>::value,
"Streambuf must inherit from std::streambuf"
);
public: public:
template <class... Ts> template <class... Ts>
...@@ -88,11 +84,7 @@ public: ...@@ -88,11 +84,7 @@ public:
} }
void begin_sequence(size_t& num_elements) override { void begin_sequence(size_t& num_elements) override {
static_assert(sizeof(size_t) >= sizeof(uint32_t), varbyte_decode(num_elements);
"sizeof(size_t) < sizeof(uint32_t)");
uint32_t tmp;
apply_int(tmp);
num_elements = static_cast<size_t>(tmp);
} }
void end_sequence() override { void end_sequence() override {
...@@ -105,6 +97,25 @@ public: ...@@ -105,6 +97,25 @@ public:
} }
protected: protected:
// Decode an unsigned integral type as variable-byte-encoded byte sequence.
template <class T>
size_t varbyte_decode(T& x) {
static_assert(std::is_unsigned<T>::value, "T must be an unsigned type");
auto n = T{0};
x = 0;
uint8_t low7;
do {
auto c = streambuf_.sbumpc();
using traits = typename streambuf_type::traits_type;
if (traits::eq_int_type(c, traits::eof()))
throw std::out_of_range{"stream_deserializer<T>::begin_sequence"};
low7 = static_cast<uint8_t>(traits::to_char_type(c));
x |= (low7 & 0x7F) << (7 * n);
++n;
} while (low7 & 0x80);
return n;
}
void apply_builtin(builtin type, void* val) override { void apply_builtin(builtin type, void* val) override {
CAF_ASSERT(val != nullptr); CAF_ASSERT(val != nullptr);
switch (type) { switch (type) {
...@@ -141,37 +152,43 @@ protected: ...@@ -141,37 +152,43 @@ protected:
} }
case string8_v: { case string8_v: {
auto& str = *reinterpret_cast<std::string*>(val); auto& str = *reinterpret_cast<std::string*>(val);
uint32_t str_size; size_t str_size;
apply_int(str_size); begin_sequence(str_size);
range_check(str_size); range_check(str_size);
str.resize(str_size); str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a // TODO: When using C++14, switch to str.data(), which then has a
// non-const overload. // non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), str_size); size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), str_size);
CAF_ASSERT(n == str_size);
end_sequence();
break; break;
} }
case string16_v: { case string16_v: {
auto& str = *reinterpret_cast<std::u16string*>(val); auto& str = *reinterpret_cast<std::u16string*>(val);
uint32_t str_size; size_t str_size;
apply_int(str_size); begin_sequence(str_size);
auto bytes = str_size * sizeof(std::u16string::value_type); auto bytes = str_size * sizeof(std::u16string::value_type);
range_check(bytes); range_check(bytes);
str.resize(str_size); str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a // TODO: When using C++14, switch to str.data(), which then has a
// non-const overload. // non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes); size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
CAF_ASSERT(n == bytes);
end_sequence();
break; break;
} }
case string32_v: { case string32_v: {
auto& str = *reinterpret_cast<std::u32string*>(val); auto& str = *reinterpret_cast<std::u32string*>(val);
uint32_t str_size; size_t str_size;
apply_int(str_size); begin_sequence(str_size);
auto bytes = str_size * sizeof(std::u32string::value_type); auto bytes = str_size * sizeof(std::u32string::value_type);
range_check(bytes); range_check(bytes);
str.resize(str_size); str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a // TODO: When using C++14, switch to str.data(), which then has a
// non-const overload. // non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes); size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
CAF_ASSERT(n == bytes);
end_sequence();
break; break;
} }
} }
......
...@@ -40,13 +40,9 @@ namespace caf { ...@@ -40,13 +40,9 @@ namespace caf {
/// Implements the serializer interface with a binary serialization protocol. /// Implements the serializer interface with a binary serialization protocol.
template <class Streambuf> template <class Streambuf>
class stream_serializer : public serializer { class stream_serializer : public serializer {
static_assert( using streambuf_type = typename std::remove_reference<Streambuf>::type;
std::is_base_of< static_assert(std::is_base_of<std::streambuf, streambuf_type>::value,
std::streambuf, "Streambuf must inherit from std::streambuf");
typename std::remove_reference<Streambuf>::type
>::value,
"Streambuf must inherit from std::streambuf"
);
public: public:
template <class... Ts> template <class... Ts>
...@@ -86,8 +82,7 @@ public: ...@@ -86,8 +82,7 @@ public:
} }
void begin_sequence(size_t& list_size) override { void begin_sequence(size_t& list_size) override {
auto s = static_cast<uint32_t>(list_size); varbyte_encode(list_size);
apply(s);
} }
void end_sequence() override { void end_sequence() override {
...@@ -99,6 +94,23 @@ public: ...@@ -99,6 +94,23 @@ public:
} }
protected: protected:
// Encode an unsigned integral type as variable-byte sequence.
template <class T>
size_t varbyte_encode(T x) {
static_assert(std::is_unsigned<T>::value, "T must be an unsigned type");
// For 64-bit values, the encoded representation cannot get larger than 10
// bytes. A scratch space of 16 bytes suffices as upper bound.
uint8_t buf[16];
auto i = buf;
while (x > 0x7f) {
*i++ = (static_cast<uint8_t>(x) & 0x7f) | 0x80;
x >>= 7;
}
*i++ = static_cast<uint8_t>(x) & 0x7f;
using char_type = typename streambuf_type::traits_type::char_type;
return streambuf_.sputn(reinterpret_cast<char_type*>(buf), i - buf);
}
void apply_builtin(builtin type, void* val) override { void apply_builtin(builtin type, void* val) override {
CAF_ASSERT(val != nullptr); CAF_ASSERT(val != nullptr);
switch (type) { switch (type) {
...@@ -138,31 +150,34 @@ protected: ...@@ -138,31 +150,34 @@ protected:
} }
case string8_v: { case string8_v: {
auto str = reinterpret_cast<std::string*>(val); auto str = reinterpret_cast<std::string*>(val);
auto s = static_cast<uint32_t>(str->size()); auto s = str->size();
apply(s); begin_sequence(s);
apply_raw(str->size(), const_cast<char*>(str->c_str())); apply_raw(str->size(), const_cast<char*>(str->c_str()));
end_sequence();
break; break;
} }
case string16_v: { case string16_v: {
auto& str = *reinterpret_cast<std::u16string*>(val); auto str = reinterpret_cast<std::u16string*>(val);
auto s = static_cast<uint32_t>(str.size()); auto s = str->size();
apply(s); begin_sequence(s);
for (auto c : str) { for (auto c : *str) {
// the standard does not guarantee that char16_t is exactly 16 bits... // the standard does not guarantee that char16_t is exactly 16 bits...
auto tmp = static_cast<uint16_t>(c); auto tmp = static_cast<uint16_t>(c);
apply_raw(sizeof(tmp), &tmp); apply_raw(sizeof(tmp), &tmp);
} }
end_sequence();
break; break;
} }
case string32_v: { case string32_v: {
auto& str = *reinterpret_cast<std::u32string*>(val); auto str = reinterpret_cast<std::u32string*>(val);
auto s = static_cast<uint32_t>(str.size()); auto s = str->size();
apply(s); begin_sequence(s);
for (auto c : str) { for (auto c : *str) {
// the standard does not guarantee that char32_t is exactly 32 bits... // the standard does not guarantee that char32_t is exactly 32 bits...
auto tmp = static_cast<uint32_t>(c); auto tmp = static_cast<uint32_t>(c);
apply_raw(sizeof(tmp), &tmp); apply_raw(sizeof(tmp), &tmp);
} }
end_sequence();
break; break;
} }
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment