Commit a2418241 authored by Matthias Vallentin's avatar Matthias Vallentin

Write out sequences in variable byte encoding

This optimization pays is a common one in binary serialization
protocols. We lifted the implementation directly from VAST's
serialization framework. The optimization pays off when a sequence has
less than 2^21 elements, which is the common case.

Moreover, with this change we treat strings as orindary sequences.
Consequently, strings are now binary-compatible with byte vectors, in
case that ever turns out to be important.
parent b23455db
......@@ -42,13 +42,9 @@ namespace caf {
/// Implements the deserializer interface with a binary serialization protocol.
template <class Streambuf>
class stream_deserializer : public deserializer {
static_assert(
std::is_base_of<
std::streambuf,
typename std::remove_reference<Streambuf>::type
>::value,
"Streambuf must inherit from std::streambuf"
);
using streambuf_type = typename std::remove_reference<Streambuf>::type;
static_assert(std::is_base_of<std::streambuf, streambuf_type>::value,
"Streambuf must inherit from std::streambuf");
public:
template <class... Ts>
......@@ -88,11 +84,7 @@ public:
}
void begin_sequence(size_t& num_elements) override {
static_assert(sizeof(size_t) >= sizeof(uint32_t),
"sizeof(size_t) < sizeof(uint32_t)");
uint32_t tmp;
apply_int(tmp);
num_elements = static_cast<size_t>(tmp);
varbyte_decode(num_elements);
}
void end_sequence() override {
......@@ -105,6 +97,25 @@ public:
}
protected:
// Decode an unsigned integral type as variable-byte-encoded byte sequence.
template <class T>
size_t varbyte_decode(T& x) {
static_assert(std::is_unsigned<T>::value, "T must be an unsigned type");
auto n = T{0};
x = 0;
uint8_t low7;
do {
auto c = streambuf_.sbumpc();
using traits = typename streambuf_type::traits_type;
if (traits::eq_int_type(c, traits::eof()))
throw std::out_of_range{"stream_deserializer<T>::begin_sequence"};
low7 = static_cast<uint8_t>(traits::to_char_type(c));
x |= (low7 & 0x7F) << (7 * n);
++n;
} while (low7 & 0x80);
return n;
}
void apply_builtin(builtin type, void* val) override {
CAF_ASSERT(val != nullptr);
switch (type) {
......@@ -141,37 +152,43 @@ protected:
}
case string8_v: {
auto& str = *reinterpret_cast<std::string*>(val);
uint32_t str_size;
apply_int(str_size);
size_t str_size;
begin_sequence(str_size);
range_check(str_size);
str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a
// non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), str_size);
size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), str_size);
CAF_ASSERT(n == str_size);
end_sequence();
break;
}
case string16_v: {
auto& str = *reinterpret_cast<std::u16string*>(val);
uint32_t str_size;
apply_int(str_size);
size_t str_size;
begin_sequence(str_size);
auto bytes = str_size * sizeof(std::u16string::value_type);
range_check(bytes);
str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a
// non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
CAF_ASSERT(n == bytes);
end_sequence();
break;
}
case string32_v: {
auto& str = *reinterpret_cast<std::u32string*>(val);
uint32_t str_size;
apply_int(str_size);
size_t str_size;
begin_sequence(str_size);
auto bytes = str_size * sizeof(std::u32string::value_type);
range_check(bytes);
str.resize(str_size);
// TODO: When using C++14, switch to str.data(), which then has a
// non-const overload.
streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
size_t n = streambuf_.sgetn(reinterpret_cast<char*>(&str[0]), bytes);
CAF_ASSERT(n == bytes);
end_sequence();
break;
}
}
......
......@@ -40,13 +40,9 @@ namespace caf {
/// Implements the serializer interface with a binary serialization protocol.
template <class Streambuf>
class stream_serializer : public serializer {
static_assert(
std::is_base_of<
std::streambuf,
typename std::remove_reference<Streambuf>::type
>::value,
"Streambuf must inherit from std::streambuf"
);
using streambuf_type = typename std::remove_reference<Streambuf>::type;
static_assert(std::is_base_of<std::streambuf, streambuf_type>::value,
"Streambuf must inherit from std::streambuf");
public:
template <class... Ts>
......@@ -86,8 +82,7 @@ public:
}
void begin_sequence(size_t& list_size) override {
auto s = static_cast<uint32_t>(list_size);
apply(s);
varbyte_encode(list_size);
}
void end_sequence() override {
......@@ -99,6 +94,23 @@ public:
}
protected:
// Encode an unsigned integral type as variable-byte sequence.
template <class T>
size_t varbyte_encode(T x) {
static_assert(std::is_unsigned<T>::value, "T must be an unsigned type");
// For 64-bit values, the encoded representation cannot get larger than 10
// bytes. A scratch space of 16 bytes suffices as upper bound.
uint8_t buf[16];
auto i = buf;
while (x > 0x7f) {
*i++ = (static_cast<uint8_t>(x) & 0x7f) | 0x80;
x >>= 7;
}
*i++ = static_cast<uint8_t>(x) & 0x7f;
using char_type = typename streambuf_type::traits_type::char_type;
return streambuf_.sputn(reinterpret_cast<char_type*>(buf), i - buf);
}
void apply_builtin(builtin type, void* val) override {
CAF_ASSERT(val != nullptr);
switch (type) {
......@@ -138,31 +150,34 @@ protected:
}
case string8_v: {
auto str = reinterpret_cast<std::string*>(val);
auto s = static_cast<uint32_t>(str->size());
apply(s);
auto s = str->size();
begin_sequence(s);
apply_raw(str->size(), const_cast<char*>(str->c_str()));
end_sequence();
break;
}
case string16_v: {
auto& str = *reinterpret_cast<std::u16string*>(val);
auto s = static_cast<uint32_t>(str.size());
apply(s);
for (auto c : str) {
auto str = reinterpret_cast<std::u16string*>(val);
auto s = str->size();
begin_sequence(s);
for (auto c : *str) {
// the standard does not guarantee that char16_t is exactly 16 bits...
auto tmp = static_cast<uint16_t>(c);
apply_raw(sizeof(tmp), &tmp);
}
end_sequence();
break;
}
case string32_v: {
auto& str = *reinterpret_cast<std::u32string*>(val);
auto s = static_cast<uint32_t>(str.size());
apply(s);
for (auto c : str) {
auto str = reinterpret_cast<std::u32string*>(val);
auto s = str->size();
begin_sequence(s);
for (auto c : *str) {
// the standard does not guarantee that char32_t is exactly 32 bits...
auto tmp = static_cast<uint32_t>(c);
apply_raw(sizeof(tmp), &tmp);
}
end_sequence();
break;
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment