Commit 9a120d9a authored by Dominik Charousset's avatar Dominik Charousset

Implement JSON parser using the monotonic resource

parent d1573452
......@@ -109,6 +109,7 @@ caf_add_component(
src/detail/glob_match.cpp
src/detail/group_tunnel.cpp
src/detail/invoke_result_visitor.cpp
src/detail/json.cpp
src/detail/local_group_module.cpp
src/detail/message_builder_element.cpp
src/detail/message_data.cpp
......@@ -248,6 +249,7 @@ caf_add_component(
detail.encode_base64
detail.group_tunnel
detail.ieee_754
detail.json
detail.limited_vector
detail.local_group_module
detail.meta_object
......
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#pragma once
#include <cstdint>
#include <variant>
#include <vector>
#include "caf/detail/monotonic_buffer_resource.hpp"
#include "caf/parser_state.hpp"
#include "caf/string_view.hpp"
// This JSON abstraction is designed to allocate its entire state in a monotonic
// buffer resource. This minimizes memory allocations and also enables us to
// "wink out" the entire JSON object by simply reclaiming the memory without
// having to call a single destructor. The API is not optimized for convenience
// or safety, since the only place we use this API is the json_reader.
namespace caf::detail::json {
struct null_t {};
class value {
public:
using array_allocator = monotonic_buffer_resource::allocator<value>;
using array = std::vector<value, array_allocator>;
struct member {
string_view key;
value* val = nullptr;
};
using member_allocator = monotonic_buffer_resource::allocator<member>;
using object = std::vector<member, member_allocator>;
using data_type
= std::variant<null_t, int64_t, double, bool, string_view, array, object>;
data_type data;
};
using array = value::array;
using member = value::member;
using object = value::object;
value* make_value(monotonic_buffer_resource* storage);
array* make_array(monotonic_buffer_resource* storage);
object* make_object(monotonic_buffer_resource* storage);
value* parse(string_parser_state& ps, monotonic_buffer_resource* storage);
} // namespace caf::detail::json
......@@ -157,6 +157,18 @@ private:
std::map<size_t, bucket> var_;
};
template <class T, class U>
bool operator==(monotonic_buffer_resource::allocator<T> x,
monotonic_buffer_resource::allocator<U> y) {
return x.resource() == y.resource();
}
template <class T, class U>
bool operator!=(monotonic_buffer_resource::allocator<T> x,
monotonic_buffer_resource::allocator<U> y) {
return x.resource() != y.resource();
}
} // namespace caf::detail
#ifdef CAF_CLANG
......
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#include "caf/detail/json.hpp"
#include <iterator>
#include <memory>
#include "caf/config.hpp"
#include "caf/detail/parser/chars.hpp"
#include "caf/detail/parser/is_char.hpp"
#include "caf/detail/parser/read_bool.hpp"
#include "caf/detail/parser/read_number.hpp"
#include "caf/detail/scope_guard.hpp"
#include "caf/pec.hpp"
CAF_PUSH_UNUSED_LABEL_WARNING
#include "caf/detail/parser/fsm.hpp"
namespace caf::detail::parser {
struct obj_consumer;
struct arr_consumer;
struct val_consumer {
monotonic_buffer_resource* storage;
json::value* ptr;
template <class T>
void value(T x) {
ptr->data = x;
}
arr_consumer begin_array();
obj_consumer begin_object();
};
struct key_consumer {
string_view* ptr;
void value(string_view str) {
*ptr = str;
}
};
struct member_consumer {
monotonic_buffer_resource* storage;
json::member* ptr;
key_consumer begin_key() {
return {std::addressof(ptr->key)};
}
val_consumer begin_val() {
ptr->val = json::make_value(storage);
return {storage, ptr->val};
}
};
struct obj_consumer {
json::object* ptr;
member_consumer begin_member() {
ptr->emplace_back();
return {ptr->get_allocator().resource(), std::addressof(ptr->back())};
}
};
struct arr_consumer {
json::array* ptr;
val_consumer begin_value() {
ptr->emplace_back();
return {ptr->get_allocator().resource(), std::addressof(ptr->back())};
}
};
arr_consumer val_consumer::begin_array() {
ptr->data = json::array(json::value::array_allocator{storage});
auto& arr = std::get<json::array>(ptr->data);
arr.reserve(16);
return {&arr};
}
obj_consumer val_consumer::begin_object() {
ptr->data = json::object(json::value::member_allocator{storage});
auto& obj = std::get<json::object>(ptr->data);
obj.reserve(16);
return {&obj};
}
void read_value(string_parser_state& ps, val_consumer consumer);
void read_json_null(string_parser_state& ps, val_consumer consumer) {
auto g = make_scope_guard([&] {
if (ps.code <= pec::trailing_character)
consumer.value(json::null_t{});
});
// clang-format off
start();
state(init) {
transition(init, " \t\n")
transition(has_n, 'n')
}
state(has_n) {
transition(has_nu, 'u')
}
state(has_nu) {
transition(has_nul, 'l')
}
state(has_nul) {
transition(done, 'l')
}
term_state(done) {
transition(init, " \t\n")
}
fin();
// clang-format on
}
template <class Consumer>
void read_json_string(string_parser_state& ps, Consumer consumer) {
auto first = string_view::iterator{};
// clang-format off
start();
state(init) {
transition(init, " \t\n")
transition(read_chars, '"', first = ps.i + 1)
}
state(read_chars) {
transition(escape, '\\')
transition(done, '"', consumer.value(string_view{first, ps.i}))
transition(read_chars, any_char)
}
state(escape) {
// TODO: Add support for JSON's \uXXXX escaping.
transition(read_chars, "\"\\/bfnrt")
}
term_state(done) {
transition(done, " \t\n")
}
fin();
// clang-format on
}
void read_member(string_parser_state& ps, member_consumer consumer) {
// clang-format off
start();
state(init) {
transition(init, " \t\n")
fsm_epsilon(read_json_string(ps, consumer.begin_key()), after_key, '"')
}
state(after_key) {
transition(after_key, " \t\n")
fsm_transition(read_value(ps, consumer.begin_val()), done, ':')
}
term_state(done) {
transition(done, " \t\n")
}
fin();
// clang-format on
}
void read_json_object(string_parser_state& ps, obj_consumer consumer) {
// clang-format off
start();
state(init) {
transition(init, " \t\n")
transition(has_open_brace, '{')
}
state(has_open_brace) {
transition(has_open_brace, " \t\n")
fsm_epsilon(read_member(ps, consumer.begin_member()), after_member, '"')
transition(done, '}')
}
state(after_member) {
transition(after_member, " \t\n")
transition(after_comma, ',')
transition(done, '}')
}
state(after_comma) {
transition(after_comma, " \t\n")
fsm_epsilon(read_member(ps, consumer.begin_member()), after_member, '"')
}
term_state(done) {
transition(done, " \t\n")
}
fin();
// clang-format on
}
void read_json_array(string_parser_state& ps, arr_consumer consumer) {
// clang-format off
start();
state(init) {
transition(init, " \t\n")
transition(has_open_brace, '[')
}
state(has_open_brace) {
transition(has_open_brace, " \t\n")
transition(done, ']')
fsm_epsilon(read_value(ps, consumer.begin_value()), after_value)
}
state(after_value) {
transition(after_value, " \t\n")
transition(after_comma, ',')
transition(done, ']')
}
state(after_comma) {
transition(after_comma, " \t\n")
fsm_epsilon(read_value(ps, consumer.begin_value()), after_value)
}
term_state(done) {
transition(done, " \t\n")
}
fin();
// clang-format on
}
void read_value(string_parser_state& ps, val_consumer consumer) {
// clang-format off
start();
state(init) {
transition(init, " \t\n")
fsm_epsilon(read_json_string(ps, consumer), done, '"')
fsm_epsilon(read_bool(ps, consumer), done, "ft")
fsm_epsilon(read_json_null(ps, consumer), done, "n")
fsm_epsilon(read_number(ps, consumer), done, "+-.0123456789")
fsm_epsilon(read_json_object(ps, consumer.begin_object()), done, '{')
fsm_epsilon(read_json_array(ps, consumer.begin_array()), done, '[')
}
term_state(done) {
transition(done, " \t\n")
}
fin();
// clang-format on
}
} // namespace caf::detail::parser
#include "caf/detail/parser/fsm_undef.hpp"
namespace caf::detail::json {
namespace {
template <class T, class Allocator>
void init(std::vector<T, Allocator>* ptr, monotonic_buffer_resource* storage) {
new (ptr) std::vector<T, Allocator>(Allocator{storage});
}
void init(value* ptr, monotonic_buffer_resource*) {
new (ptr) value();
}
template <class T>
T* make_impl(monotonic_buffer_resource* storage) {
monotonic_buffer_resource::allocator<T> alloc{storage};
auto result = alloc.allocate(1);
init(result, storage);
return result;
}
} // namespace
value* make_value(monotonic_buffer_resource* storage) {
return make_impl<value>(storage);
}
array* make_array(monotonic_buffer_resource* storage) {
auto result = make_impl<array>(storage);
result->reserve(16);
return result;
}
object* make_object(monotonic_buffer_resource* storage) {
auto result = make_impl<object>(storage);
result->reserve(16);
return result;
}
value* parse(string_parser_state& ps, monotonic_buffer_resource* storage) {
monotonic_buffer_resource::allocator<value> alloc{storage};
auto result = new (alloc.allocate(1)) value();
parser::read_value(ps, {storage, result});
return result;
}
} // namespace caf::detail::json
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#define CAF_SUITE detail.json
#include "caf/detail/json.hpp"
#include "core-test.hpp"
using namespace caf;
using namespace caf::literals;
namespace {
// Worth mentioning: the output we check against is the trivial format produced
// by stringify(), which is not valid JSON due to trailing commas.
constexpr std::pair<string_view, string_view> baselines[] = {
{
R"({})",
R"({})",
},
{
R"( { } )",
R"({})",
},
{
R"(42)",
R"(42)",
},
{
R"(true)",
R"(true)",
},
{
R"(false)",
R"(false)",
},
{
R"(null)",
R"(null)",
},
{
R"({"foo":"bar"})",
R"({
"foo": "bar",
})",
},
{
R"(["foo","bar"])",
R"([
"foo",
"bar",
])",
},
{
R"({
"ints":[1,2,3],"awesome?":true,"ptr":null,"empty-list":[],"nested":{
"hello": "world",
"greeting": "hello world!"
},
"empty-object": {}
})",
R"({
"ints": [
1,
2,
3,
],
"awesome?": true,
"ptr": null,
"empty-list": [],
"nested": {
"hello": "world",
"greeting": "hello world!",
},
"empty-object": {},
})",
},
};
void stringify(std::string& str, size_t indent, const detail::json::value& val);
void stringify(std::string& str, size_t, int64_t val) {
str += std::to_string(val);
}
void stringify(std::string& str, size_t, double val) {
str += std::to_string(val);
}
void stringify(std::string& str, size_t, bool val) {
if (val)
str += "true";
else
str += "false";
}
void stringify(std::string& str, size_t, string_view val) {
str.push_back('"');
str.insert(str.end(), val.begin(), val.end());
str.push_back('"');
}
void stringify(std::string& str, size_t, detail::json::null_t) {
str += "null";
}
void stringify(std::string& str, size_t indent, const detail::json::array& xs) {
if (xs.empty()) {
str += "[]";
} else {
str += "[\n";
for (const auto& x : xs) {
str.insert(str.end(), indent + 2, ' ');
stringify(str, indent + 2, x);
str += ",\n";
}
str.insert(str.end(), indent, ' ');
str += "]";
}
}
void stringify(std::string& str, size_t indent,
const detail::json::object& obj) {
if (obj.empty()) {
str += "{}";
} else {
str += "{\n";
for (const auto& [key, val] : obj) {
str.insert(str.end(), indent + 2, ' ');
stringify(str, indent + 2, key);
str += ": ";
stringify(str, indent + 2, *val);
str += ",\n";
}
str.insert(str.end(), indent, ' ');
str += "}";
}
}
void stringify(std::string& str, size_t indent,
const detail::json::value& val) {
auto f = [&str, indent](auto&& x) { stringify(str, indent, x); };
std::visit(f, val.data);
}
std::string stringify(const detail::json::value& val) {
std::string result;
stringify(result, 0, val);
return result;
}
} // namespace
CAF_TEST(json baselines) {
size_t baseline_index = 0;
detail::monotonic_buffer_resource resource;
for (auto [input, output] : baselines) {
MESSAGE("test baseline at index " << baseline_index++);
string_parser_state ps{input.begin(), input.end()};
auto val = detail::json::parse(ps, &resource);
CHECK_EQ(ps.code, pec::success);
CHECK_EQ(stringify(*val), output);
resource.reclaim();
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment