Unverified Commit 52fdb196 authored by Samir Halilčević's avatar Samir Halilčević Committed by GitHub

Implement fragmented UTF-8 validation

parent 3e4f2dfa
......@@ -219,6 +219,7 @@ caf_add_component(
TEST_SOURCES
test/core-test.cpp
TEST_SUITES
detail.rfc3629
hash.fnv
LEGACY_TEST_SOURCES
test/core-legacy-test.cpp
......@@ -278,7 +279,6 @@ caf_add_component(
detail.parser.read_timespan
detail.parser.read_unsigned_integer
detail.private_thread_pool
detail.rfc3629
detail.ringbuffer
detail.ripemd_160
detail.type_id_list_builder
......
......@@ -22,6 +22,14 @@ public:
static bool valid(std::string_view str) noexcept {
return valid(as_bytes(make_span(str)));
}
/// Checks whether `bytes` is a valid UTF-8 string.
static std::pair<size_t, bool> validate(const_byte_span bytes) noexcept;
/// Checks whether `str` is a valid UTF-8 string.
static std::pair<size_t, bool> validate(std::string_view str) noexcept {
return validate(as_bytes(make_span(str)));
}
};
} // namespace caf::detail
......@@ -39,8 +39,13 @@ constexpr bool is_continuation_byte(std::byte value) noexcept {
// The following code is based on the algorithm described in
// http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
bool validate_rfc3629(const std::byte* first, const std::byte* last) {
// Returns a pair consisting of an iterator to the of the valid range, and a
// boolean stating whether the validation failed because of incomplete data, or
// other failures like malformed encoding or invalid code point.
std::pair<const std::byte*, bool> validate_rfc3629(const std::byte* first,
const std::byte* last) {
while (first != last) {
auto checkpoint = first;
auto x = *first++;
// First bit is zero: ASCII character.
if (head<1>(x) == 0b0000'0000_b)
......@@ -49,49 +54,61 @@ bool validate_rfc3629(const std::byte* first, const std::byte* last) {
if (head<3>(x) == 0b1100'0000_b) {
// No non-shortest form.
if (head<7>(x) == 0b1100'0000_b)
return false;
if (first == last || !is_continuation_byte(*first++))
return false;
return {checkpoint, false};
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first++))
return {checkpoint, false};
continue;
}
// 1110'xxxx: 3-byte sequence.
if (head<4>(x) == 0b1110'0000_b) {
if (first == last || !is_continuation_byte(*first))
return false;
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first))
return {checkpoint, false};
// No non-shortest form.
if (x == 0b1110'0000_b && head<3>(*first) == 0b1000'0000_b)
return false;
return {checkpoint, false};
// No surrogate characters.
if (x == 0b1110'1101_b && head<3>(*first) == 0b1010'0000_b)
return false;
return {checkpoint, false};
++first;
if (first == last || !is_continuation_byte(*first++))
return false;
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first++))
return {checkpoint, false};
continue;
}
// 1111'0xxx: 4-byte sequence.
if (head<5>(x) == 0b1111'0000_b) {
// Check if code point is in the valid UTF range.
if (x > std::byte{0xf4})
return false;
if (first == last || !is_continuation_byte(*first))
return false;
return {checkpoint, false};
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first))
return {checkpoint, false};
// No non-shortest form.
if (x == 0b1111'0000_b && head<4>(*first) == 0b1000'0000_b)
return false;
return {checkpoint, false};
// Check if code point is in the valid UTF range.
if (x == std::byte{0xf4} && *first >= std::byte{0x90})
return false;
return {checkpoint, false};
++first;
if (first == last || !is_continuation_byte(*first++))
return false;
if (first == last || !is_continuation_byte(*first++))
return false;
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first++))
return {checkpoint, false};
if (first == last)
return {checkpoint, true};
if (!is_continuation_byte(*first++))
return {checkpoint, false};
continue;
}
return false;
return {checkpoint, false};
}
return true;
return {last, false};
}
} // namespace
......@@ -99,7 +116,12 @@ bool validate_rfc3629(const std::byte* first, const std::byte* last) {
namespace caf::detail {
bool rfc3629::valid(const_byte_span bytes) noexcept {
return validate_rfc3629(bytes.begin(), bytes.end());
return validate_rfc3629(bytes.begin(), bytes.end()).first == bytes.end();
}
std::pair<size_t, bool> rfc3629::validate(const_byte_span bytes) noexcept {
auto [last, incomplete] = validate_rfc3629(bytes.begin(), bytes.end());
return {static_cast<size_t>(last - bytes.begin()), incomplete};
}
} // namespace caf::detail
......@@ -2,15 +2,16 @@
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#define CAF_SUITE detail.rfc3629
#include "caf/detail/rfc3629.hpp"
#include "core-test.hpp"
#include "caf/test/test.hpp"
using namespace caf;
using detail::rfc3629;
using res_t = std::pair<size_t, bool>;
namespace {
SUITE("detail.rfc3629") {
bool valid_utf8(const_byte_span bytes) noexcept {
return detail::rfc3629::valid(bytes);
......@@ -122,43 +123,102 @@ constexpr std::string_view ascii_2 = R"__(
* \____/_/ \_|_| *
)__";
} // namespace
TEST_CASE("ASCII input") {
CHECK(valid_utf8(ascii_1));
CHECK(valid_utf8(ascii_2));
TEST("rfc3629::valid checks whether an input is valid UTF-8") {
SECTION("valid ASCII input") {
check(valid_utf8(ascii_1));
check(valid_utf8(ascii_2));
}
SECTION("valid UTF-8 input") {
check(valid_utf8(valid_two_byte_1));
check(valid_utf8(valid_two_byte_2));
check(valid_utf8(valid_three_byte_1));
check(valid_utf8(valid_three_byte_2));
check(valid_utf8(valid_four_byte_1));
check(valid_utf8(valid_four_byte_2));
}
SECTION("invalid UTF-8 input") {
check(!valid_utf8(invalid_two_byte_1));
check(!valid_utf8(invalid_two_byte_2));
check(!valid_utf8(invalid_two_byte_3));
check(!valid_utf8(invalid_two_byte_4));
check(!valid_utf8(invalid_three_byte_1));
check(!valid_utf8(invalid_three_byte_2));
check(!valid_utf8(invalid_three_byte_3));
check(!valid_utf8(invalid_three_byte_4));
check(!valid_utf8(invalid_three_byte_5));
check(!valid_utf8(invalid_three_byte_6));
check(!valid_utf8(invalid_three_byte_7));
check(!valid_utf8(invalid_three_byte_8));
check(!valid_utf8(invalid_four_byte_1));
check(!valid_utf8(invalid_four_byte_2));
check(!valid_utf8(invalid_four_byte_3));
check(!valid_utf8(invalid_four_byte_4));
check(!valid_utf8(invalid_four_byte_5));
check(!valid_utf8(invalid_four_byte_6));
check(!valid_utf8(invalid_four_byte_7));
check(!valid_utf8(invalid_four_byte_8));
check(!valid_utf8(invalid_four_byte_9));
check(!valid_utf8(invalid_four_byte_10));
}
}
TEST_CASE("valid UTF-8 input") {
CHECK(valid_utf8(valid_two_byte_1));
CHECK(valid_utf8(valid_two_byte_2));
CHECK(valid_utf8(valid_three_byte_1));
CHECK(valid_utf8(valid_three_byte_2));
CHECK(valid_utf8(valid_four_byte_1));
CHECK(valid_utf8(valid_four_byte_2));
TEST("rfc3629::validate returns the end index if the range is valid") {
SECTION("valid ASCII input") {
check_eq(rfc3629::validate(ascii_1), res_t{ascii_1.size(), false});
check_eq(rfc3629::validate(ascii_2), res_t{ascii_2.size(), false});
}
SECTION("valid UTF-8 input") {
check_eq(rfc3629::validate(valid_two_byte_1), res_t{2, false});
check_eq(rfc3629::validate(valid_two_byte_2), res_t{2, false});
check_eq(rfc3629::validate(valid_three_byte_1), res_t{3, false});
check_eq(rfc3629::validate(valid_three_byte_2), res_t{3, false});
check_eq(rfc3629::validate(valid_four_byte_1), res_t{4, false});
check_eq(rfc3629::validate(valid_four_byte_2), res_t{4, false});
}
}
TEST_CASE("invalid UTF-8 input") {
CHECK(!valid_utf8(invalid_two_byte_1));
CHECK(!valid_utf8(invalid_two_byte_2));
CHECK(!valid_utf8(invalid_two_byte_3));
CHECK(!valid_utf8(invalid_two_byte_4));
CHECK(!valid_utf8(invalid_three_byte_1));
CHECK(!valid_utf8(invalid_three_byte_2));
CHECK(!valid_utf8(invalid_three_byte_3));
CHECK(!valid_utf8(invalid_three_byte_4));
CHECK(!valid_utf8(invalid_three_byte_5));
CHECK(!valid_utf8(invalid_three_byte_6));
CHECK(!valid_utf8(invalid_three_byte_7));
CHECK(!valid_utf8(invalid_three_byte_8));
CHECK(!valid_utf8(invalid_four_byte_1));
CHECK(!valid_utf8(invalid_four_byte_2));
CHECK(!valid_utf8(invalid_four_byte_3));
CHECK(!valid_utf8(invalid_four_byte_4));
CHECK(!valid_utf8(invalid_four_byte_5));
CHECK(!valid_utf8(invalid_four_byte_6));
CHECK(!valid_utf8(invalid_four_byte_7));
CHECK(!valid_utf8(invalid_four_byte_8));
CHECK(!valid_utf8(invalid_four_byte_9));
CHECK(!valid_utf8(invalid_four_byte_10));
TEST("rfc3629::validate stops at the first invalid byte") {
SECTION("UTF-8 input missing continuation bytes") {
check_eq(rfc3629::validate(invalid_two_byte_1), res_t{0, true});
check_eq(rfc3629::validate(invalid_three_byte_1), res_t{0, true});
check_eq(rfc3629::validate(invalid_three_byte_2), res_t{0, true});
check_eq(rfc3629::validate(invalid_four_byte_1), res_t{0, true});
check_eq(rfc3629::validate(invalid_four_byte_2), res_t{0, true});
check_eq(rfc3629::validate(invalid_four_byte_3), res_t{0, true});
}
SECTION("UTF-8 input with malformed data") {
check_eq(rfc3629::validate(invalid_two_byte_2), res_t{0, false});
check_eq(rfc3629::validate(invalid_two_byte_3), res_t{0, false});
check_eq(rfc3629::validate(invalid_two_byte_4), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_3), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_4), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_5), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_6), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_7), res_t{0, false});
check_eq(rfc3629::validate(invalid_three_byte_8), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_4), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_5), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_6), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_7), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_8), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_9), res_t{0, false});
check_eq(rfc3629::validate(invalid_four_byte_10), res_t{0, false});
}
SECTION("invalid UTF-8 input fails on the invalid byte") {
check_eq(rfc3629::validate(make_span(invalid_four_byte_9, 2)),
res_t{0, false});
check_eq(rfc3629::validate(make_span(invalid_four_byte_10, 1)),
res_t{0, false});
}
SECTION("invalid UTF-8 input with valid prefix") {
byte_buffer data;
data.insert(data.end(), begin(valid_four_byte_1), end(valid_four_byte_1));
data.insert(data.end(), begin(valid_four_byte_2), end(valid_four_byte_2));
data.insert(data.end(), begin(valid_two_byte_1), end(valid_two_byte_1));
data.insert(data.end(), begin(invalid_four_byte_4),
end(invalid_four_byte_4));
check_eq(rfc3629::validate(data), res_t{10, false});
}
}
} // SUITE("detail.rfc3629")
......@@ -152,10 +152,14 @@ private:
shutdown(err);
}
/// Checks whether the current input is valid UTF-8. Stores the last position
/// while scanning in order to avoid validating the same bytes again.
bool payload_valid() noexcept;
// -- member variables -------------------------------------------------------
/// Points to the transport layer below.
octet_stream::lower_layer* down_;
octet_stream::lower_layer* down_ = nullptr;
/// Buffer for assembling binary frames.
binary_buffer binary_buf_;
......@@ -172,6 +176,9 @@ private:
/// Assembles fragmented payloads.
binary_buffer payload_buf_;
/// Stores where to resume the UTF-8 input validation.
size_t validation_offset_ = 0;
/// Next layer in the processing chain.
upper_layer_ptr up_;
};
......
......@@ -97,6 +97,11 @@ ptrdiff_t framing::consume(byte_span buffer, byte_span) {
if (hdr.fin) {
if (opcode_ == nil_code) {
// Call upper layer.
if (hdr.opcode == detail::rfc6455::text_frame
&& !detail::rfc3629::valid(payload)) {
abort_and_shutdown(sec::malformed_message, "invalid UTF-8 sequence");
return -1;
}
return handle(hdr.opcode, payload, frame_size);
}
if (hdr.opcode != detail::rfc6455::continuation_frame) {
......@@ -107,9 +112,14 @@ ptrdiff_t framing::consume(byte_span buffer, byte_span) {
}
// End of fragmented input.
payload_buf_.insert(payload_buf_.end(), payload.begin(), payload.end());
if (opcode_ == detail::rfc6455::text_frame && !payload_valid()) {
abort_and_shutdown(sec::malformed_message, "invalid UTF-8 sequence");
return -1;
}
auto result = handle(opcode_, payload_buf_, frame_size);
opcode_ = nil_code;
payload_buf_.clear();
validation_offset_ = 0;
return result;
}
// The first frame must not be a continuation frame. Any frame that is not
......@@ -122,6 +132,10 @@ ptrdiff_t framing::consume(byte_span buffer, byte_span) {
return -1;
}
payload_buf_.insert(payload_buf_.end(), payload.begin(), payload.end());
if (opcode_ == detail::rfc6455::text_frame && !payload_valid()) {
abort_and_shutdown(sec::malformed_message, "invalid UTF-8 sequence");
return -1;
}
return static_cast<ptrdiff_t>(frame_size);
}
......@@ -203,10 +217,6 @@ ptrdiff_t framing::handle(uint8_t opcode, byte_span payload,
case detail::rfc6455::text_frame: {
std::string_view text{reinterpret_cast<const char*>(payload.data()),
payload.size()};
if (!detail::rfc3629::valid(text)) {
abort_and_shutdown(sec::malformed_message, "invalid UTF-8 sequence");
return -1;
}
if (up_->consume_text(text) < 0)
return -1;
break;
......@@ -269,4 +279,16 @@ void framing::ship_closing_message(status code, std::string_view msg) {
down_->end_output();
}
bool framing::payload_valid() noexcept {
// validate from the index where we left off last time
auto [index, incomplete] = detail::rfc3629::validate(
make_span(payload_buf_).subspan(validation_offset_));
validation_offset_ += index;
if (validation_offset_ == payload_buf_.size())
return true;
// incomplete will be true if the last code point is missing continuation
// bytes but might be valid
return incomplete;
}
} // namespace caf::net::web_socket
......@@ -45,6 +45,11 @@ auto bytes(std::initializer_list<uint8_t> xs) {
return result;
}
int fetch_status(const_byte_span payload) {
return (std::to_integer<int>(payload[2]) << 8)
+ std::to_integer<int>(payload[3]);
}
} // namespace
BEGIN_FIXTURE_SCOPE(fixture)
......@@ -112,9 +117,8 @@ TEST_CASE("calling shutdown with protocol_error sets status in close header") {
detail::rfc6455::decode_header(transport->output_buffer(), hdr);
CHECK_EQ(hdr.opcode, detail::rfc6455::connection_close);
CHECK(hdr.payload_len >= 2);
auto status = (std::to_integer<int>(transport->output_buffer().at(2)) << 8)
+ std::to_integer<int>(transport->output_buffer().at(3));
CHECK_EQ(status, static_cast<int>(net::web_socket::status::protocol_error));
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::protocol_error));
CHECK(!app->has_aborted());
}
......@@ -138,9 +142,7 @@ SCENARIO("the client sends an invalid ping that closes the connection") {
MESSAGE("Buffer: " << transport->output_buffer());
CHECK_EQ(hdr.opcode, detail::rfc6455::connection_close);
CHECK(hdr.payload_len >= 2);
auto status = (std::to_integer<int>(transport->output_buffer()[2]) << 8)
+ std::to_integer<int>(transport->output_buffer()[3]);
CHECK_EQ(status,
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::protocol_error));
}
}
......@@ -166,9 +168,8 @@ SCENARIO("the client closes the connection with a closing handshake") {
CHECK_EQ(hdr.opcode, detail::rfc6455::connection_close);
CHECK(hdr.fin);
CHECK(hdr.payload_len >= 2);
auto status = (std::to_integer<int>(transport->output_buffer()[2]) << 8)
+ std::to_integer<int>(transport->output_buffer()[3]);
CHECK_EQ(status, static_cast<int>(net::web_socket::status::normal_close));
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::normal_close));
}
}
}
......@@ -193,9 +194,7 @@ SCENARIO("ping messages may not be fragmented") {
MESSAGE("Buffer: " << transport->output_buffer());
CHECK_EQ(hdr.opcode, detail::rfc6455::connection_close);
CHECK(hdr.payload_len >= 2);
auto status = (std::to_integer<int>(transport->output_buffer()[2]) << 8)
+ std::to_integer<int>(transport->output_buffer()[3]);
CHECK_EQ(status,
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::protocol_error));
}
}
......@@ -283,6 +282,78 @@ SCENARIO("ping messages may arrive between message fragments") {
}
}
SCENARIO("the application shuts down on invalid UTF-8 message") {
GIVEN("a text message with an invalid UTF-8 code point") {
auto data = bytes({
0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, // valid
0x83, 0xce, 0xbc, 0xce, 0xb5, // valid
0xf4, 0x90, 0x80, 0x80, // invalid code point
0x65, 0x64, 0x69, 0x74, 0x65, 0x64 // valid
});
auto data_span = const_byte_span(data);
WHEN("the client sends the whole message as a single frame") {
reset();
byte_buffer frame;
detail::rfc6455::assemble_frame(detail::rfc6455::text_frame, 0x0,
data_span, frame);
transport->push(frame);
THEN("the server aborts the application") {
CHECK_EQ(transport->handle_input(), 0);
CHECK_EQ(app->abort_reason, sec::malformed_message);
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::inconsistent_data));
MESSAGE("Aborted with: " << app->abort_reason);
}
}
WHEN("the client sends the first part of the message") {
reset();
byte_buffer frame;
detail::rfc6455::assemble_frame(detail::rfc6455::text_frame, 0x0,
data_span.subspan(0, 11), frame, 0);
transport->push(frame);
THEN("the connection did not abort") {
CHECK_EQ(transport->handle_input(),
static_cast<ptrdiff_t>(frame.size()));
CHECK(!app->has_aborted());
}
}
AND_WHEN("the client sends the second frame containing invalid data") {
byte_buffer frame;
detail::rfc6455::assemble_frame(detail::rfc6455::continuation_frame, 0x0,
data_span.subspan(11), frame);
transport->push(frame);
THEN("the server aborts the application") {
CHECK_EQ(transport->handle_input(), 0);
CHECK_EQ(app->abort_reason, sec::malformed_message);
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::inconsistent_data));
MESSAGE("Aborted with: " << app->abort_reason);
}
}
WHEN("the client sends the first invalid byte") {
reset();
byte_buffer frame;
detail::rfc6455::assemble_frame(detail::rfc6455::text_frame, 0x0,
data_span.subspan(0, 12), frame, 0);
transport->push(frame);
CHECK_EQ(transport->handle_input(), static_cast<ptrdiff_t>(frame.size()));
CHECK(!app->has_aborted());
frame.clear();
detail::rfc6455::assemble_frame(detail::rfc6455::continuation_frame, 0x0,
data_span.subspan(12, 1), frame, 0);
transport->push(frame);
THEN("the server aborts the application") {
CHECK_EQ(transport->handle_input(), 0);
CHECK_EQ(app->abort_reason, sec::malformed_message);
CHECK_EQ(fetch_status(transport->output_buffer()),
static_cast<int>(net::web_socket::status::inconsistent_data));
MESSAGE("Aborted with: " << app->abort_reason);
}
}
}
}
END_FIXTURE_SCOPE()
namespace {
......@@ -490,6 +561,26 @@ SCENARIO("the application shuts down on invalid frame fragments") {
CHECK_EQ(app->abort_reason, sec::protocol_error);
}
}
WHEN("the final frame is not a continuation frame") {
reset();
byte_buffer input;
const auto data = make_test_data(10);
detail::rfc6455::assemble_frame(detail::rfc6455::binary_frame, 0x0, data,
input, 0);
THEN("the app accepts the first frame") {
transport->push(input);
CHECK_EQ(transport->handle_input(),
static_cast<ptrdiff_t>(input.size()));
}
AND("the app closes the connection after the second frame") {
input.clear();
detail::rfc6455::assemble_frame(detail::rfc6455::binary_frame, 0x0,
data, input);
transport->push(input);
CHECK_EQ(transport->handle_input(), 0);
CHECK_EQ(app->abort_reason, sec::protocol_error);
}
}
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment