Unverified Commit 5d11121f authored by Dominik Charousset's avatar Dominik Charousset Committed by GitHub

Merge pull request #1460

Add utility for UTF-8 validation
parents 4f20d587 eaa0a228
......@@ -131,6 +131,7 @@ caf_add_component(
src/detail/print.cpp
src/detail/private_thread.cpp
src/detail/private_thread_pool.cpp
src/detail/rfc3629.cpp
src/detail/ripemd_160.cpp
src/detail/set_thread_name.cpp
src/detail/stream_bridge.cpp
......@@ -271,6 +272,7 @@ caf_add_component(
detail.parser.read_timespan
detail.parser.read_unsigned_integer
detail.private_thread_pool
detail.rfc3629
detail.ringbuffer
detail.ripemd_160
detail.type_id_list_builder
......
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#pragma once
#include "caf/byte_span.hpp"
#include "caf/detail/core_export.hpp"
#include <cstddef>
namespace caf::detail {
/// Wraps functions for processing RFC 3629 encoding, i.e., UTF-8. See
/// https://datatracker.ietf.org/doc/html/rfc3629 for details.
class CAF_CORE_EXPORT rfc3629 {
public:
/// Checks whether `bytes` is a valid UTF-8 string.
static bool valid(const_byte_span bytes) noexcept;
/// Checks whether `str` is a valid UTF-8 string.
static bool valid(std::string_view str) noexcept {
return valid(as_bytes(make_span(str)));
}
};
} // namespace caf::detail
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#include "caf/detail/rfc3629.hpp"
namespace {
// Convenient literal for std::byte.
constexpr std::byte operator""_b(unsigned long long x) {
return static_cast<std::byte>(x);
}
// Takes the first N bits of `value`.
template <size_t N>
constexpr std::byte head(std::byte value) noexcept {
if constexpr (N == 1) {
return value & 0b1000'0000_b;
} else if constexpr (N == 2) {
return value & 0b1100'0000_b;
} else if constexpr (N == 3) {
return value & 0b1110'0000_b;
} else if constexpr (N == 4) {
return value & 0b1111'0000_b;
} else if constexpr (N == 5) {
return value & 0b1111'1000_b;
} else if constexpr (N == 6) {
return value & 0b1111'1100_b;
} else {
static_assert(N == 7);
return value & 0b1111'1110_b;
}
}
// Checks whether `value` is an UTF-8 continuation byte.
constexpr bool is_continuation_byte(std::byte value) noexcept {
return head<2>(value) == 0b1000'0000_b;
}
// The following code is based on the algorithm described in
// http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String
bool validate_rfc3629(const std::byte* first, const std::byte* last) {
while (first != last) {
auto x = *first++;
// Null byte (terminator) is not allowed.
if (x == 0_b)
return false;
// First bit is zero: ASCII character.
if (head<1>(x) == 0b0000'0000_b)
continue;
// 110b'xxxx: 2-byte sequence.
if (head<3>(x) == 0b1100'0000_b) {
// No non-shortest form.
if (head<7>(x) == 0b1100'0000_b)
return false;
if (first == last || !is_continuation_byte(*first++))
return false;
continue;
}
// 1110'xxxx: 3-byte sequence.
if (head<4>(x) == 0b1110'0000_b) {
if (first == last || !is_continuation_byte(*first))
return false;
// No non-shortest form.
if (x == 0b1110'0000_b && head<3>(*first) == 0b1000'0000_b)
return false;
// No surrogate characters.
if (x == 0b1110'1101_b && head<3>(*first) == 0b1010'0000_b)
return false;
++first;
if (first == last || !is_continuation_byte(*first++))
return false;
continue;
}
// 1111'0bxx: 4-byte sequence.
if (head<5>(x) == 0b1111'0000_b) {
if (first == last || !is_continuation_byte(*first))
return false;
// No non-shortest form.
if (x == 0b1111'0000_b && head<4>(*first) == 0b1000'0000_b)
return false;
++first;
if (first == last || !is_continuation_byte(*first++))
return false;
if (first == last || !is_continuation_byte(*first++))
return false;
continue;
}
return false;
}
return true;
}
} // namespace
namespace caf::detail {
bool rfc3629::valid(const_byte_span bytes) noexcept {
return validate_rfc3629(bytes.begin(), bytes.end());
}
} // namespace caf::detail
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#define CAF_SUITE detail.rfc3629
#include "caf/detail/rfc3629.hpp"
#include "core-test.hpp"
using namespace caf;
namespace {
bool valid_utf8(const_byte_span bytes) noexcept {
return detail::rfc3629::valid(bytes);
}
bool valid_utf8(std::string_view bytes) noexcept {
return detail::rfc3629::valid(bytes);
}
constexpr std::byte operator""_b(unsigned long long x) {
return static_cast<std::byte>(x);
}
// Missing continuation byte.
constexpr std::byte invalid_two_byte_1[] = {0xc8_b};
// Illegal non-shortest form (1).
constexpr std::byte invalid_two_byte_2[] = {0xc0_b, 0x80_b};
// Illegal non-shortest form (2).
constexpr std::byte invalid_two_byte_3[] = {0xc1_b, 0x80_b};
// Invalid continuation byte.
constexpr std::byte invalid_two_byte_4[] = {0xc8_b, 0x0f_b};
// Missing continuation bytes.
constexpr std::byte invalid_three_byte_1[] = {0xe8_b};
// Missing continuation byte.
constexpr std::byte invalid_three_byte_2[] = {0xe8_b, 0x80_b};
// Invalid continuation byte (1).
constexpr std::byte invalid_three_byte_3[] = {0xe8_b, 0x0f_b, 0x80_b};
// Invalid continuation byte (2).
constexpr std::byte invalid_three_byte_4[] = {0xe8_b, 0x80_b, 0x0f_b};
// Illegal non-shortest form (1).
constexpr std::byte invalid_three_byte_5[] = {0xe0_b, 0x80_b, 0x80_b};
// Illegal non-shortest form (2).
constexpr std::byte invalid_three_byte_6[] = {0xe0_b, 0x9f_b, 0x8f_b};
// Illegal surrogate (smallest).
constexpr std::byte invalid_three_byte_7[] = {0xed_b, 0xa0_b, 0x80_b};
// Illegal surrogate (largest).
constexpr std::byte invalid_three_byte_8[] = {0xed_b, 0xbf_b, 0xbf_b};
// Missing continuation bytes.
constexpr std::byte invalid_four_byte_1[] = {0xf1_b};
// Missing continuation bytes 3 and 4.
constexpr std::byte invalid_four_byte_2[] = {0xf1_b, 0xbc_b};
// Missing continuation byte 4.
constexpr std::byte invalid_four_byte_3[] = {0xf1_b, 0xbc_b, 0xbc_b};
// Invalid continuation byte (1).
constexpr std::byte invalid_four_byte_4[] = {0xf1_b, 0x08_b, 0x80_b, 0x80_b};
// Invalid continuation byte (2).
constexpr std::byte invalid_four_byte_5[] = {0xf1_b, 0x80_b, 0x08_b, 0x80_b};
// Invalid continuation byte (3).
constexpr std::byte invalid_four_byte_6[] = {0xf1_b, 0x80_b, 0x80_b, 0x08_b};
// Illegal non-shortest form.
constexpr std::byte invalid_four_byte_7[] = {0xf0_b, 0x8f_b, 0x8f_b, 0x8f_b};
// Illegal start of a sequence.
constexpr std::byte invalid_four_byte_8[] = {0xf8_b, 0x80_b, 0x80_b, 0x80_b};
// Smallest valid 2-byte sequence.
constexpr std::byte valid_two_byte_1[] = {0xc2_b, 0x80_b};
// Largest valid 2-byte sequence.
constexpr std::byte valid_two_byte_2[] = {0xdf_b, 0xbf_b};
// Smallest valid 3-byte sequence.
constexpr std::byte valid_three_byte_1[] = {0xe0_b, 0xa0_b, 0x80_b};
// Largest valid 3-byte sequence.
constexpr std::byte valid_three_byte_2[] = {0xef_b, 0xbf_b, 0xbf_b};
// Smallest valid 4-byte sequence.
constexpr std::byte valid_four_byte_1[] = {0xf0_b, 0x90_b, 0x80_b, 0x80_b};
// Largest valid 4-byte sequence.
constexpr std::byte valid_four_byte_2[] = {0xf7_b, 0xbf_b, 0xbf_b, 0xbf_b};
// Single line ASCII text.
constexpr std::string_view ascii_1 = "Hello World!";
// Multi-line ASCII text.
constexpr std::string_view ascii_2 = R"__(
* ____ _ _____ *
* / ___| / \ | ___| CAF *
* | | / _ \ | |_ Actor *
* | |___ / ___ \| _| Framework *
* \____/_/ \_|_| *
)__";
} // namespace
TEST_CASE("ASCII input") {
CHECK(valid_utf8(ascii_1));
CHECK(valid_utf8(ascii_2));
}
TEST_CASE("valid UTF-8 input") {
CHECK(valid_utf8(valid_two_byte_1));
CHECK(valid_utf8(valid_two_byte_2));
CHECK(valid_utf8(valid_three_byte_1));
CHECK(valid_utf8(valid_three_byte_2));
CHECK(valid_utf8(valid_four_byte_1));
CHECK(valid_utf8(valid_four_byte_2));
}
TEST_CASE("invalid UTF-8 input") {
CHECK(!valid_utf8(invalid_two_byte_1));
CHECK(!valid_utf8(invalid_two_byte_2));
CHECK(!valid_utf8(invalid_two_byte_3));
CHECK(!valid_utf8(invalid_two_byte_4));
CHECK(!valid_utf8(invalid_three_byte_1));
CHECK(!valid_utf8(invalid_three_byte_2));
CHECK(!valid_utf8(invalid_three_byte_3));
CHECK(!valid_utf8(invalid_three_byte_4));
CHECK(!valid_utf8(invalid_three_byte_5));
CHECK(!valid_utf8(invalid_three_byte_6));
CHECK(!valid_utf8(invalid_three_byte_7));
CHECK(!valid_utf8(invalid_three_byte_8));
CHECK(!valid_utf8(invalid_four_byte_1));
CHECK(!valid_utf8(invalid_four_byte_2));
CHECK(!valid_utf8(invalid_four_byte_3));
CHECK(!valid_utf8(invalid_four_byte_4));
CHECK(!valid_utf8(invalid_four_byte_5));
CHECK(!valid_utf8(invalid_four_byte_6));
CHECK(!valid_utf8(invalid_four_byte_7));
CHECK(!valid_utf8(invalid_four_byte_8));
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment