Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve error_on_const_read logic for arrays and more direct array reading #1354

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Improve error_on_const_read logic for arrays and more direct array reading #1354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4750970
Update json_performance.cpp
stephenberry Oct 3, 2024
835d89c
Cleanup of array reading
stephenberry Oct 3, 2024
65dfbb1
Update json_performance.cpp
stephenberry Oct 3, 2024
be9ffb2
reset testing
stephenberry Oct 3, 2024
55e44ee
else constexpr branch
stephenberry Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
330 changes: 178 additions & 152 deletions include/glaze/json/read.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1364,6 +1364,8 @@ namespace glz
requires(readable_array_t<T> && (emplace_backable<T> || !resizable<T>) && !emplaceable<T>)
struct from<JSON, T>
{
static constexpr bool is_const = std::is_const_v<std::remove_reference_t<typename T::value_type>>;

template <auto Options>
static void op(auto&& value, is_context auto&& ctx, auto&& it, auto&& end) noexcept
{
Expand All @@ -1390,192 +1392,216 @@ namespace glz
}
return;
}

// It is valid to clear a non-const container holding const elements
// But, once we read here we're trying to parse into const elements, which is invalid
if constexpr (is_const && Options.error_on_const_read) {
ctx.error = error_code::attempt_const_read;
return;
}
else {
const size_t ws_size = size_t(it - ws_start);

const size_t ws_size = size_t(it - ws_start);
const auto n = value.size();

const auto n = value.size();
auto value_it = value.begin();

auto value_it = value.begin();

for (size_t i = 0; i < n; ++i) {
read<JSON>::op<ws_handled<Opts>()>(*value_it++, ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') {
++it;
for (size_t i = 0; i < n; ++i) {
if constexpr (is_const) {
// do not read anything into the const value
skip_value<JSON>::op<ws_handled<Opts>()>(ctx, it, end);
}
else {
using V = std::remove_cvref_t<decltype(*value_it)>;
from<JSON, V>::template op<ws_handled<Opts>()>(*value_it++, ctx, it, end);
}

if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') {
++it;

if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
}
}
}

GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
if constexpr (erasable<T>) {
value.erase(value_it,
value.end()); // use erase rather than resize for non-default constructible elements
GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
if constexpr (erasable<T>) {
value.erase(value_it,
value.end()); // use erase rather than resize for non-default constructible elements

if constexpr (Opts.shrink_to_fit) {
value.shrink_to_fit();
if constexpr (Opts.shrink_to_fit) {
value.shrink_to_fit();
}
}
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
}
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;

if constexpr (Opts.partial_read) {
return;
}
}

if constexpr (Opts.partial_read) {
return;
}
else {
// growing
if constexpr (emplace_backable<T>) {
// This optimization is useful when a std::vector contains large types (greater than 4096 bytes)
// It is faster to simply use emplace_back for small types and reasonably lengthed vectors
// (less than a million elements)
// https://baptiste-wicht.com/posts/2012/12/cpp-benchmark-vector-list-deque.html
if constexpr (has_reserve<T> && has_capacity<T> &&
requires { requires(sizeof(typename T::value_type) > 4096); }) {
// If we can reserve memmory, like std::vector, then we want to check the capacity
// and use a temporary buffer if the capacity needs to grow
if (value.capacity() == 0) {
value.reserve(1); // we want to directly use our vector for the first element
}
const auto capacity = value.capacity();
for (size_t i = value.size(); i < capacity; ++i) {
// emplace_back while we have capacity
read<JSON>::op<ws_handled<Opts>()>(value.emplace_back(), ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;
else {
// growing
if constexpr (emplace_backable<T>) {
// This optimization is useful when a std::vector contains large types (greater than 4096 bytes)
// It is faster to simply use emplace_back for small types and reasonably lengthed vectors
// (less than a million elements)
// https://baptiste-wicht.com/posts/2012/12/cpp-benchmark-vector-list-deque.html
if constexpr ((not is_const) && has_reserve<T> && has_capacity<T> &&
requires { requires(sizeof(typename T::value_type) > 4096); }) {
// If we can reserve memmory, like std::vector, then we want to check the capacity
// and use a temporary buffer if the capacity needs to grow
if (value.capacity() == 0) {
value.reserve(1); // we want to directly use our vector for the first element
}
const auto capacity = value.capacity();
for (size_t i = value.size(); i < capacity; ++i) {
// emplace_back while we have capacity
read<JSON>::op<ws_handled<Opts>()>(value.emplace_back(), ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;

if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
}
}
}

GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
}
}
}

using value_type = typename T::value_type;

std::vector<std::vector<value_type>> intermediate;
intermediate.reserve(48);
auto* active = &intermediate.emplace_back();
active->reserve(2);
while (it < end) {
if (active->size() == active->capacity()) {
// we want to populate the next vector
const auto former_capacity = active->capacity();
active = &intermediate.emplace_back();
active->reserve(2 * former_capacity);
}
using value_type = typename T::value_type;

std::vector<std::vector<value_type>> intermediate;
intermediate.reserve(48);
auto* active = &intermediate.emplace_back();
active->reserve(2);
while (it < end) {
if (active->size() == active->capacity()) {
// we want to populate the next vector
const auto former_capacity = active->capacity();
active = &intermediate.emplace_back();
active->reserve(2 * former_capacity);
}

read<JSON>::op<ws_handled<Opts>()>(active->emplace_back(), ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;
read<JSON>::op<ws_handled<Opts>()>(active->emplace_back(), ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;

if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
}
}
}

GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
break;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
break;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
}
}
}

const auto intermediate_size = intermediate.size();
size_t reserve_size = value.size();
for (size_t i = 0; i < intermediate_size; ++i) {
reserve_size += intermediate[i].size();
}
const auto intermediate_size = intermediate.size();
size_t reserve_size = value.size();
for (size_t i = 0; i < intermediate_size; ++i) {
reserve_size += intermediate[i].size();
}

if constexpr (std::is_trivially_copyable_v<value_type> && !std::same_as<T, std::vector<bool>>) {
const auto original_size = value.size();
value.resize(reserve_size);
auto* dest = value.data() + original_size;
for (const auto& vector : intermediate) {
const auto vector_size = vector.size();
std::memcpy(dest, vector.data(), vector_size * sizeof(value_type));
dest += vector_size;
if constexpr (std::is_trivially_copyable_v<value_type> && !std::same_as<T, std::vector<bool>>) {
const auto original_size = value.size();
value.resize(reserve_size);
auto* dest = value.data() + original_size;
for (const auto& vector : intermediate) {
const auto vector_size = vector.size();
std::memcpy(dest, vector.data(), vector_size * sizeof(value_type));
dest += vector_size;
}
}
}
else {
value.reserve(reserve_size);
for (const auto& vector : intermediate) {
const auto inter_end = vector.end();
for (auto inter = vector.begin(); inter < inter_end; ++inter) {
value.emplace_back(std::move(*inter));
else {
value.reserve(reserve_size);
for (const auto& vector : intermediate) {
const auto inter_end = vector.end();
for (auto inter = vector.begin(); inter < inter_end; ++inter) {
value.emplace_back(std::move(*inter));
}
}
}
}
}
else {
// If we don't have reserve (like std::deque) or we have small sized elements
while (it < end) {
read<JSON>::op<ws_handled<Opts>()>(value.emplace_back(), ctx, it, end);
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;
else {
// If we don't have reserve (like std::deque) or we have small sized elements
while (it < end) {
if constexpr (is_const) {
value.emplace_back();
// do not read anything into the const value
skip_value<JSON>::op<ws_handled<Opts>()>(ctx, it, end);
}
else {
using V = std::remove_cvref_t<decltype(value.emplace_back())>;
from<JSON, V>::template op<ws_handled<Opts>()>(value.emplace_back(), ctx, it, end);
}
if (bool(ctx.error)) [[unlikely]]
return;
GLZ_SKIP_WS();
if (*it == ',') [[likely]] {
++it;

if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
if constexpr (!Opts.minified) {
if (ws_size && ws_size < size_t(end - it)) {
skip_matching_ws(ws_start, it, ws_size);
}
}
}

GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
GLZ_SKIP_WS();
}
else if (*it == ']') {
GLZ_SUB_LEVEL;
++it;
return;
}
else [[unlikely]] {
ctx.error = error_code::expected_bracket;
return;
}
}
}
}
}
else {
ctx.error = error_code::exceeded_static_array_size;
else {
ctx.error = error_code::exceeded_static_array_size;
}
}
}
}
Expand Down
Loading
Loading