Skip to content

Commit

Permalink
Replace vector/map LUTs in binary_reader with arrays
Browse files Browse the repository at this point in the history
  • Loading branch information
@falbrechtskirchinger
falbrechtskirchinger committed Aug 1, 2022
1 parent 817a4a2 commit a64da9e
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 38 deletions.
61 changes: 47 additions & 14 deletions include/nlohmann/detail/input/binary_reader.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#include <string> // char_traits, string
#include <utility> // make_pair, move
#include <vector> // vector
#include <map> // map

#include <nlohmann/detail/exceptions.hpp>
#include <nlohmann/detail/input/input_adapters.hpp>
Expand Down Expand Up @@ -2240,14 +2239,16 @@ class binary_reader

if (current == '$')
{
std::vector<char_int_type> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type

result.second = get(); // must not ignore 'N', because 'N' maybe the type
if (JSON_HEDLEY_UNLIKELY( input_format == input_format_t::bjdata && std::find(bjdx.begin(), bjdx.end(), result.second) != bjdx.end() ))
if (input_format == input_format_t::bjdata)
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr));
auto it = std::lower_bound(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second);
if (JSON_HEDLEY_UNLIKELY(it != bjd_optimized_type_markers.end() && *it == result.second))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr));
}
}

if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type")))
Expand Down Expand Up @@ -2494,21 +2495,21 @@ class binary_reader

if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && (size_and_type.second & (1 << 8)) != 0)
{
std::map<char_int_type, string_t> bjdtype = {{'U', "uint8"}, {'i', "int8"}, {'u', "uint16"}, {'I', "int16"},
{'m', "uint32"}, {'l', "int32"}, {'M', "uint64"}, {'L', "int64"}, {'d', "single"}, {'D', "double"}, {'C', "char"}
};

size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker

auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t)
{
return p.first < t;
});
string_t key = "_ArrayType_";
if (JSON_HEDLEY_UNLIKELY(bjdtype.count(size_and_type.second) == 0))
if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr));
}

if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(bjdtype[size_and_type.second]) ))
string_t type = it->second;
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
{
return false;
}
Expand Down Expand Up @@ -2967,6 +2968,38 @@ class binary_reader

/// the SAX parser
json_sax_t* sax = nullptr;

// excluded markers in bjdata optimized type
#define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \
make_array<char_int_type>('F', 'H', 'N', 'S', 'T', 'Z', '[', '{')

#define JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_ \
make_array<bjd_type>( \
bjd_type{'C', "char"}, \
bjd_type{'D', "double"}, \
bjd_type{'I', "int16"}, \
bjd_type{'L', "int64"}, \
bjd_type{'M', "uint64"}, \
bjd_type{'U', "uint8"}, \
bjd_type{'d', "single"}, \
bjd_type{'i', "int8"}, \
bjd_type{'l', "int32"}, \
bjd_type{'m', "uint32"}, \
bjd_type{'u', "uint16"})

JSON_PRIVATE_UNLESS_TESTED:
// lookup tables
// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
const decltype(JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_) bjd_optimized_type_markers =
JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_;

using bjd_type = std::pair<char_int_type, string_t>;
// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
const decltype(JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_) bjd_types_map =
JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_;

#undef JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_
#undef JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_
};

} // namespace detail
Expand Down
13 changes: 9 additions & 4 deletions include/nlohmann/detail/meta/cpp_future.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#pragma once

#include <array> // array
#include <cstddef> // size_t
#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
#include <utility> // index_sequence, make_index_sequence, index_sequence_for
Expand Down Expand Up @@ -152,15 +153,19 @@ template<> struct priority_tag<0> {};
template<typename T>
struct static_const
{
static constexpr T value{};
static JSON_INLINE_VARIABLE constexpr T value{};
};

#ifndef JSON_HAS_CPP_17

template<typename T>
constexpr T static_const<T>::value; // NOLINT(readability-redundant-declaration)

constexpr T static_const<T>::value;
#endif

template<typename T, typename... Args>
inline constexpr std::array<T, sizeof...(Args)> make_array(Args&& ... args)
{
return std::array<T, sizeof...(Args)> {{static_cast<T>(std::forward<Args>(args))...}};
}

} // namespace detail
NLOHMANN_JSON_NAMESPACE_END
74 changes: 56 additions & 18 deletions single_include/nlohmann/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3028,6 +3028,7 @@ NLOHMANN_JSON_NAMESPACE_END



#include <array> // array
#include <cstddef> // size_t
#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type
#include <utility> // index_sequence, make_index_sequence, index_sequence_for
Expand Down Expand Up @@ -3172,16 +3173,20 @@ template<> struct priority_tag<0> {};
template<typename T>
struct static_const
{
static constexpr T value{};
static JSON_INLINE_VARIABLE constexpr T value{};
};

#ifndef JSON_HAS_CPP_17

template<typename T>
constexpr T static_const<T>::value; // NOLINT(readability-redundant-declaration)

constexpr T static_const<T>::value;
#endif

template<typename T, typename... Args>
inline constexpr std::array<T, sizeof...(Args)> make_array(Args&& ... args)
{
return std::array<T, sizeof...(Args)> {{static_cast<T>(std::forward<Args>(args))...}};
}

} // namespace detail
NLOHMANN_JSON_NAMESPACE_END

Expand Down Expand Up @@ -5977,7 +5982,6 @@ NLOHMANN_JSON_NAMESPACE_END
#include <string> // char_traits, string
#include <utility> // make_pair, move
#include <vector> // vector
#include <map> // map

// #include <nlohmann/detail/exceptions.hpp>

Expand Down Expand Up @@ -11230,14 +11234,16 @@ class binary_reader

if (current == '$')
{
std::vector<char_int_type> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type

result.second = get(); // must not ignore 'N', because 'N' maybe the type
if (JSON_HEDLEY_UNLIKELY( input_format == input_format_t::bjdata && std::find(bjdx.begin(), bjdx.end(), result.second) != bjdx.end() ))
if (input_format == input_format_t::bjdata)
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr));
auto it = std::lower_bound(bjd_optimized_type_markers.begin(), bjd_optimized_type_markers.end(), result.second);
if (JSON_HEDLEY_UNLIKELY(it != bjd_optimized_type_markers.end() && *it == result.second))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, concat("marker 0x", last_token, " is not a permitted optimized array type"), "type"), nullptr));
}
}

if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type")))
Expand Down Expand Up @@ -11484,21 +11490,21 @@ class binary_reader

if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && (size_and_type.second & (1 << 8)) != 0)
{
std::map<char_int_type, string_t> bjdtype = {{'U', "uint8"}, {'i', "int8"}, {'u', "uint16"}, {'I', "int16"},
{'m', "uint32"}, {'l', "int32"}, {'M', "uint64"}, {'L', "int64"}, {'d', "single"}, {'D', "double"}, {'C', "char"}
};

size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker

auto it = std::lower_bound(bjd_types_map.begin(), bjd_types_map.end(), size_and_type.second, [](const bjd_type & p, char_int_type t)
{
return p.first < t;
});
string_t key = "_ArrayType_";
if (JSON_HEDLEY_UNLIKELY(bjdtype.count(size_and_type.second) == 0))
if (JSON_HEDLEY_UNLIKELY(it == bjd_types_map.end() || it->first != size_and_type.second))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message(input_format, "invalid byte: 0x" + last_token, "type"), nullptr));
}

if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(bjdtype[size_and_type.second]) ))
string_t type = it->second;
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
{
return false;
}
Expand Down Expand Up @@ -11957,6 +11963,38 @@ class binary_reader

/// the SAX parser
json_sax_t* sax = nullptr;

// excluded markers in bjdata optimized type
#define JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_ \
make_array<char_int_type>('F', 'H', 'N', 'S', 'T', 'Z', '[', '{')

#define JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_ \
make_array<bjd_type>( \
bjd_type{'C', "char"}, \
bjd_type{'D', "double"}, \
bjd_type{'I', "int16"}, \
bjd_type{'L', "int64"}, \
bjd_type{'M', "uint64"}, \
bjd_type{'U', "uint8"}, \
bjd_type{'d', "single"}, \
bjd_type{'i', "int8"}, \
bjd_type{'l', "int32"}, \
bjd_type{'m', "uint32"}, \
bjd_type{'u', "uint16"})

JSON_PRIVATE_UNLESS_TESTED:
// lookup tables
// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
const decltype(JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_) bjd_optimized_type_markers =
JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_;

using bjd_type = std::pair<char_int_type, string_t>;
// NOLINTNEXTLINE(cppcoreguidelines-non-private-member-variables-in-classes)
const decltype(JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_) bjd_types_map =
JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_;

#undef JSON_BINARY_READER_MAKE_BJD_OPTIMIZED_TYPE_MARKERS_
#undef JSON_BINARY_READER_MAKE_BJD_TYPES_MAP_
};

} // namespace detail
Expand Down
17 changes: 15 additions & 2 deletions tests/src/unit-bjdata.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@

#include "doctest_compatibility.h"

#include <climits>
#include <limits>
#define JSON_TESTS_PRIVATE
#include <nlohmann/json.hpp>
using nlohmann::json;

#include <algorithm>
#include <climits>
#include <limits>
#include <iostream>
#include <fstream>
#include <set>
Expand Down Expand Up @@ -205,6 +207,17 @@ TEST_CASE_TEMPLATE_INVOKE(value_in_range_of_test, \

TEST_CASE("BJData")
{
SECTION("binary_reader LUT array is sorted")
{
std::vector<std::uint8_t> data;
auto ia = nlohmann::detail::input_adapter(data);
// NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
nlohmann::detail::binary_reader<json, decltype(ia)> br{std::move(ia), json::input_format_t::bjdata};

CHECK(std::is_sorted(br.bjd_optimized_type_markers.begin(), br.bjd_optimized_type_markers.end()));
CHECK(std::is_sorted(br.bjd_types_map.begin(), br.bjd_types_map.end()));
}

SECTION("individual values")
{
SECTION("discarded")
Expand Down

0 comments on commit a64da9e

Please sign in to comment.