Merge pull request #21 from ebu/switch_to_nlohmann_json

Switch to nlohmann json

Author: firthm01

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
     - run: nix build .#bear --no-link
 
     - run: nix build .#bear_docker -o bear_docker.tar.gz
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
         name: bear_docker.tar.gz
         path: bear_docker.tar.gz

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "visr_bear/submodules/rapidjson"]
-	path = visr_bear/submodules/rapidjson
-	url = https://github.com/Tencent/rapidjson.git
 [submodule "visr_bear/submodules/libear"]
 	path = visr_bear/submodules/libear
 	url = https://github.com/ebu/libear.git

README.md

@@ -155,7 +155,9 @@ This is in the `visr_bear` subdirectory:
 
     cd visr_bear
 
-The process is the same as for VISR; to configure:
+Most dependencies are provided through submodules (some via submodules of submodules). The exception to this is the [nlohmann JSON library](https://github.com/nlohmann/json) which should be installed seperately. Cmake will attempt to locate this using `find_package`.
+
+The build process is the same as for VISR; to configure:
 
     cmake -B build .
 

nix/visr_bear.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, src, cmake, ninja, boost, python, visr ? null, visr_python ? null, data_files, eigen, libear, rapidjson, enable_python ? true }:
+{ lib, stdenv, src, cmake, ninja, boost, python, visr ? null, visr_python ? null, data_files, eigen, libear, nlohmann_json, enable_python ? true }:
 stdenv.mkDerivation rec {
   name = "visr_bear";
   inherit src;
@@ -8,15 +8,14 @@ stdenv.mkDerivation rec {
     boost
     eigen
     libear
-    rapidjson
+    nlohmann_json
   ] ++ (if enable_python then [ python visr_python ] else [ visr ]);
   cmakeFlags = [
     "-DBEAR_UNIT_TESTS=true"
     "-DBEAR_DATA_PATH_DEFAULT=${data_files.default.file}"
     "-DBEAR_DATA_PATH_DEFAULT_SMALL=${data_files.default_small.file}"
     "-DBEAR_SYMLINK_DATA_FILES=true"
     "-DBEAR_USE_INTERNAL_LIBEAR=false"
-    "-DBEAR_USE_INTERNAL_RAPIDJSON=false"
   ] ++ lib.optional (!enable_python) "-DBUILD_PYTHON_BINDINGS=false";
   preConfigure = ''cmakeFlags="$cmakeFlags -DBEAR_PYTHON_SITE_PACKAGES=$out/${python.sitePackages}"'';
 

visr_bear/CMakeLists.txt

@@ -90,18 +90,7 @@ if(NOT BEAR_USE_INTERNAL_LIBEAR)
   find_package(Eigen3 REQUIRED)
 endif()
 
-option(
-  BEAR_USE_INTERNAL_RAPIDJSON
-  "should we use our own version of RapidJSON, or find one with find_package?"
-  ON)
-if(NOT BEAR_USE_INTERNAL_RAPIDJSON)
-  find_package(RapidJSON REQUIRED)
-  if(NOT TARGET RapidJSON)
-    # rapidjson cmake currently doesn't export a target
-    add_library(RapidJSON INTERFACE)
-    target_include_directories(RapidJSON INTERFACE ${RAPIDJSON_INCLUDE_DIRS})
-  endif()
-endif()
+find_package(nlohmann_json 3.11.0 REQUIRED)
 
 find_package(Boost REQUIRED)
 

visr_bear/pythonwrappers/internals.cpp

@@ -20,7 +20,7 @@ namespace python {
   {
     init_parameters();
 
-    // make Config convertable to ConfigImpl; this is what the internals use,
+    // make Config convertible to ConfigImpl; this is what the internals use,
     // and this means that there's only one config interface on the python side
     // where we don't need to care about ABI compatibility.
     py::class_<ConfigImpl>(m, "_ConfigImpl").def(py::init([](const Config &config) {

visr_bear/src/CMakeLists.txt

@@ -2,7 +2,7 @@
 # internals (test,
 add_library(bear-internals INTERFACE)
 target_link_libraries(bear-internals INTERFACE mio)
-target_link_libraries(bear-internals INTERFACE RapidJSON)
+target_link_libraries(bear-internals INTERFACE nlohmann_json::nlohmann_json)
 target_link_libraries(bear-internals INTERFACE Eigen3::Eigen)
 target_include_directories(bear-internals INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
 

visr_bear/src/api.cpp

@@ -286,7 +286,7 @@ Renderer::~Renderer() = default;
 
 class DataFileMetadataImpl {
  public:
-  rapidjson::Value metadata;
+  nlohmann::json metadata;
 
   bool has_metadata = false;
   std::string label;
@@ -295,34 +295,34 @@ class DataFileMetadataImpl {
 };
 
 namespace {
-  std::string parse_label(rapidjson::Value &metadata)
+  std::string parse_label(const nlohmann::json &metadata)
   {
-    auto it = metadata.FindMember("label");
-    if (it == metadata.MemberEnd()) throw std::runtime_error("expected label");
+    auto it = metadata.find("label");
+    if (it == metadata.end()) throw std::runtime_error("expected label");
 
-    if (!it->value.IsString()) throw std::runtime_error("label should be a string");
+    if (!it->is_string()) throw std::runtime_error("label should be a string");
 
-    return it->value.GetString();
+    return it->template get<std::string>();
   }
 
-  std::string parse_description(rapidjson::Value &metadata)
+  std::string parse_description(const nlohmann::json &metadata)
   {
-    auto it = metadata.FindMember("description");
-    if (it == metadata.MemberEnd()) return "";
+    auto it = metadata.find("description");
+    if (it == metadata.end()) return "";
 
-    if (!it->value.IsString()) throw std::runtime_error("description should be a string");
+    if (!it->is_string()) throw std::runtime_error("description should be a string");
 
-    return it->value.GetString();
+    return it->template get<std::string>();
   }
 
-  bool parse_released(rapidjson::Value &metadata)
+  bool parse_released(const nlohmann::json &metadata)
   {
-    auto it = metadata.FindMember("released");
-    if (it == metadata.MemberEnd()) throw std::runtime_error("expected released");
+    auto it = metadata.find("released");
+    if (it == metadata.end()) throw std::runtime_error("expected released");
 
-    if (!it->value.IsBool()) throw std::runtime_error("released should be a bool");
+    if (!it->is_boolean()) throw std::runtime_error("released should be a bool");
 
-    return it->value.GetBool();
+    return it->template get<bool>();
   }
 }  // namespace
 
@@ -338,13 +338,13 @@ DataFileMetadata DataFileMetadata::read_from_file(const std::string &path)
 
   auto impl = std::make_unique<DataFileMetadataImpl>();
 
-  auto it = tf.metadata.FindMember("metadata");
-  if (it != tf.metadata.MemberEnd()) {
-    if (!it->value.IsObject()) throw std::runtime_error("data file metadata should be object");
+  auto it = tf.metadata.find("metadata");
+  if (it != tf.metadata.end()) {
+    if (!it->is_object()) throw std::runtime_error("data file metadata should be object");
 
     impl->has_metadata = true;
 
-    impl->metadata = it->value;
+    impl->metadata = *it;
 
     impl->label = parse_label(impl->metadata);
     impl->description = parse_description(impl->metadata);

visr_bear/src/data_file.cpp

@@ -5,12 +5,12 @@
 namespace bear {
 int get_data_file_version(tensorfile::TensorFile &tf)
 {
-  auto it = tf.metadata.FindMember("bear_data_version");
-  if (it == tf.metadata.MemberEnd()) return 0;
+  auto it = tf.metadata.find("bear_data_version");
+  if (it == tf.metadata.end()) return 0;
 
-  if (!it->value.IsInt()) throw std::runtime_error("version must be an integer");
+  if (!it->is_number_integer()) throw std::runtime_error("version must be an integer");
 
-  return it->value.GetInt();
+  return it->template get<int>();
 }
 
 void check_data_file_version(tensorfile::TensorFile &tf)

visr_bear/src/panner.cpp

@@ -3,41 +3,43 @@
 #include "data_file.hpp"
 #include "utils.hpp"
 
+using json = nlohmann::json;
+
 namespace {
-ear::PolarPosition load_position(const rapidjson::Value &position_j)
+ear::PolarPosition load_position(const json &position_j)
 {
-  return {position_j["azimuth"].GetDouble(),
-          position_j["elevation"].GetDouble(),
-          position_j["distance"].GetDouble()};
+  return {position_j.at("azimuth").template get<double>(),
+          position_j.at("elevation").template get<double>(),
+          position_j.at("distance").template get<double>()};
 }
 
-ear::Channel load_channel(const rapidjson::Value &channel_j)
+ear::Channel load_channel(const json &channel_j)
 {
   ear::Channel channel;
-  channel.name(channel_j["name"].GetString());
+  channel.name(channel_j.at("name").template get<std::string>());
 
-  channel.polarPosition(load_position(channel_j["polar_position"]));
-  channel.polarPositionNominal(load_position(channel_j["polar_nominal_position"]));
+  channel.polarPosition(load_position(channel_j.at("polar_position")));
+  channel.polarPositionNominal(load_position(channel_j.at("polar_nominal_position")));
 
-  const auto &az_range_j = channel_j["az_range"].GetArray();
-  channel.azimuthRange({{az_range_j[0].GetDouble(), az_range_j[1].GetDouble()}});
+  auto az_range = channel_j.at("az_range").template get<std::pair<double, double>>();
+  channel.azimuthRange(az_range);
 
-  const auto &el_range_j = channel_j["el_range"].GetArray();
-  channel.elevationRange({{el_range_j[0].GetDouble(), el_range_j[1].GetDouble()}});
+  auto el_range = channel_j.at("el_range").template get<std::pair<double, double>>();
+  channel.elevationRange(el_range);
 
-  channel.isLfe(channel_j["is_lfe"].GetBool());
+  channel.isLfe(channel_j.at("is_lfe").get<bool>());
 
   return channel;
 }
 
-ear::Layout load_layout(const rapidjson::Value &layout_j)
+ear::Layout load_layout(const json &layout_j)
 {
+  const json::array_t &channels_j = layout_j.at("channels").template get_ref<const json::array_t &>();
+
   std::vector<ear::Channel> channels;
-  for (auto &channel_j : layout_j["channels"].GetArray()) {
-    channels.push_back(load_channel(channel_j));
-  }
+  for (const auto &channel_j : channels_j) channels.push_back(load_channel(channel_j));
 
-  return {layout_j["name"].GetString(), channels};
+  return {layout_j.at("name").template get<std::string>(), channels};
 }
 
 void check(bool x, const std::string &message)
@@ -55,13 +57,13 @@ Panner::Panner(const std::string &brir_file_name)
 
   check_data_file_version(tf);
 
-  views = tf.unpack<float>(tf.metadata["views"]);
-  brirs = tf.unpack<float>(tf.metadata["brirs"]);
-  delays = tf.unpack<float>(tf.metadata["delays"]);
-  decorrelation_filters = tf.unpack<float>(tf.metadata["decorrelation_filters"]);
-  fs = tf.metadata["fs"].GetDouble();
-  decorrelation_delay_ = tf.metadata["decorrelation_delay"].GetDouble();
-  front_loudspeaker_ = tf.metadata["front_loudspeaker"].GetUint();
+  views = tf.unpack<float>(tf.metadata.at("views"));
+  brirs = tf.unpack<float>(tf.metadata.at("brirs"));
+  delays = tf.unpack<float>(tf.metadata.at("delays"));
+  decorrelation_filters = tf.unpack<float>(tf.metadata.at("decorrelation_filters"));
+  fs = tf.metadata.at("fs").template get<double>();
+  decorrelation_delay_ = tf.metadata.at("decorrelation_delay").template get<double>();
+  front_loudspeaker_ = tf.metadata.at("front_loudspeaker").template get<size_t>();
 
   // check and unpack dimensions
   check(views->ndim() == 2, "views must have 2 dimensions");
@@ -93,11 +95,11 @@ Panner::Panner(const std::string &brir_file_name)
 
   // load layout
   ear::Layout layout;
-  if (tf.metadata["layout"].IsString()) {
-    std::string layout_name = tf.metadata["layout"].GetString();
+  if (tf.metadata.at("layout").is_string()) {
+    std::string layout_name = tf.metadata.at("layout").template get<std::string>();
     layout = ear::getLayout(layout_name).withoutLfe();
   } else {
-    layout = load_layout(tf.metadata["layout"]);
+    layout = load_layout(tf.metadata.at("layout"));
   }
 
   // make gain calculators
@@ -110,9 +112,9 @@ Panner::Panner(const std::string &brir_file_name)
   temp_direct_diffuse.resize(num_gains() * 2);
   temp_direct_speakers.resize(num_gains());
 
-  if (tf.metadata.HasMember("gain_norm_quick")) {
+  if (tf.metadata.contains("gain_norm_quick")) {
     gain_comp_type = GainCompType::QUICK;
-    gain_comp_factors = tf.unpack<float>(tf.metadata["gain_norm_quick"]["factors"]);
+    gain_comp_factors = tf.unpack<float>(tf.metadata.at("gain_norm_quick").at("factors"));
     check(gain_comp_factors->ndim() == 5, "gain comp factors must have 5 dimensions");
     // TODO: check max delays?
     check(gain_comp_factors->shape(1) == n_views_, "gain comp factors axis 1 is wrong size");
@@ -123,13 +125,14 @@ Panner::Panner(const std::string &brir_file_name)
     check(gain_comp_factors->shape(4) == 2, "gain comp factors axis 4 is wrong size");
   }
 
-  check(tf.metadata.HasMember("hoa"), "HOA decoder not present");
-  hoa_irs = tf.unpack<float>(tf.metadata["hoa"]["irs"]);
+  check(tf.metadata.contains("hoa"), "HOA decoder not present");
+  hoa_irs = tf.unpack<float>(tf.metadata.at("hoa").at("irs"));
   check(hoa_irs->ndim() == 3, "HOA decoder must have 3 dimensions");
   n_hoa_channels_ = hoa_irs->shape(0);
   check(hoa_irs->shape(1) == 2, "HOA decoder axis 1 is wrong size");
 
-  if (tf.metadata["hoa"].HasMember("delay")) hoa_delay_ = tf.metadata["hoa"]["delay"].GetDouble();
+  if (tf.metadata.at("hoa").contains("delay"))
+    hoa_delay_ = tf.metadata.at("hoa").at("delay").template get<double>();
 
   hoa_order_ = ((size_t)std::round(std::sqrt(n_hoa_channels_))) - 1;
   size_t nch_for_order = (hoa_order_ + 1) * (hoa_order_ + 1);

visr_bear/src/tensorfile.cpp

@@ -6,7 +6,6 @@
 #define WIN32_LEAN_AND_MEAN
 #define NOMINMAX
 #include "mio.hpp"
-#include "rapidjson/error/en.h"
 
 namespace tensorfile {
 
@@ -176,19 +175,17 @@ namespace detail {
 
 }  // namespace detail
 
-std::shared_ptr<NDArray> TensorFile::unpack(const rapidjson::Value &v) const
+std::shared_ptr<NDArray> TensorFile::unpack(const nlohmann::json &v) const
 {
-  if (std::string(v["_tenf_type"].GetString()) == "array") {
-    std::vector<size_t> shape;
-    for (auto &shape_ax : v["shape"].GetArray()) shape.push_back(shape_ax.GetUint64());
+  if (std::string(v.at("_tenf_type").template get<std::string>()) == "array") {
+    std::vector<size_t> shape = v.at("shape").template get<std::vector<size_t>>();
 
-    std::vector<size_t> strides;
-    for (auto &stride : v["strides"].GetArray()) strides.push_back(stride.GetUint64());
+    std::vector<size_t> strides = v.at("strides").template get<std::vector<size_t>>();
 
     if (strides.size() != shape.size()) throw format_error("mismatched strides and shape");
 
-    std::string dtype(v["dtype"].GetString());
-    size_t offset = v["base"].GetInt64();
+    std::string dtype = v.at("dtype").template get<std::string>();
+    size_t offset = v.at("base").template get<size_t>();
 
     auto type_handler = detail::get_type_handler(dtype);
     return (*type_handler)(std::move(dtype), std::move(shape), std::move(strides), mmap, offset);
@@ -215,13 +212,13 @@ TensorFile read(const std::string &path)
   if (mmap->length() < header_len + data_len + metadata_len) throw format_error("file not long enough");
   if (metadata_len == 0) throw format_error("no JSON metadata found");
 
-  rapidjson::Document metadata;
-  metadata.Parse((char *)data + header_len + data_len, metadata_len);
-  if (metadata.HasParseError()) {
-    std::stringstream err;
-    err << "could not parse JSON metadata at " << metadata.GetErrorOffset() << ": "
-        << GetParseError_En(metadata.GetParseError());
-    throw format_error(err.str());
+  const char *metadata_start = reinterpret_cast<const char *>(data) + header_len + data_len;
+  const char *metadata_end = metadata_start + metadata_len;
+  nlohmann::json metadata;
+  try {
+    metadata = nlohmann::json::parse(metadata_start, metadata_end);
+  } catch (const nlohmann::json::parse_error &e) {
+    throw format_error(std::string{"could not parse JSON metadata: "} + e.what());
   }
 
   return TensorFile(std::move(mmap), std::move(metadata));

visr_bear/src/tensorfile.hpp

@@ -5,7 +5,7 @@
 #include <string>
 #include <vector>
 
-#include "rapidjson/document.h"
+#include "nlohmann/json.hpp"
 
 namespace tensorfile {
 
@@ -82,16 +82,16 @@ struct NDArrayT : public NDArray {
 
 class TensorFile {
  public:
-  TensorFile(std::shared_ptr<MMap> mmap, rapidjson::Document metadata)
+  TensorFile(std::shared_ptr<MMap> mmap, nlohmann::json metadata)
       : metadata(std::move(metadata)), mmap(std::move(mmap))
   {
   }
-  rapidjson::Document metadata;
+  nlohmann::json metadata;
 
-  std::shared_ptr<NDArray> unpack(const rapidjson::Value &v) const;
+  std::shared_ptr<NDArray> unpack(const nlohmann::json &v) const;
 
   template <typename T>
-  std::shared_ptr<NDArrayT<T>> unpack(const rapidjson::Value &v) const
+  std::shared_ptr<NDArrayT<T>> unpack(const nlohmann::json &v) const
   {
     return std::dynamic_pointer_cast<NDArrayT<T>>(unpack(v));
   }