chore: introduce conversion routines between JsonType and FlatJson (#…

…2785)

src/core/flatbuffers.h

@@ -11,3 +11,7 @@
 #include <flatbuffers/flatbuffers.h>
 #include <flatbuffers/flexbuffers.h>
 #include <flatbuffers/idl.h>
+
+namespace dfly {
+using FlatJson = flexbuffers::Reference;
+}  // namespace dfly

src/core/flatbuffers_test.cc

@@ -49,8 +49,8 @@ TEST_F(FlatBuffersTest, FlexiParser) {
   const auto& buffer = fbb.GetBuffer();
   string_view buf_view{reinterpret_cast<const char*>(buffer.data()), buffer.size()};
   LOG(INFO) << "Binary buffer: " << absl::CHexEscape(buf_view);
-
-  auto map = flexbuffers::GetRoot(buffer).AsMap();
+  flexbuffers::Reference root = flexbuffers::GetRoot(buffer);
+  auto map = root.AsMap();
   EXPECT_EQ("bar", map["foo"].AsString().str());
 }
 

src/core/json/json_object.h

@@ -14,9 +14,6 @@
 
 namespace dfly {
 
-// This is temporary, there is an issue right now with jsoncons about using jsonpath
-// with custom allocator. once it would resolved, we would change this to use custom allocator
-// that allocate memory from mimalloc
 using JsonType = jsoncons::pmr::json;
 
 // Build a json object from string. If the string is not legal json, will return nullopt

src/core/json/jsonpath_test.cc

@@ -20,7 +20,6 @@ namespace dfly::json {
 using namespace std;
 
 using testing::ElementsAre;
-using FlatJson = flexbuffers::Reference;
 
 MATCHER_P(SegType, value, "") {
   return ExplainMatchResult(testing::Property(&PathSegment::type, value), arg, result_listener);
@@ -159,6 +158,30 @@ TEST_F(ScannerTest, Basic) {
   NEXT_TOK(WILDCARD);
 }
 
+TEST_F(ScannerTest, FlatToJson) {
+  flatbuffers::Parser parser;
+  const char* json = R"(
+    {
+      "foo": "bar",
+      "bar": 1.5,
+      "strs": ["hello", "world"]
+    }
+  )";
+  flexbuffers::Builder fbb;
+  ASSERT_TRUE(parser.ParseFlexBuffer(json, nullptr, &fbb));
+  fbb.Finish();
+
+  flexbuffers::Reference root = flexbuffers::GetRoot(fbb.GetBuffer());
+  JsonType res = FromFlat(root);
+  EXPECT_EQ(res, JsonType::parse(json));
+  fbb.Clear();
+  FromJsonType(res, &fbb);
+  fbb.Finish();
+  string actual;
+  flexbuffers::GetRoot(fbb.GetBuffer()).ToString(false, true, actual);
+  EXPECT_EQ(res, JsonType::parse(actual));
+}
+
 TYPED_TEST(JsonPathTest, Parser) {
   EXPECT_NE(0, this->Parse("foo"));
   EXPECT_NE(0, this->Parse("$foo"));

src/core/json/path.cc

@@ -57,7 +57,7 @@ void PathSegment::Evaluate(const JsonType& json) const {
   func->Apply(json);
 }
 
-void PathSegment::Evaluate(flexbuffers::Reference json) const {
+void PathSegment::Evaluate(FlatJson json) const {
   CHECK(type() == SegmentType::FUNCTION);
   AggFunction* func = std::get<shared_ptr<AggFunction>>(value_).get();
   CHECK(func);
@@ -136,7 +136,7 @@ unsigned MutatePath(const Path& path, MutateCallback callback, JsonType* json) {
 }
 
 // Flat json path evaluation
-void EvaluatePath(const Path& path, flexbuffers::Reference json, PathFlatCallback callback) {
+void EvaluatePath(const Path& path, FlatJson json, PathFlatCallback callback) {
   if (path.empty()) {  // root node
     callback(nullopt, json);
     return;
@@ -149,22 +149,21 @@ void EvaluatePath(const Path& path, flexbuffers::Reference json, PathFlatCallbac
 
   // Handling the case of `func($.somepath)`
   // We pass our own callback to gather all the results and then call the function.
-  flexbuffers::Reference result;
+  FlatJson result;
   absl::Span<const PathSegment> path_tail(path.data() + 1, path.size() - 1);
 
   const PathSegment& func_segment = path.front();
 
   if (path_tail.empty()) {
     LOG(DFATAL) << "Invalid path";  // parser should not allow this.
   } else {
-    FlatDfs::Traverse(path_tail, json,
-                      [&](auto, flexbuffers::Reference val) { func_segment.Evaluate(val); });
+    FlatDfs::Traverse(path_tail, json, [&](auto, FlatJson val) { func_segment.Evaluate(val); });
   }
   AggFunction::Result res = func_segment.GetResult();
   flexbuffers::Builder fbb;
-  flexbuffers::Reference val = visit(  // Transform the result to a flexbuffer reference.
+  FlatJson val = visit(  // Transform the result to a flexbuffer reference.
       Overloaded{
-          [](monostate) { return flexbuffers::Reference{}; },
+          [](monostate) { return FlatJson{}; },
           [&](double d) {
             fbb.Double(d);
             fbb.Finish();
@@ -182,4 +181,84 @@ void EvaluatePath(const Path& path, flexbuffers::Reference json, PathFlatCallbac
   callback(nullopt, val);
 }
 
+JsonType FromFlat(FlatJson src) {
+  if (src.IsNull()) {
+    return JsonType::null();
+  }
+
+  if (src.IsBool()) {
+    return JsonType(src.AsBool());
+  }
+
+  if (src.IsInt()) {
+    return JsonType(src.AsInt64());
+  }
+
+  if (src.IsFloat()) {
+    return JsonType(src.AsDouble());
+  }
+  if (src.IsString()) {
+    flexbuffers::String str = src.AsString();
+    return JsonType(string_view{str.c_str(), str.size()});
+  }
+
+  CHECK(src.IsVector());
+  auto vec = src.AsVector();
+  JsonType js =
+      src.IsMap() ? JsonType{jsoncons::json_object_arg} : JsonType{jsoncons::json_array_arg};
+  auto keys = src.AsMap().Keys();
+  for (unsigned i = 0; i < vec.size(); ++i) {
+    JsonType value = FromFlat(vec[i]);
+    if (src.IsMap()) {
+      js[keys[i].AsKey()] = std::move(value);
+    } else {
+      js.push_back(std::move(value));
+    }
+  }
+  return js;
+}
+
+void FromJsonType(const JsonType& src, flexbuffers::Builder* fbb) {
+  if (src.is_null()) {
+    return fbb->Null();
+  }
+
+  if (src.is_bool()) {
+    return fbb->Bool(src.as_bool());
+  }
+
+  if (src.is_int64()) {
+    return fbb->Int(src.as<int64_t>());
+  }
+
+  if (src.is_double()) {
+    return fbb->Double(src.as_double());
+  }
+
+  if (src.is_string()) {
+    string_view sv = src.as_string_view();
+    fbb->String(sv.data(), sv.size());
+    return;
+  }
+
+  if (src.is_object()) {
+    auto range = src.object_range();
+    size_t start = fbb->StartMap();
+    for (auto it = range.cbegin(); it != range.cend(); ++it) {
+      fbb->Key(it->key().c_str(), it->key().size());
+      FromJsonType(it->value(), fbb);
+    }
+    fbb->EndMap(start);
+    return;
+  }
+
+  CHECK(src.is_array());
+  auto range = src.array_range();
+  size_t start = fbb->StartVector();
+  for (auto it = range.cbegin(); it != range.cend(); ++it) {
+    FromJsonType(*it, fbb);
+  }
+  fbb->EndVector(start, false, false);
+}
+
 }  // namespace dfly::json

src/core/json/path.h

@@ -37,7 +37,7 @@ class AggFunction {
       valid_ = ApplyImpl(src);
   }
 
-  void Apply(flexbuffers::Reference src) {
+  void Apply(FlatJson src) {
     if (valid_ != 0)
       valid_ = ApplyImpl(src);
   }
@@ -49,7 +49,7 @@ class AggFunction {
 
  protected:
   virtual bool ApplyImpl(const JsonType& src) = 0;
-  virtual bool ApplyImpl(flexbuffers::Reference src) = 0;
+  virtual bool ApplyImpl(FlatJson src) = 0;
   virtual Result GetResultImpl() const = 0;
 
   int valid_ = -1;
@@ -84,7 +84,7 @@ class PathSegment {
   }
 
   void Evaluate(const JsonType& json) const;
-  void Evaluate(flexbuffers::Reference json) const;
+  void Evaluate(FlatJson json) const;
   AggFunction::Result GetResult() const;
 
  private:
@@ -99,19 +99,28 @@ using Path = std::vector<PathSegment>;
 // Passes the key name for object fields or nullopt for array elements.
 // The second argument is a json value of either object fields or array elements.
 using PathCallback = absl::FunctionRef<void(std::optional<std::string_view>, const JsonType&)>;
-using PathFlatCallback =
-    absl::FunctionRef<void(std::optional<std::string_view>, flexbuffers::Reference)>;
+using PathFlatCallback = absl::FunctionRef<void(std::optional<std::string_view>, FlatJson)>;
 
 // Returns true if the entry should be deleted, false otherwise.
 using MutateCallback = absl::FunctionRef<bool(std::optional<std::string_view>, JsonType*)>;
 
 void EvaluatePath(const Path& path, const JsonType& json, PathCallback callback);
 
 // Same as above but for flatbuffers.
-void EvaluatePath(const Path& path, flexbuffers::Reference json, PathFlatCallback callback);
+void EvaluatePath(const Path& path, FlatJson json, PathFlatCallback callback);
 
 // returns number of matches found with the given path.
 unsigned MutatePath(const Path& path, MutateCallback callback, JsonType* json);
+
+// utility function to parse a jsonpath. Returns an error message if a parse error was
+// encountered.
 nonstd::expected<Path, std::string> ParsePath(std::string_view path);
 
+// Transforms FlatJson to JsonType.
+JsonType FromFlat(FlatJson src);
+
+// Transforms JsonType to a buffer using flexbuffers::Builder.
+// Does not call flexbuffers::Builder::Finish.
+void FromJsonType(const JsonType& src, flexbuffers::Builder* fbb);
+
 }  // namespace dfly::json