From 0ce4f61bfbfbf329a3d91b468446280bfa7c600f Mon Sep 17 00:00:00 2001 From: Adam Sapek Date: Sun, 1 Jan 2017 19:34:51 -0800 Subject: [PATCH] Extend bf utility to support multiple payloads The `bf` C++ example is a handy utility to work with files containing arbitrary Bond payloads. This change extends it to support files with more than one Bond payload. The most obvious use case is files with a sequence of records (e.g. logs). The less obvious but very powerful scenario is extracting some kind of header that precedes the actual Bond payload of interest. This is possible because a Bond schema in Simple Binary protocol can be used to model many kinds of arbitrary headers (e.g. any fixed size header aligned to octet boundary). The change is backward compatible and existing command line arguments retain their old semantics. In order to process multiple payloads user can specified multiple `--schema` and/or multiple `--from` arguments, e.g.: bf --from=simple --schema=header.json,payload.json file In the above example `bf` will first use Simple Binary to decode header in schema specified in `header.json` and then will try to guess the protocol of the next payload (since only one `--from` argument was specified) and decode it using schema specified by the `payload.json` file. Multiple values for the `--schema` and `--from` arguments can be specified either as comma delimited values (like in the example above) or by passing the argument multiple times, e.g.: bf --from=fast --from=fast file --- .../bond/protocol/detail/rapidjson_helper.h | 2 +- examples/cpp/core/bf/cmd_arg.bond | 8 +-- examples/cpp/core/bf/main.cpp | 63 ++++++++++++++----- 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/cpp/inc/bond/protocol/detail/rapidjson_helper.h b/cpp/inc/bond/protocol/detail/rapidjson_helper.h index 39cfe47c22..31de90f8d4 100644 --- a/cpp/inc/bond/protocol/detail/rapidjson_helper.h +++ b/cpp/inc/bond/protocol/detail/rapidjson_helper.h @@ -85,7 +85,7 @@ class RapidJsonInputStream } private: - Buffer* input; + typename std::remove_reference::type* input; uint8_t current; size_t count; }; diff --git a/examples/cpp/core/bf/cmd_arg.bond b/examples/cpp/core/bf/cmd_arg.bond index 517f9ea2c1..fa85497056 100644 --- a/examples/cpp/core/bf/cmd_arg.bond +++ b/examples/cpp/core/bf/cmd_arg.bond @@ -24,17 +24,17 @@ struct Options [help("output file")] 1: string output = "stdout"; - [help("guess | marshal | compact | compact2 | fast | simple | simple2")] - 2: Protocol from = guess; + [help("guess | marshal | compact | compact2 | fast | simple | simple2, default guess")] + 2: list from; [help("json | compact | compact2 | fast | simple | simple2")] 3: Protocol to = json; - [help("include values for omitted optional fields when transcoding to json format with input schema)")] + [help("include values for omitted optional fields when transcoding to json format with input schema")] 4: bool all_fields; [help("file with marshaled schema of the input; required when input format is simple*")] - 5: string schema; + 5: list schema; [help("input file")] [naked("")] diff --git a/examples/cpp/core/bf/main.cpp b/examples/cpp/core/bf/main.cpp index dba0c81aea..ada8516dfc 100644 --- a/examples/cpp/core/bf/main.cpp +++ b/examples/cpp/core/bf/main.cpp @@ -102,7 +102,7 @@ bond::SchemaDef LoadSchema(const std::string& file) char c; tryJson.Read(c); - + return (c == '{') ? bond::Deserialize(bond::SimpleJsonReader(input)) : bond::Unmarshal(input); @@ -111,9 +111,9 @@ bond::SchemaDef LoadSchema(const std::string& file) template void TranscodeFromTo(Reader& reader, Writer& writer, const Options& options) { - if (!options.schema.empty()) + if (!options.schema.empty() && !options.schema.front().empty()) { - bond::SchemaDef schema(LoadSchema(options.schema)); + bond::SchemaDef schema(LoadSchema(options.schema.front())); bond::bonded >(reader, bond::RuntimeSchema(schema)).Serialize(writer); } else @@ -126,9 +126,9 @@ void TranscodeFromTo(Reader& reader, Writer& writer, const Options& options) template void TranscodeFromTo(InputFile& input, Writer& writer, const Options& options) { - if (!options.schema.empty()) + if (!options.schema.empty() && !options.schema.front().empty()) { - bond::SchemaDef schema(LoadSchema(options.schema)); + bond::SchemaDef schema(LoadSchema(options.schema.front())); bond::SelectProtocolAndApply(bond::RuntimeSchema(schema), input, SerializeTo(writer)); } else @@ -193,23 +193,31 @@ bool TranscodeFrom(Reader reader, const Options& options) } } - -bool Transcode(InputFile& input, const Options& options) +template +bool Transcode(Input input, const Options& options) { - switch (options.from) + bf::Protocol from = options.from.empty() ? guess : options.from.front(); + + if (from == guess) + { + from = Guess(input); + std::cerr << std::endl << "Guessed " << ToString(from) << std::endl; + } + + switch (from) { case marshal: return TranscodeFrom(input, options); case compact: - return TranscodeFrom(bond::CompactBinaryReader(input), options); + return TranscodeFrom(bond::CompactBinaryReader(input), options); case compact2: - return TranscodeFrom(bond::CompactBinaryReader(input, bond::v2), options); + return TranscodeFrom(bond::CompactBinaryReader(input, bond::v2), options); case fast: - return TranscodeFrom(bond::FastBinaryReader(input), options); + return TranscodeFrom(bond::FastBinaryReader(input), options); case simple: - return TranscodeFrom(bond::SimpleBinaryReader(input), options); + return TranscodeFrom(bond::SimpleBinaryReader(input), options); case simple2: - return TranscodeFrom(bond::SimpleBinaryReader(input, bond::v2), options); + return TranscodeFrom(bond::SimpleBinaryReader(input, bond::v2), options); default: return false; } @@ -226,11 +234,32 @@ int main(int argc, char** argv) { InputFile input(options.file); - if (options.from == guess) - std::cerr << "Guessed " << ToString(options.from = Guess(input)) << std::endl; + do + { + // In order to decode multiple payloads from a file we need to + // use InputFile& however that usage doesn't support marshalled + // bonded in untagged protocols. As a compromise we use + // InputFile for the last payload and InputFile& otherwise. + if (options.schema.size() > 1 || options.from.size() > 1) + { + if (!Transcode(input, options)) + return 1; + } + else + { + if (!Transcode(input, options)) + return 1; + } + + if (!options.schema.empty()) + options.schema.pop_front(); + + if (!options.from.empty()) + options.from.pop_front(); + } + while (!options.schema.empty() || !options.from.empty()); - if (Transcode(input, options)) - return 0; + return 0; } } catch(const std::exception& error)