Properly encode results from the tx RPC command:

The `tx` command supports output in both "text" and "binary" modes,
controlled by the binary flag. For more details on the command and
the possible arguments, please see: https://xrpl.org/tx.html.

The existing handler would incorrectly deal with metadata when in
binary mode. This commit corrects this issue, ensuring that the
metadata is properly encoded, depending on the mode.

src/ripple/rpc/handlers/Tx.cpp

@@ -236,7 +236,14 @@ doTxHelp(RPC::Context& context, TxArgs const& args)
 
     if (ledger && meta)
     {
-        result.meta = meta;
+        if (args.binary)
+        {
+            result.meta = meta->getAsObject().getSerializer().getData();
+        }
+        else
+        {
+            result.meta = meta;
+        }
         result.validated = isValidated(
             context.ledgerMaster, ledger->info().seq, ledger->info().hash);
     }

src/test/rpc/Transaction_test.cpp

@@ -49,18 +49,23 @@ class Transaction_test : public beast::unit_test::suite
         env.close();
 
         std::vector<std::shared_ptr<STTx const>> txns;
+        std::vector<std::shared_ptr<STObject const>> metas;
         auto const startLegSeq = env.current()->info().seq;
         for (int i = 0; i < 750; ++i)
         {
             env(noop(alice));
             txns.emplace_back(env.tx());
             env.close();
+            metas.emplace_back(
+                env.closed()->txRead(env.tx()->getTransactionID()).second);
         }
         auto const endLegSeq = env.closed()->info().seq;
 
         // Find the existing transactions
-        for (auto&& tx : txns)
+        for (size_t i = 0; i < txns.size(); ++i)
         {
+            auto const& tx = txns[i];
+            auto const& meta = metas[i];
             auto const result = env.rpc(
                 COMMAND,
                 to_string(tx->getTransactionID()),
@@ -69,6 +74,12 @@ class Transaction_test : public beast::unit_test::suite
                 to_string(endLegSeq));
 
             BEAST_EXPECT(result[jss::result][jss::status] == jss::success);
+            BEAST_EXPECT(
+                result[jss::result][jss::tx] ==
+                strHex(tx->getSerializer().getData()));
+            BEAST_EXPECT(
+                result[jss::result][jss::meta] ==
+                strHex(meta->getSerializer().getData()));
         }
 
         auto const tx = env.jt(noop(alice), seq(env.seq(alice))).stx;