Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rpcdaemon: negotiation and validation of HTTP compression scheme #1914

Merged
merged 5 commits into from
Mar 19, 2024
Merged

rpcdaemon: negotiation and validation of HTTP compression scheme #1914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9ffec9f
add algo validation and selection
lupin012 Mar 17, 2024
3d8cac6
add algo validation and selection
lupin012 Mar 17, 2024
19ef83f
modified tempi buffer into string
lupin012 Mar 17, 2024
5e6390f
zlib using boost:iostreams
lupin012 Mar 18, 2024
ced901e
Fix naming conventions
canepat Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 52 additions & 35 deletions silkworm/rpc/http/connection.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#ifndef ZLIB_CONST
#define ZLIB_CONST
#endif
#include <zlib.h>

#include <array>
#include <exception>
Expand All @@ -31,6 +30,9 @@
#include <boost/asio/write.hpp>
#include <boost/beast/http/chunk_encode.hpp>
#include <boost/beast/http/write.hpp>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <jwt-cpp/jwt.h>
#include <jwt-cpp/traits/nlohmann-json/defaults.h>

Expand All @@ -42,6 +44,7 @@ namespace silkworm::rpc::http {
static constexpr std::string_view kMaxAge{"600"};
static constexpr auto kMaxPayloadSize{30 * kMebi}; // 30MiB
static constexpr std::array kAcceptedContentTypes{"application/json", "application/jsonrequest", "application/json-rpc"};
static std::vector<std::string> kSupportedCompressionList{"gzip"}; // specify the compression algo in priority level

Connection::Connection(boost::asio::io_context& io_context,
commands::RpcApi& api,
Expand Down Expand Up @@ -174,6 +177,31 @@ Task<void> Connection::handle_actual_request(const boost::beast::http::request<b
co_return;
}

const auto accept_encoding = req[boost::beast::http::field::accept_encoding];
if (!http_compression_ && !accept_encoding.empty()) {
co_await do_write("unsupported compression\n", boost::beast::http::status::unsupported_media_type, "identity");
co_return;
}

std::string selected_compression = "";
if (http_compression_ && !accept_encoding.empty()) {
selected_compression = select_compression_algo(accept_encoding);
if (selected_compression.empty()) {
std::string complete_list;
bool first = true;
for (std::string curr_compression : kSupportedCompressionList) {
if (first) {
first = false;
} else {
complete_list += ", ";
}
complete_list += curr_compression;
}
co_await do_write("unsupported requested compression\n", boost::beast::http::status::unsupported_media_type, complete_list);
co_return;
}
}

if (!is_method_allowed(req.method())) {
co_await do_write("method not allowed\n", boost::beast::http::status::method_not_allowed);
co_return;
Expand All @@ -200,11 +228,10 @@ Task<void> Connection::handle_actual_request(const boost::beast::http::request<b
vary_ = req[boost::beast::http::field::vary];
origin_ = req[boost::beast::http::field::origin];
method_ = req.method();
auto encoding = req[boost::beast::http::field::accept_encoding];

auto rsp_content = co_await request_handler_.handle(req.body());
if (rsp_content) {
co_await do_write(rsp_content->append("\n"), boost::beast::http::status::ok, !encoding.empty());
co_await do_write(rsp_content->append("\n"), boost::beast::http::status::ok, selected_compression);
}
}

Expand Down Expand Up @@ -262,7 +289,7 @@ Task<std::size_t> Connection::write(std::string_view content, bool /*last*/) {
co_return bytes_transferred;
}

Task<void> Connection::do_write(const std::string& content, boost::beast::http::status http_status, bool compress) {
Task<void> Connection::do_write(const std::string& content, boost::beast::http::status http_status, const std::string& content_encoding) {
try {
SILK_TRACE << "Connection::do_write response: " << http_status << " content: " << content;
boost::beast::http::response<boost::beast::http::string_body> res{http_status, request_http_version_};
Expand All @@ -276,19 +303,23 @@ Task<void> Connection::do_write(const std::string& content, boost::beast::http::
res.set(boost::beast::http::field::date, get_date_time());
res.erase(boost::beast::http::field::host);
res.keep_alive(request_keep_alive_);
if (compress) {
const std::string compression_type = "gzip";
res.set(boost::beast::http::field::content_encoding, compression_type);
if (http_status == boost::beast::http::status::ok && !content_encoding.empty()) {
// Positive response w/ compression required
res.set(boost::beast::http::field::content_encoding, content_encoding);
std::string compressed_data;
try {
compress_data(content, compressed_data);
} catch (const std::exception& e) {
SILK_ERROR << "Connection::compress_data exception: " << e.what();
SILK_ERROR << "Connection::do_write cannot compress exception: " << e.what();
throw;
}
res.content_length(compressed_data.length());
res.body() = std::move(compressed_data);
} else {
// Any negative response or positive response w/o compression
if (!content_encoding.empty()) {
res.set(boost::beast::http::field::accept_encoding, content_encoding); // Indicate the supported encoding
}
res.content_length(content.size());
res.body() = content;
}
Expand All @@ -310,33 +341,10 @@ Task<void> Connection::do_write(const std::string& content, boost::beast::http::
}

void Connection::compress_data(const std::string& clear_data, std::string& compressed_data) {
z_stream strm;

std::memset(&strm, 0, sizeof(strm));
int ret = Z_OK;
ret = deflateInit2(&strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 | 16, 8, Z_DEFAULT_STRATEGY);
if (ret != Z_OK) {
throw std::runtime_error("deflateInit2 fail");
}
strm.avail_in = static_cast<unsigned int>(clear_data.size());
strm.next_in = reinterpret_cast<const Bytef*>(clear_data.c_str());

do {
strm.next_out = reinterpret_cast<Bytef*>(temp_compressed_buffer_);
strm.avail_out = sizeof(temp_compressed_buffer_);

ret = deflate(&strm, Z_FINISH);
if (ret < 0) {
deflateEnd(&strm);
throw std::runtime_error("deflate fail");
}
if (compressed_data.size() < strm.total_out) {
// append the block to the output string
compressed_data.append(temp_compressed_buffer_, strm.total_out - compressed_data.size());
}
} while (ret != Z_STREAM_END);

deflateEnd(&strm);
boost::iostreams::filtering_ostream out;
out.push(boost::iostreams::gzip_compressor());
out.push(boost::iostreams::back_inserter(compressed_data));
boost::iostreams::copy(boost::make_iterator_range(clear_data), out);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can probably improve here by keeping the boost::iostreams::filtering_ostream instance and the compressed data buffer as state variables in http::Connection in order to have gzip initialisation/finalisation happen only once per connection (as opposed to doing it for every data compression). Next PR

}

Connection::AuthorizationResult Connection::is_request_authorized(const boost::beast::http::request<boost::beast::http::string_body>& req) {
Expand Down Expand Up @@ -407,6 +415,15 @@ void Connection::set_cors(boost::beast::http::response<Body>& res) {
}
}

std::string Connection::select_compression_algo(const std::string& requested_compression) {
for (std::string curr_compression : kSupportedCompressionList) {
if (requested_compression.find(curr_compression) != std::string::npos) {
return curr_compression;
}
}
return "";
}

bool Connection::is_origin_allowed(const std::vector<std::string>& allowed_origins, const std::string& origin) {
if (allowed_origins.size() == 1 && allowed_origins[0] == "*") {
return true;
Expand Down
6 changes: 3 additions & 3 deletions silkworm/rpc/http/connection.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,12 @@ class Connection : public StreamWriter {
Task<bool> do_read();

//! Perform an asynchronous write operation.
Task<void> do_write(const std::string& content, boost::beast::http::status http_status, bool compress = false);
Task<void> do_write(const std::string& content, boost::beast::http::status http_status, const std::string& content_encoding = "");

static std::string get_date_time();

std::string select_compression_algo(const std::string& request_compression);

void compress_data(const std::string& clear_data, std::string& compressed_data);

//! Socket for the connection.
Expand Down Expand Up @@ -119,8 +121,6 @@ class Connection : public StreamWriter {
std::string vary_;
std::string origin_;
boost::beast::http::verb method_{boost::beast::http::verb::unknown};

char temp_compressed_buffer_[10 * kMebi]{};
};

} // namespace silkworm::rpc::http
Loading