Skip to content

Commit

Permalink
Move merkel benchmarks from unittest to benchmark.
Browse files Browse the repository at this point in the history
  • Loading branch information
@greg7mdp
greg7mdp committed Apr 3, 2024
1 parent cc56c51 commit facbee8
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 114 deletions.
15 changes: 11 additions & 4 deletions benchmark/benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ std::map<std::string, std::function<void()>> features {
{ "key", key_benchmarking },
{ "hash", hash_benchmarking },
{ "blake2", blake2_benchmarking },
{ "bls", bls_benchmarking }
{ "bls", bls_benchmarking },
{ "merkle", merkle_benchmarking }
};

// values to control cout format
Expand All @@ -34,6 +35,10 @@ void set_num_runs(uint32_t runs) {
num_runs = runs;
}

uint32_t get_num_runs() {
return num_runs;
}

void print_header() {
std::cout << std::left << std::setw(name_width) << "function"
<< std::setw(runs_width) << "runs"
Expand Down Expand Up @@ -63,12 +68,14 @@ bytes to_bytes(const std::string& source) {
return output;
};

void benchmarking(const std::string& name, const std::function<void()>& func) {
void benchmarking(const std::string& name, const std::function<void()>& func,
std::optional<size_t> opt_num_runs /* = {} */) {
uint64_t total{0};
uint64_t min{std::numeric_limits<uint64_t>::max()};
uint64_t max{0};
uint32_t runs = opt_num_runs ? *opt_num_runs : num_runs;

for (auto i = 0U; i < num_runs; ++i) {
for (auto i = 0U; i < runs; ++i) {
auto start_time = std::chrono::high_resolution_clock::now();
func();
auto end_time = std::chrono::high_resolution_clock::now();
Expand All @@ -79,7 +86,7 @@ void benchmarking(const std::string& name, const std::function<void()>& func) {
max = std::max(max, duration);
}

print_results(name, num_runs, total, min, max);
print_results(name, runs, total, min, max);
}

} // benchmark
5 changes: 4 additions & 1 deletion benchmark/benchmark.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@
#include <map>
#include <vector>
#include <limits>
#include <optional>

#include <fc/crypto/hex.hpp>

namespace eosio::benchmark {
using bytes = std::vector<char>;

void set_num_runs(uint32_t runs);
uint32_t get_num_runs();
std::map<std::string, std::function<void()>> get_features();
void print_header();
bytes to_bytes(const std::string& source);
Expand All @@ -21,7 +23,8 @@ void key_benchmarking();
void hash_benchmarking();
void blake2_benchmarking();
void bls_benchmarking();
void merkle_benchmarking();

void benchmarking(const std::string& name, const std::function<void()>& func);
void benchmarking(const std::string& name, const std::function<void()>& func, std::optional<size_t> num_runs = {});

} // benchmark
69 changes: 69 additions & 0 deletions benchmark/merkle.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#include <benchmark.hpp>
#include <eosio/chain/incremental_merkle.hpp>
#include <eosio/chain/incremental_merkle_legacy.hpp>
#include <random>

namespace eosio::benchmark {

using namespace eosio::chain;

std::vector<digest_type> create_test_digests(size_t n) {
std::vector<digest_type> v;
v.reserve(n);
for (size_t i=0; i<n; ++i)
v.push_back(fc::sha256::hash(std::string{"Node"} + std::to_string(i)));
return v;
}

void benchmark_calc_merkle(uint32_t size_boost) {
using namespace std::string_literals;
const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case

const std::vector<digest_type> digests = create_test_digests(num_digests);
const deque<digest_type> deq { digests.begin(), digests.end() };

auto num_str = std::to_string(size_boost);
while(num_str.size() < 4)
num_str.insert(0, 1, ' ');
auto msg_header = "Calc, "s + num_str + ",000 digests, "s;
uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));
benchmarking(msg_header + "legacy: ", [&]() { calculate_merkle_legacy(deq); }, num_runs);
benchmarking(msg_header + "savanna:", [&]() { calculate_merkle(digests.begin(), digests.end()); }, num_runs);
}

void benchmark_incr_merkle(uint32_t size_boost) {
using namespace std::string_literals;
const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case

const std::vector<digest_type> digests = create_test_digests(num_digests);

auto num_str = std::to_string(size_boost);
while(num_str.size() < 4)
num_str.insert(0, 1, ' ');
auto msg_header = "Incr, "s + num_str + ",000 digests, "s;
uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));

auto incr = [&](const auto& incr_tree) {
auto work_tree = incr_tree;
for (const auto& d : digests)
work_tree.append(d);
return work_tree.get_root();
};

benchmarking(msg_header + "legacy: ", [&]() { incr(incremental_merkle_tree_legacy()); }, num_runs);
benchmarking(msg_header + "savanna:", [&]() { incr(incremental_merkle_tree()); }, num_runs);
}

// register benchmarking functions
void merkle_benchmarking() {
benchmark_calc_merkle(1000); // calculate_merkle of very large sequence (1,000,000 digests)
benchmark_calc_merkle(50); // calculate_merkle of large sequence (50,000 digests)
benchmark_calc_merkle(1); // calculate_merkle of small sequence (1000 digests)
std::cout << "\n";

benchmark_incr_merkle(100); // incremental_merkle of very large sequence (100,000 digests)
benchmark_incr_merkle(25); // incremental_merkle of large sequence (25,000 digests)
benchmark_incr_merkle(1); // incremental_merkle of small sequence (1000 digests)
}

}
109 changes: 0 additions & 109 deletions unittests/merkle_tree_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
#include <eosio/chain/incremental_merkle_legacy.hpp>
#include <boost/test/unit_test.hpp>
#include <fc/crypto/sha256.hpp>
#include <chrono>

using namespace eosio::chain;
using eosio::chain::detail::make_legacy_digest_pair;
Expand Down Expand Up @@ -181,112 +180,4 @@ BOOST_AUTO_TEST_CASE(consistency_over_large_range) {
}
}

class stopwatch {
public:
stopwatch(std::string msg) : _msg(std::move(msg)) { _start = clock::now(); }

~stopwatch() { std::cout << _msg << get_time_us()/1000000 << "s\n"; }

double get_time_us() const {
using duration_t = std::chrono::duration<double, std::micro>;
return std::chrono::duration_cast<duration_t>(clock::now() - _start).count();
}

using clock = std::chrono::high_resolution_clock;
using point = std::chrono::time_point<clock>;

std::string _msg;
point _start;
};

BOOST_AUTO_TEST_CASE(perf_test_one_large) {
auto perf_test = [](const std::string& type, auto&& incr_tree, auto&& calc_fn) {
using namespace std::string_literals;
constexpr size_t num_digests = 1000ull * 1000ull; // don't use exact powers of 2 as it is a special case

const std::vector<digest_type> digests = create_test_digests(num_digests);
const deque<digest_type> deq { digests.begin(), digests.end() };

auto msg_header = "1 sequence of "s + std::to_string(num_digests) + " digests: time for "s;

auto incr_root = [&]() {
stopwatch s(msg_header + type + " incremental_merkle: ");
for (const auto& d : digests)
incr_tree.append(d);
return incr_tree.get_root();
}();

auto calc_root = [&]() {
stopwatch s(msg_header + type + " calculate_merkle: ");
return calc_fn(deq);
}();

return std::make_pair(incr_root, calc_root);
};

{
auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(),
[](const deque<digest_type>& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here
BOOST_CHECK_EQUAL(incr_root, calc_root);
}

{
auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy);
BOOST_CHECK_EQUAL(incr_root, calc_root);
}
}


BOOST_AUTO_TEST_CASE(perf_test_many_small) {

auto perf_test = [](const std::string& type, const auto& incr_tree, auto&& calc_fn) {
using namespace std::string_literals;
constexpr size_t num_digests = 10000; // don't use exact powers of 2 as it is a special case
constexpr size_t num_runs = 100;

const std::vector<digest_type> digests = create_test_digests(num_digests);
const deque<digest_type> deq { digests.begin(), digests.end() };

deque<digest_type> results(num_runs);

auto incr = [&]() {
auto work_tree = incr_tree;
for (const auto& d : digests)
work_tree.append(d);
return work_tree.get_root();
};

auto calc = [&]() { return calc_fn(deq); };

auto msg_header = std::to_string(num_runs) + " runs for a sequence of "s + std::to_string(num_digests) + " digests: time for "s;

auto incr_root = [&]() {
stopwatch s(msg_header + type + " incremental_merkle: ");
for (auto& r : results)
r = incr();
return calc_fn(results);
}();

auto calc_root = [&]() {
stopwatch s(msg_header + type + " calculate_merkle: ");
for (auto& r : results)
r = calc();
return calc_fn(results);
}();

return std::make_pair(incr_root, calc_root);
};

{
auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(),
[](const deque<digest_type>& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here
BOOST_CHECK_EQUAL(incr_root, calc_root);
}

{
auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy);
BOOST_CHECK_EQUAL(incr_root, calc_root);
}
}

BOOST_AUTO_TEST_SUITE_END()

0 comments on commit facbee8

Please sign in to comment.