Add blockchain tests loading and running

test/CMakeLists.txt

@@ -15,6 +15,7 @@ find_package(benchmark CONFIG REQUIRED)
 
 add_subdirectory(utils)
 add_subdirectory(bench)
+add_subdirectory(blockchaintest)
 add_subdirectory(eofparse)
 add_subdirectory(integration)
 add_subdirectory(internal_benchmarks)
@@ -24,7 +25,7 @@ add_subdirectory(eoftest)
 add_subdirectory(t8n)
 add_subdirectory(unittests)
 
-set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-state evmone-statetest evmone-eoftest evmone-t8n evmone-unittests)
+set(targets evmone-bench evmone-bench-internal evmone-eofparse evmone-blockchaintest evmone-state evmone-statetest evmone-eoftest evmone-t8n evmone-unittests)
 
 if(EVMONE_FUZZING)
     add_subdirectory(eofparsefuzz)

test/blockchaintest/.clang-tidy

@@ -0,0 +1,3 @@
+InheritParentConfig: true
+Checks: >
+  -clang-analyzer-cplusplus.NewDeleteLeaks

test/blockchaintest/CMakeLists.txt

@@ -0,0 +1,17 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2022 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+hunter_add_package(nlohmann_json)
+find_package(nlohmann_json CONFIG REQUIRED)
+
+add_executable(evmone-blockchaintest)
+target_link_libraries(evmone-blockchaintest PRIVATE evmone evmone::statetestutils GTest::gtest)
+target_include_directories(evmone-blockchaintest PRIVATE ${evmone_private_include_dir})
+target_sources(
+    evmone-blockchaintest PRIVATE
+    blockchaintest.hpp
+    blockchaintest.cpp
+    blockchaintest_loader.cpp
+    blockchaintest_runner.cpp
+)

test/blockchaintest/blockchaintest.cpp

@@ -0,0 +1,93 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2022 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "blockchaintest.hpp"
+#include <CLI/CLI.hpp>
+#include <evmone/evmone.h>
+#include <gtest/gtest.h>
+#include <iostream>
+namespace fs = std::filesystem;
+
+namespace
+{
+class BlockchainTest : public testing::Test
+{
+    fs::path m_json_test_file;
+    evmc::VM& m_vm;
+
+public:
+    explicit BlockchainTest(fs::path json_test_file, evmc::VM& vm) noexcept
+      : m_json_test_file{std::move(json_test_file)}, m_vm{vm}
+    {}
+
+    void TestBody() final
+    {
+        std::ifstream f{m_json_test_file};
+        evmone::test::run_blockchain_test(evmone::test::load_blockchain_test(f), m_vm);
+    }
+};
+
+void register_test(const std::string& suite_name, const fs::path& file, evmc::VM& vm)
+{
+    testing::RegisterTest(suite_name.c_str(), file.stem().string().c_str(), nullptr, nullptr,
+        file.string().c_str(), 0,
+        [file, &vm]() -> testing::Test* { return new BlockchainTest(file, vm); });
+}
+
+void register_test_files(const fs::path& root, evmc::VM& vm)
+{
+    if (is_directory(root))
+    {
+        std::vector<fs::path> test_files;
+        std::copy_if(fs::recursive_directory_iterator{root}, fs::recursive_directory_iterator{},
+            std::back_inserter(test_files), [](const fs::directory_entry& entry) {
+                return entry.is_regular_file() && entry.path().extension() == ".json";
+            });
+        std::sort(test_files.begin(), test_files.end());
+
+        for (const auto& p : test_files)
+            register_test(fs::relative(p, root).parent_path().string(), p, vm);
+    }
+    else  // Treat as a file.
+    {
+        register_test(root.parent_path().string(), root, vm);
+    }
+}
+}  // namespace
+
+
+int main(int argc, char* argv[])
+{
+    try
+    {
+        testing::InitGoogleTest(&argc, argv);  // Process GoogleTest flags.
+
+        CLI::App app{"evmone blockchain test runner"};
+
+        std::vector<std::string> paths;
+        app.add_option("path", paths, "Path to test file or directory")
+            ->required()
+            ->check(CLI::ExistingPath);
+
+        bool trace_flag = false;
+        app.add_flag("--trace", trace_flag, "Enable EVM tracing");
+
+        CLI11_PARSE(app, argc, argv);
+
+        evmc::VM vm{evmc_create_evmone()};
+
+        if (trace_flag)
+            vm.set_option("trace", "1");
+
+        for (const auto& p : paths)
+            register_test_files(p, vm);
+
+        return RUN_ALL_TESTS();
+    }
+    catch (const std::exception& ex)
+    {
+        std::cerr << ex.what() << "\n";
+        return -1;
+    }
+}

test/blockchaintest/blockchaintest.hpp

@@ -0,0 +1,75 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include "../state/bloom_filter.hpp"
+#include "../state/state.hpp"
+#include <evmc/evmc.hpp>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+namespace json = nlohmann;
+
+namespace evmone::test
+{
+
+// https://ethereum.org/en/developers/docs/blocks/
+struct BlockHeader
+{
+    hash256 parent_hash;
+    address coinbase;
+    hash256 state_root;
+    hash256 receipts_root;
+    state::BloomFilter logs_bloom;
+    int64_t difficulty;
+    bytes32 prev_randao;
+    int64_t block_number;
+    int64_t gas_limit;
+    int64_t gas_used;
+    int64_t timestamp;
+    bytes extra_data;
+    uint64_t base_fee_per_gas;
+    hash256 hash;
+    hash256 transactions_root;
+    hash256 withdrawal_root;
+};
+
+struct TestBlock
+{
+    state::BlockInfo block_info;
+    state::State pre_state;
+    std::vector<state::Transaction> transactions;
+
+    BlockHeader expected_block_header;
+};
+
+TestBlock load_test_block(const json::json& j);
+
+struct BlockchainTransitionTest
+{
+    struct Case
+    {
+        std::string name;
+        std::vector<TestBlock> test_blocks;
+        struct Expectation
+        {
+            hash256 last_block_hash;
+            std::variant<state::State, hash256> post_state;
+        };
+
+        BlockHeader genesis_block_header;
+        state::State pre_state;
+        evmc_revision rev;
+
+        Expectation expectation;
+    };
+
+    std::vector<Case> cases;
+};
+
+BlockchainTransitionTest load_blockchain_test(std::istream& input);
+
+void run_blockchain_test(const BlockchainTransitionTest& test, evmc::VM& vm);
+
+}  // namespace evmone::test

test/blockchaintest/blockchaintest_loader.cpp

@@ -0,0 +1,137 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2023 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+
+#include "../statetest/statetest.hpp"
+#include "../utils/utils.hpp"
+#include "blockchaintest.hpp"
+
+namespace evmone::test
+{
+
+namespace
+{
+template <typename T>
+T load_if_exists(const json::json& j, std::string_view key)
+{
+    if (const auto it = j.find(key); it != j.end())
+        return from_json<T>(*it);
+    return {};
+}
+}  // namespace
+
+template <>
+BlockHeader from_json<BlockHeader>(const json::json& j)
+{
+    return {from_json<hash256>(j.at("parentHash")), from_json<address>(j.at("coinbase")),
+        from_json<hash256>(j.at("stateRoot")), from_json<hash256>(j.at("receiptTrie")),
+        state::bloom_filter_from_bytes(from_json<bytes>(j.at("bloom"))),
+        load_if_exists<int64_t>(j, "difficulty"), load_if_exists<bytes32>(j, "mixHash"),
+        from_json<int64_t>(j.at("number")), from_json<int64_t>(j.at("gasLimit")),
+        from_json<int64_t>(j.at("gasUsed")), from_json<int64_t>(j.at("timestamp")),
+        from_json<bytes>(j.at("extraData")), load_if_exists<uint64_t>(j, "baseFeePerGas"),
+        from_json<hash256>(j.at("hash")), from_json<hash256>(j.at("transactionsTrie")),
+        load_if_exists<hash256>(j, "withdrawalsRoot")};
+}
+
+static TestBlock load_test_block(const json::json& j, evmc_revision rev)
+{
+    using namespace state;
+    TestBlock tb;
+
+    if (const auto it = j.find("blockHeader"); it != j.end())
+    {
+        tb.expected_block_header = from_json<BlockHeader>(*it);
+        tb.block_info.number = tb.expected_block_header.block_number;
+        tb.block_info.timestamp = tb.expected_block_header.timestamp;
+        tb.block_info.gas_limit = tb.expected_block_header.gas_limit;
+        tb.block_info.coinbase = tb.expected_block_header.coinbase;
+        tb.block_info.difficulty = tb.expected_block_header.difficulty;
+        tb.block_info.prev_randao = tb.expected_block_header.prev_randao;
+        tb.block_info.base_fee = tb.expected_block_header.base_fee_per_gas;
+
+        // Override prev_randao with difficulty pre-Merge
+        if (rev < EVMC_PARIS)
+        {
+            tb.block_info.prev_randao =
+                intx::be::store<bytes32>(intx::uint256{tb.block_info.difficulty});
+        }
+    }
+
+    if (const auto it = j.find("expectException"); it != j.end())
+    {
+        // TODO: Add support for invalid blocks.
+        throw std::runtime_error("tests invalid blocks are not supported");
+    }
+
+    if (const auto it = j.find("transactionSequence"); it != j.end())
+    {
+        // TODO: Add support for invalid blocks.
+        throw std::runtime_error("tests invalid transactions are not supported");
+    }
+
+    if (const auto it = j.find("uncleHeaders"); it != j.end())
+    {
+        const auto current_block_number = tb.block_info.number;
+        for (const auto& ommer : *it)
+        {
+            tb.block_info.ommers.push_back({from_json<address>(ommer.at("coinbase")),
+                static_cast<uint32_t>(
+                    current_block_number - from_json<int64_t>(ommer.at("number")))});
+        }
+    }
+
+    if (auto it = j.find("withdrawals"); it != j.end())
+    {
+        for (const auto& withdrawal : *it)
+            tb.block_info.withdrawals.emplace_back(from_json<Withdrawal>(withdrawal));
+    }
+
+    if (auto it = j.find("transactions"); it != j.end())
+    {
+        for (const auto& tx : *it)
+            tb.transactions.emplace_back(from_json<Transaction>(tx));
+    }
+
+    return tb;
+}
+
+namespace
+{
+BlockchainTransitionTest::Case load_blockchain_test_case(
+    const std::string& name, const json::json& j)
+{
+    using namespace state;
+
+    BlockchainTransitionTest::Case c;
+    c.name = name;
+    c.genesis_block_header = from_json<BlockHeader>(j.at("genesisBlockHeader"));
+    c.pre_state = from_json<State>(j.at("pre"));
+    c.rev = to_rev(j.at("network").get<std::string>());
+
+    for (const auto& el : j.at("blocks"))
+        c.test_blocks.emplace_back(load_test_block(el, c.rev));
+
+    c.expectation.last_block_hash = from_json<hash256>(j.at("lastblockhash"));
+
+    if (const auto it = j.find("postState"); it != j.end())
+        c.expectation.post_state = from_json<State>(*it);
+    else if (const auto it_hash = j.find("postStateHash"); it_hash != j.end())
+        c.expectation.post_state = from_json<hash256>(*it_hash);
+
+    return c;
+}
+}  // namespace
+
+static void from_json(const json::json& j, BlockchainTransitionTest& o)
+{
+    for (const auto& elem_it : j.items())
+        o.cases.emplace_back(load_blockchain_test_case(elem_it.key(), elem_it.value()));
+}
+
+BlockchainTransitionTest load_blockchain_test(std::istream& input)
+{
+    return json::json::parse(input).get<BlockchainTransitionTest>();
+}
+
+}  // namespace evmone::test