Merge branch 'main' into mining-options-refactor

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
-## Next Release
+## Next release
+- Cache last n blocks by using a new Besu flag --cache-last-blocks=n
 
 ### Breaking Changes
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1233,6 +1233,11 @@ static class PermissionsOptionGroup {
           "Specifies the maximum number of blocks to retrieve logs from via RPC. Must be >=0. 0 specifies no limit  (default: ${DEFAULT-VALUE})")
   private final Long rpcMaxLogsRange = 5000L;
 
+  @CommandLine.Option(
+      names = {"--cache-last-blocks"},
+      description = "Specifies the number of last blocks to cache  (default: ${DEFAULT-VALUE})")
+  private final Integer numberOfblocksToCache = 0;
+
   @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions;
 
   @Mixin private PkiBlockCreationOptions pkiBlockCreationOptions;
@@ -2160,7 +2165,8 @@ public BesuControllerBuilder getControllerBuilder() {
         .lowerBoundPeers(peersLowerBound)
         .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers)
         .randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority)
-        .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject());
+        .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
+        .cacheLastBlocks(numberOfblocksToCache);
   }
 
   @NotNull

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -188,6 +188,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
 
   private PluginTransactionValidatorFactory pluginTransactionValidatorFactory;
 
+  private int numberOfBlocksToCache = 0;
+
   /**
    * Provide a BesuComponent which can be used to get other dependencies
    *
@@ -505,6 +507,17 @@ public BesuControllerBuilder chainPruningConfiguration(
     return this;
   }
 
+  /**
+   * Chain pruning configuration besu controller builder.
+   *
+   * @param numberOfBlocksToCache the number of blocks to cache
+   * @return the besu controller builder
+   */
+  public BesuControllerBuilder cacheLastBlocks(final Integer numberOfBlocksToCache) {
+    this.numberOfBlocksToCache = numberOfBlocksToCache;
+    return this;
+  }
+
   /**
    * sets the networkConfiguration in the builder
    *
@@ -592,7 +605,8 @@ public BesuController build() {
             blockchainStorage,
             metricsSystem,
             reorgLoggingThreshold,
-            dataDirectory.toString());
+            dataDirectory.toString(),
+            numberOfBlocksToCache);
 
     final CachedMerkleTrieLoader cachedMerkleTrieLoader =
         besuComponent

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -5213,4 +5213,16 @@ public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
         .contains(
             "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
   }
+
+  @Test
+  public void cacheLastBlocksOptionShouldWork() {
+    int numberOfBlocksToCache = 512;
+    parseCommand("--cache-last-blocks", String.valueOf(numberOfBlocksToCache));
+    verify(mockControllerBuilder).cacheLastBlocks(intArgumentCaptor.capture());
+    verify(mockControllerBuilder).build();
+
+    assertThat(intArgumentCaptor.getValue()).isEqualTo(numberOfBlocksToCache);
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+  }
 }

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -261,6 +261,8 @@ public void initMocks() throws Exception {
         .thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.besuComponent(any(BesuComponent.class)))
         .thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.cacheLastBlocks(any())).thenReturn(mockControllerBuilder);
+
     // doReturn used because of generic BesuController
     doReturn(mockController).when(mockControllerBuilder).build();
     lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);

besu/src/test/resources/everything_config.toml

@@ -88,6 +88,7 @@ rpc-http-max-batch-size=1
 rpc-http-max-request-content-length = 5242880
 rpc-max-logs-range=100
 json-pretty-print-enabled=false
+cache-last-blocks=512
 
 # PRIVACY TLS
 privacy-tls-enabled=false
@@ -226,4 +227,4 @@ Xp2p-tls-crl-file="none.file"
 Xp2p-tls-clienthello-sni=false
 
 #contracts
-Xevm-jumpdest-cache-weight-kb=32000
+Xevm-jumpdest-cache-weight-kb=32000

ethereum/api/build.gradle

@@ -78,6 +78,7 @@ dependencies {
   implementation 'io.tmio:tuweni-bytes'
   implementation 'io.tmio:tuweni-units'
   implementation 'org.web3j:abi'
+  implementation 'com.github.ben-manes.caffeine:caffeine'
 
   annotationProcessor "org.immutables:value"
   implementation "org.immutables:value-annotations"

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthFeeHistory.java

@@ -16,6 +16,7 @@
 
 import static java.util.stream.Collectors.toUnmodifiableList;
 
+import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod;
 import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext;
@@ -36,25 +37,30 @@
 import org.hyperledger.besu.ethereum.mainnet.feemarket.BaseFeeMarket;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
 
-import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.stream.LongStream;
 import java.util.stream.Stream;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.collect.Streams;
 
 public class EthFeeHistory implements JsonRpcMethod {
   private final ProtocolSchedule protocolSchedule;
   private final Blockchain blockchain;
+  private final Cache<RewardCacheKey, List<Wei>> cache;
+  private static final int MAXIMUM_CACHE_SIZE = 100_000;
+
+  record RewardCacheKey(Hash blockHash, List<Double> rewardPercentiles) {}
 
   public EthFeeHistory(final ProtocolSchedule protocolSchedule, final Blockchain blockchain) {
     this.protocolSchedule = protocolSchedule;
     this.blockchain = blockchain;
+    this.cache = Caffeine.newBuilder().maximumSize(MAXIMUM_CACHE_SIZE).build();
   }
 
   @Override
@@ -96,6 +102,7 @@ public JsonRpcResponse response(final JsonRpcRequestContext request) {
 
     final List<BlockHeader> blockHeaders =
         LongStream.range(oldestBlock, lastBlock)
+            .parallel()
             .mapToObj(blockchain::getBlockHeader)
             .flatMap(Optional::stream)
             .collect(toUnmodifiableList());
@@ -143,14 +150,31 @@ public JsonRpcResponse response(final JsonRpcRequestContext request) {
     final Optional<List<List<Wei>>> maybeRewards =
         maybeRewardPercentiles.map(
             rewardPercentiles ->
-                LongStream.range(oldestBlock, lastBlock)
-                    .mapToObj(blockchain::getBlockByNumber)
-                    .flatMap(Optional::stream)
+                blockHeaders.stream()
+                    .parallel()
                     .map(
-                        block ->
-                            computeRewards(
-                                rewardPercentiles.stream().sorted().collect(toUnmodifiableList()),
-                                block))
+                        blockHeader -> {
+                          final RewardCacheKey key =
+                              new RewardCacheKey(blockHeader.getBlockHash(), rewardPercentiles);
+                          return Optional.ofNullable(cache.getIfPresent(key))
+                              .or(
+                                  () -> {
+                                    Optional<Block> block =
+                                        blockchain.getBlockByHash(blockHeader.getBlockHash());
+                                    return block.map(
+                                        b -> {
+                                          List<Wei> rewards =
+                                              computeRewards(
+                                                  rewardPercentiles.stream()
+                                                      .sorted()
+                                                      .collect(toUnmodifiableList()),
+                                                  b);
+                                          cache.put(key, rewards);
+                                          return rewards;
+                                        });
+                                  });
+                        })
+                    .flatMap(Optional::stream)
                     .collect(toUnmodifiableList()));
 
     return new JsonRpcSuccessResponse(
@@ -188,13 +212,21 @@ private List<Wei> computeRewards(final List<Double> rewardPercentiles, final Blo
               : transactionReceipt.getCumulativeGasUsed()
                   - transactionsGasUsed.get(transactionsGasUsed.size() - 1));
     }
-    final List<Map.Entry<Transaction, Long>> transactionsAndGasUsedAscendingEffectiveGasFee =
+
+    record TransactionInfo(Transaction transaction, Long gasUsed, Wei effectivePriorityFeePerGas) {}
+
+    final List<TransactionInfo> transactionsInfo =
         Streams.zip(
-                transactions.stream(), transactionsGasUsed.stream(), AbstractMap.SimpleEntry::new)
-            .sorted(
-                Comparator.comparing(
-                    transactionAndGasUsed ->
-                        transactionAndGasUsed.getKey().getEffectivePriorityFeePerGas(baseFee)))
+                transactions.stream(),
+                transactionsGasUsed.stream(),
+                (transaction, gasUsed) ->
+                    new TransactionInfo(
+                        transaction, gasUsed, transaction.getEffectivePriorityFeePerGas(baseFee)))
+            .collect(toUnmodifiableList());
+
+    final List<TransactionInfo> transactionsAndGasUsedAscendingEffectiveGasFee =
+        transactionsInfo.stream()
+            .sorted(Comparator.comparing(TransactionInfo::effectivePriorityFeePerGas))
             .collect(toUnmodifiableList());
 
     // We need to weight the percentile of rewards by the gas used in the transaction.
@@ -203,18 +235,21 @@ private List<Wei> computeRewards(final List<Double> rewardPercentiles, final Blo
     final ArrayList<Wei> rewards = new ArrayList<>();
     int rewardPercentileIndex = 0;
     long gasUsed = 0;
-    for (final Map.Entry<Transaction, Long> transactionAndGasUsed :
+    for (final TransactionInfo transactionAndGasUsed :
         transactionsAndGasUsedAscendingEffectiveGasFee) {
 
-      gasUsed += transactionAndGasUsed.getValue();
+      gasUsed += transactionAndGasUsed.gasUsed();
 
       while (rewardPercentileIndex < rewardPercentiles.size()
           && 100.0 * gasUsed / block.getHeader().getGasUsed()
               >= rewardPercentiles.get(rewardPercentileIndex)) {
-        rewards.add(transactionAndGasUsed.getKey().getEffectivePriorityFeePerGas(baseFee));
+        rewards.add(transactionAndGasUsed.effectivePriorityFeePerGas);
         rewardPercentileIndex++;
       }
     }
+    // Put the computed rewards in the cache
+    cache.put(new RewardCacheKey(block.getHeader().getBlockHash(), rewardPercentiles), rewards);
+
     return rewards;
   }
 }