Merge branch 'mining-options-refactor' into time-limited-block-creation

# Conflicts:
#	consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeCoordinator.java
#	consensus/merge/src/test/java/org/hyperledger/besu/consensus/merge/blockcreation/MergeReorgTest.java
#	ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/PoWMinerExecutor.java
#	plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionSelectionResult.java

CHANGELOG.md

@@ -1,10 +1,16 @@
 # Changelog
 
-## Next Release
+## Next release
+- Cache last n blocks by using a new Besu flag --cache-last-blocks=n
+- Optimize performances of RPC method Eth_feeHistory
 
 ### Breaking Changes
 
+### Deprecations
+- `--tx-pool-disable-locals` has been deprecated for removal in favor of `--tx-pool-no-local-priority`, no semantic change, only a renaming [#5959](https://github.com/hyperledger/besu/pull/5959)
+
 ### Additions and Improvements
+- New option `--tx-pool-priority-senders` to specify a list of senders, that has the effect to prioritize any transactions sent by these senders from any source [#5959](https://github.com/hyperledger/besu/pull/5959)
 
 ### Bug Fixes
 
@@ -35,16 +41,16 @@ By default, the txpool is tuned for mainnet usage, but if you are using private
 
 ### Breaking Changes
 - Removed support for Kotti network (ETC) [#5816](https://github.com/hyperledger/besu/pull/5816)
-- Layered transaction pool implementation is now stable and enabled by default, so the following changes to experimental options have been done [#5772](https://github.com/hyperledger/besu):
+- Layered transaction pool implementation is now stable and enabled by default, so the following changes to experimental options have been done [#5772](https://github.com/hyperledger/besu/pull/5772):
     - `--Xlayered-tx-pool` is gone, to select the implementation use the new `--tx-pool` option with values `layered` (default) or `legacy`
-    - `--Xlayered-tx-pool-layer-max-capacity`, `--Xlayered-tx-pool-max-prioritized` and `--Xlayered-tx-pool-max-future-by-sender` just drop the `X` and keep the same behavior
+    - `--Xlayered-tx-pool-layer-max-capacity`, `--Xlayered-tx-pool-max-prioritized` and `--Xlayered-tx-pool-max-future-by-sender` just drop the `Xlayered-` and keep the same behavior
 
 ### Additions and Improvements
 - Add access to an immutable world view to start/end transaction hooks in the tracing API[#5836](https://github.com/hyperledger/besu/pull/5836)
 - Layered transaction pool implementation is now stable and enabled by default. If you want still to use the legacy implementation, use `--tx-pool=legacy`. 
-  By default, the new transaction pool is capped at using 25MB of memory, this limit can be raised using `--layered-tx-pool-layer-max-capacity` options  [#5772](https://github.com/hyperledger/besu)
+  By default, the new transaction pool is capped at using 25MB of memory, this limit can be raised using `--layered-tx-pool-layer-max-capacity` options  [#5772](https://github.com/hyperledger/besu/pull/5772)
 - Tune G1GC to reduce Besu memory footprint, and new `besu-untuned` start scripts to run without any specific G1GC flags [#5879](https://github.com/hyperledger/besu/pull/5879)
-- Reduce `engine_forkchoiceUpdatedV?` response time by asynchronously process block added events in the transaction pool [#5909](https://github.com/hyperledger/besu/pull/5909) 
+- Reduce `engine_forkchoiceUpdatedV?` response time by asynchronously process block added events in the transaction pool [#5909](https://github.com/hyperledger/besu/pull/5909)
 
 ### Bug Fixes
 - do not create ignorable storage on revert storage-variables subcommand [#5830](https://github.com/hyperledger/besu/pull/5830) 
@@ -53,7 +59,6 @@ By default, the txpool is tuned for mainnet usage, but if you are using private
 
 ### Download Links
 https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.0/besu-23.10.0.tar.gz / sha256: 3c75f3792bfdb0892705b378f0b8bfc14ef6cecf1d8afe711d8d8687ed6687cf
-
 https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.0/besu-23.10.0.zip / sha256: d5dafff4c3cbf104bf75b34a9f108dcdd7b08d2759de75ec65cd997f38f52866
 
 ## 23.7.3

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java

@@ -30,6 +30,7 @@
 import org.hyperledger.besu.ethereum.api.tls.FileBasedPasswordProvider;
 import org.hyperledger.besu.ethereum.core.AddressHelpers;
 import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters;
+import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.UpdatableInitValues;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration;
@@ -56,7 +57,10 @@ public class BesuNodeConfigurationBuilder {
   private String name;
   private Optional<Path> dataPath = Optional.empty();
   private MiningParameters miningParameters =
-      ImmutableMiningParameters.builder().coinbase(AddressHelpers.ofValue(1)).build();
+      ImmutableMiningParameters.builder()
+          .updatableInitValues(
+              UpdatableInitValues.builder().coinbase(AddressHelpers.ofValue(1)).build())
+          .build();
 
   private JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault();
   private JsonRpcConfiguration engineRpcConfiguration = JsonRpcConfiguration.createEngineDefault();

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -310,8 +310,8 @@ public BesuNode createNodeWithMultiTenantedPrivacy(
                 UpdatableInitValues.builder()
                     .isMiningEnabled(true)
                     .minTransactionGasPrice(Wei.ZERO)
+                    .coinbase(AddressHelpers.ofValue(1))
                     .build())
-            .coinbase(AddressHelpers.ofValue(1))
             .build();
 
     return create(

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftMiningAcceptanceTest.java

@@ -66,8 +66,8 @@ public void shouldMineOnSingleNodeWithFreeGas_Berlin() throws Exception {
                 UpdatableInitValues.builder()
                     .isMiningEnabled(true)
                     .minTransactionGasPrice(Wei.ZERO)
+                    .coinbase(AddressHelpers.ofValue(1))
                     .build())
-            .coinbase(AddressHelpers.ofValue(1))
             .build();
     minerNode.setMiningParameters(zeroGasMiningParams);
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -53,10 +53,10 @@
 import org.hyperledger.besu.cli.custom.RpcAuthFileValidator;
 import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler;
 import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler;
+import org.hyperledger.besu.cli.options.MiningOptions;
 import org.hyperledger.besu.cli.options.stable.DataStorageOptions;
 import org.hyperledger.besu.cli.options.stable.EthstatsOptions;
 import org.hyperledger.besu.cli.options.stable.LoggingLevelOption;
-import org.hyperledger.besu.cli.options.stable.MiningOptions;
 import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption;
 import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions;
 import org.hyperledger.besu.cli.options.stable.TransactionPoolOptions;
@@ -1233,6 +1233,11 @@ static class PermissionsOptionGroup {
           "Specifies the maximum number of blocks to retrieve logs from via RPC. Must be >=0. 0 specifies no limit  (default: ${DEFAULT-VALUE})")
   private final Long rpcMaxLogsRange = 5000L;
 
+  @CommandLine.Option(
+      names = {"--cache-last-blocks"},
+      description = "Specifies the number of last blocks to cache  (default: ${DEFAULT-VALUE})")
+  private final Integer numberOfblocksToCache = 0;
+
   @Mixin private P2PTLSConfigOptions p2pTLSConfigOptions;
 
   @Mixin private PkiBlockCreationOptions pkiBlockCreationOptions;
@@ -2160,7 +2165,8 @@ public BesuControllerBuilder getControllerBuilder() {
         .lowerBoundPeers(peersLowerBound)
         .maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers)
         .randomPeerPriority(p2PDiscoveryOptionGroup.randomPeerPriority)
-        .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject());
+        .chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
+        .cacheLastBlocks(numberOfblocksToCache);
   }
 
   @NotNull

besu/src/main/java/org/hyperledger/besu/cli/options/stable/MiningOptions.java → besu/src/main/java/org/hyperledger/besu/cli/options/MiningOptions.java

@@ -12,7 +12,7 @@
  *
  * SPDX-License-Identifier: Apache-2.0
  */
-package org.hyperledger.besu.cli.options.stable;
+package org.hyperledger.besu.cli.options;
 
 import static java.util.Arrays.asList;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.Unstable.DEFAULT_MAX_OMMERS_DEPTH;
@@ -26,11 +26,11 @@
 import static org.hyperledger.besu.ethereum.core.MiningParameters.UpdatableInitValues.DEFAULT_MIN_BLOCK_OCCUPANCY_RATIO;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.UpdatableInitValues.DEFAULT_MIN_TRANSACTION_GAS_PRICE;
 
-import org.hyperledger.besu.cli.options.CLIOptions;
 import org.hyperledger.besu.cli.util.CommandLineUtils;
 import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters;
+import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.UpdatableInitValues;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 
 import java.util.List;
@@ -282,7 +282,7 @@ static MiningOptions fromConfig(final MiningParameters miningParameters) {
   @Override
   public MiningParameters toDomainObject() {
     final var updatableInitValuesBuilder =
-        ImmutableMiningParameters.UpdatableInitValues.builder()
+        UpdatableInitValues.builder()
             .isMiningEnabled(isMiningEnabled)
             .extraData(extraData)
             .minTransactionGasPrice(minTransactionGasPrice)
@@ -291,6 +291,9 @@ public MiningParameters toDomainObject() {
     if (targetGasLimit != null) {
       updatableInitValuesBuilder.targetGasLimit(targetGasLimit);
     }
+    if (coinbase != null) {
+      updatableInitValuesBuilder.coinbase(coinbase);
+    }
 
     final var miningParametersBuilder =
         ImmutableMiningParameters.builder()
@@ -311,10 +314,6 @@ public MiningParameters toDomainObject() {
                     .txSelectionMaxTime(unstableOptions.txSelectionMaxTime)
                     .build());
 
-    if (coinbase != null) {
-      miningParametersBuilder.coinbase(coinbase);
-    }
-
     return miningParametersBuilder.build();
   }
 

besu/src/main/java/org/hyperledger/besu/cli/options/stable/TransactionPoolOptions.java

@@ -24,6 +24,7 @@
 import org.hyperledger.besu.cli.converter.PercentageConverter;
 import org.hyperledger.besu.cli.options.CLIOptions;
 import org.hyperledger.besu.cli.util.CommandLineUtils;
+import org.hyperledger.besu.datatypes.Address;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
@@ -32,19 +33,25 @@
 
 import java.io.File;
 import java.util.List;
+import java.util.Set;
 
 import picocli.CommandLine;
 
 /** The Transaction pool Cli stable options. */
 public class TransactionPoolOptions implements CLIOptions<TransactionPoolConfiguration> {
   private static final String TX_POOL_IMPLEMENTATION = "--tx-pool";
+  /** Use TX_POOL_NO_LOCAL_PRIORITY instead */
+  @Deprecated(forRemoval = true)
   private static final String TX_POOL_DISABLE_LOCALS = "--tx-pool-disable-locals";
+
+  private static final String TX_POOL_NO_LOCAL_PRIORITY = "--tx-pool-no-local-priority";
   private static final String TX_POOL_ENABLE_SAVE_RESTORE = "--tx-pool-enable-save-restore";
   private static final String TX_POOL_SAVE_FILE = "--tx-pool-save-file";
   private static final String TX_POOL_PRICE_BUMP = "--tx-pool-price-bump";
   private static final String RPC_TX_FEECAP = "--rpc-tx-feecap";
   private static final String STRICT_TX_REPLAY_PROTECTION_ENABLED_FLAG =
       "--strict-tx-replay-protection-enabled";
+  private static final String TX_POOL_PRIORITY_SENDERS = "--tx-pool-priority-senders";
 
   @CommandLine.Option(
       names = {TX_POOL_IMPLEMENTATION},
@@ -54,13 +61,13 @@ public class TransactionPoolOptions implements CLIOptions<TransactionPoolConfigu
   private TransactionPoolConfiguration.Implementation txPoolImplementation = LAYERED;
 
   @CommandLine.Option(
-      names = {TX_POOL_DISABLE_LOCALS},
+      names = {TX_POOL_NO_LOCAL_PRIORITY, TX_POOL_DISABLE_LOCALS},
       paramLabel = "<Boolean>",
       description =
-          "Set to true if transactions sent via RPC should have the same checks and not be prioritized over remote ones (default: ${DEFAULT-VALUE})",
+          "Set to true if senders of transactions sent via RPC should not have priority (default: ${DEFAULT-VALUE})",
       fallbackValue = "true",
       arity = "0..1")
-  private Boolean disableLocalTxs = TransactionPoolConfiguration.DEFAULT_DISABLE_LOCAL_TXS;
+  private Boolean noLocalPriority = TransactionPoolConfiguration.DEFAULT_NO_LOCAL_PRIORITY;
 
   @CommandLine.Option(
       names = {TX_POOL_ENABLE_SAVE_RESTORE},
@@ -104,6 +111,15 @@ public class TransactionPoolOptions implements CLIOptions<TransactionPoolConfigu
       arity = "0..1")
   private Boolean strictTxReplayProtectionEnabled = false;
 
+  @CommandLine.Option(
+      names = {TX_POOL_PRIORITY_SENDERS},
+      split = ",",
+      paramLabel = "Comma separated list of addresses",
+      description =
+          "Pending transactions sent exclusively by these addresses, from any source, are prioritized and only evicted after all others. If not specified, then only the senders submitting transactions via RPC have priority (default: ${DEFAULT-VALUE})",
+      arity = "1..*")
+  private Set<Address> prioritySenders = TransactionPoolConfiguration.DEFAULT_PRIORITY_SENDERS;
+
   @CommandLine.ArgGroup(
       validate = false,
       heading = "@|bold Tx Pool Layered Implementation Options|@%n")
@@ -200,11 +216,12 @@ public static TransactionPoolOptions fromConfig(final TransactionPoolConfigurati
     final TransactionPoolOptions options = TransactionPoolOptions.create();
     options.txPoolImplementation = config.getTxPoolImplementation();
     options.saveRestoreEnabled = config.getEnableSaveRestore();
-    options.disableLocalTxs = config.getDisableLocalTransactions();
+    options.noLocalPriority = config.getNoLocalPriority();
     options.priceBump = config.getPriceBump();
     options.txFeeCap = config.getTxFeeCap();
     options.saveFile = config.getSaveFile();
     options.strictTxReplayProtectionEnabled = config.getStrictTransactionReplayProtectionEnabled();
+    options.prioritySenders = config.getPrioritySenders();
     options.layeredOptions.txPoolLayerMaxCapacity =
         config.getPendingTransactionsLayerMaxCapacityBytes();
     options.layeredOptions.txPoolMaxPrioritized = config.getMaxPrioritizedTransactions();
@@ -242,11 +259,12 @@ public TransactionPoolConfiguration toDomainObject() {
     return ImmutableTransactionPoolConfiguration.builder()
         .txPoolImplementation(txPoolImplementation)
         .enableSaveRestore(saveRestoreEnabled)
-        .disableLocalTransactions(disableLocalTxs)
+        .noLocalPriority(noLocalPriority)
         .priceBump(priceBump)
         .txFeeCap(txFeeCap)
         .saveFile(saveFile)
         .strictTransactionReplayProtectionEnabled(strictTxReplayProtectionEnabled)
+        .prioritySenders(prioritySenders)
         .pendingTransactionsLayerMaxCapacityBytes(layeredOptions.txPoolLayerMaxCapacity)
         .maxPrioritizedTransactions(layeredOptions.txPoolMaxPrioritized)
         .maxFutureBySender(layeredOptions.txPoolMaxFutureBySender)

besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java

@@ -275,8 +275,8 @@ private MiningParameters getMiningParameters() {
                   .nonceGenerator(new IncrementingNonceGenerator(0))
                   .extraData(extraData)
                   .minTransactionGasPrice(minTransactionGasPrice)
+                  .coinbase(coinbase)
                   .build())
-          .coinbase(coinbase)
           .build();
     }
 

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -188,6 +188,8 @@ public abstract class BesuControllerBuilder implements MiningParameterOverrides
 
   private PluginTransactionValidatorFactory pluginTransactionValidatorFactory;
 
+  private int numberOfBlocksToCache = 0;
+
   /**
    * Provide a BesuComponent which can be used to get other dependencies
    *
@@ -505,6 +507,17 @@ public BesuControllerBuilder chainPruningConfiguration(
     return this;
   }
 
+  /**
+   * Chain pruning configuration besu controller builder.
+   *
+   * @param numberOfBlocksToCache the number of blocks to cache
+   * @return the besu controller builder
+   */
+  public BesuControllerBuilder cacheLastBlocks(final Integer numberOfBlocksToCache) {
+    this.numberOfBlocksToCache = numberOfBlocksToCache;
+    return this;
+  }
+
   /**
    * sets the networkConfiguration in the builder
    *
@@ -592,7 +605,8 @@ public BesuController build() {
             blockchainStorage,
             metricsSystem,
             reorgLoggingThreshold,
-            dataDirectory.toString());
+            dataDirectory.toString(),
+            numberOfBlocksToCache);
 
     final CachedMerkleTrieLoader cachedMerkleTrieLoader =
         besuComponent

besu/src/main/java/org/hyperledger/besu/controller/IbftBesuControllerBuilder.java

@@ -145,6 +145,7 @@ protected MiningCoordinator createMiningCoordinator(
 
     final Address localAddress = Util.publicKeyToAddress(nodeKey.getPublicKey());
     final BftProtocolSchedule bftProtocolSchedule = (BftProtocolSchedule) protocolSchedule;
+    miningParameters.setCoinbase(localAddress);
     final BftBlockCreatorFactory<?> blockCreatorFactory =
         new BftBlockCreatorFactory<>(
             transactionPool,

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -71,6 +71,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration;
 import org.hyperledger.besu.ethereum.api.tls.TlsConfiguration;
 import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters;
+import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.UpdatableInitValues;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
@@ -902,7 +903,10 @@ public void envVariableOverridesValueFromConfigFile() {
     verify(mockControllerBuilder)
         .miningParameters(
             ImmutableMiningParameters.builder()
-                .coinbase(Address.fromHexString(expectedCoinbase))
+                .updatableInitValues(
+                    UpdatableInitValues.builder()
+                        .coinbase(Address.fromHexString(expectedCoinbase))
+                        .build())
                 .build());
   }
 
@@ -916,7 +920,10 @@ public void cliOptionOverridesEnvVariableAndConfig() {
     verify(mockControllerBuilder)
         .miningParameters(
             ImmutableMiningParameters.builder()
-                .coinbase(Address.fromHexString(expectedCoinbase))
+                .updatableInitValues(
+                    UpdatableInitValues.builder()
+                        .coinbase(Address.fromHexString(expectedCoinbase))
+                        .build())
                 .build());
   }
 
@@ -5213,4 +5220,16 @@ public void snapsyncForHealingFeaturesShouldFailWhenHealingIsNotEnabled() {
         .contains(
             "--Xsnapsync-synchronizer-flat option can only be used when -Xsnapsync-synchronizer-flat-db-healing-enabled is true");
   }
+
+  @Test
+  public void cacheLastBlocksOptionShouldWork() {
+    int numberOfBlocksToCache = 512;
+    parseCommand("--cache-last-blocks", String.valueOf(numberOfBlocksToCache));
+    verify(mockControllerBuilder).cacheLastBlocks(intArgumentCaptor.capture());
+    verify(mockControllerBuilder).build();
+
+    assertThat(intArgumentCaptor.getValue()).isEqualTo(numberOfBlocksToCache);
+    assertThat(commandOutput.toString(UTF_8)).isEmpty();
+    assertThat(commandErrorOutput.toString(UTF_8)).isEmpty();
+  }
 }

besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java

@@ -31,8 +31,8 @@
 import org.hyperledger.besu.chainimport.JsonBlockImporter;
 import org.hyperledger.besu.chainimport.RlpBlockImporter;
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
+import org.hyperledger.besu.cli.options.MiningOptions;
 import org.hyperledger.besu.cli.options.stable.EthstatsOptions;
-import org.hyperledger.besu.cli.options.stable.MiningOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
 import org.hyperledger.besu.cli.options.unstable.MetricsCLIOptions;
 import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
@@ -261,6 +261,8 @@ public void initMocks() throws Exception {
         .thenReturn(mockControllerBuilder);
     when(mockControllerBuilder.besuComponent(any(BesuComponent.class)))
         .thenReturn(mockControllerBuilder);
+    when(mockControllerBuilder.cacheLastBlocks(any())).thenReturn(mockControllerBuilder);
+
     // doReturn used because of generic BesuController
     doReturn(mockController).when(mockControllerBuilder).build();
     lenient().when(mockController.getProtocolManager()).thenReturn(mockEthProtocolManager);