Merge remote-tracking branch 'upstream/main' into merge-besu-main

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Breaking Changes
 - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343)
+- The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360)
 
 ### Deprecations
 - Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230)
@@ -15,9 +16,15 @@
 - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297)
 - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275)
 - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328)
+- Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302)
+- Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377)
+- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378)
 
 ### Bug fixes
 - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344)
+- Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360)
+- Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382)
+- Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390)
 
 ## 23.10.3
 

acceptance-tests/dsl/build.gradle

@@ -26,7 +26,6 @@ dependencies {
   implementation project(':testutil')
   implementation project(':util')
 
-  implementation 'com.github.tomakehurst:wiremock-jre8'
   implementation 'com.google.guava:guava'
   implementation 'com.google.dagger:dagger'
   annotationProcessor 'com.google.dagger:dagger-compiler'
@@ -45,6 +44,7 @@ dependencies {
   implementation 'org.web3j:abi'
   implementation 'org.web3j:besu'
   implementation 'org.web3j:crypto'
+  implementation 'org.wiremock:wiremock'
 
   implementation 'org.testcontainers:testcontainers'
   implementation 'org.junit.jupiter:junit-jupiter'

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java

@@ -77,6 +77,10 @@ public Condition miningStatus(final boolean isMining) {
     return new MiningStatusCondition(transactions.mining(), isMining);
   }
 
+  public Condition syncingStatus(final boolean isSyncing) {
+    return new SyncingStatusCondition(transactions.syncing(), isSyncing);
+  }
+
   public Condition expectNewPendingTransactions(
       final BigInteger filterId, final List<String> transactionHashes) {
     return new NewPendingTransactionFilterChangesCondition(

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.condition.eth;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
+import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthSyncingTransaction;
+
+public class SyncingStatusCondition implements Condition {
+
+  private final EthSyncingTransaction transaction;
+  private final boolean syncingMiningStatus;
+
+  public SyncingStatusCondition(
+      final EthSyncingTransaction transaction, final boolean syncingStatus) {
+    this.transaction = transaction;
+    this.syncingMiningStatus = syncingStatus;
+  }
+
+  @Override
+  public void verify(final Node node) {
+    WaitUtils.waitFor(
+        10, () -> assertThat(node.execute(transaction)).isEqualTo(syncingMiningStatus));
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java

@@ -86,11 +86,8 @@ public void start(final List<? extends RunnableNode> nodes) {
     final Optional<? extends RunnableNode> bootnode = selectAndStartBootnode(nodes);
 
     nodes.parallelStream()
-        .filter(
-            node -> {
-              LOG.info("starting non-bootnode {}", node.getName());
-              return bootnode.map(boot -> boot != node).orElse(true);
-            })
+        .filter(node -> bootnode.map(boot -> boot != node).orElse(true))
+        .peek(node -> LOG.info("starting non-bootnode {}", node.getName()))
         .forEach(this::startNode);
 
     if (clusterConfiguration.isAwaitPeerDiscovery()) {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthSyncingTransaction.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.transaction.eth;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.NodeRequests;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction;
+
+import java.io.IOException;
+
+import org.web3j.protocol.core.methods.response.EthSyncing;
+
+public class EthSyncingTransaction implements Transaction<Boolean> {
+
+  EthSyncingTransaction() {}
+
+  @Override
+  public Boolean execute(final NodeRequests node) {
+    try {
+      EthSyncing response = node.eth().ethSyncing().send();
+      assertThat(response).isNotNull();
+      return response.isSyncing();
+    } catch (final IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/transaction/eth/EthTransactions.java

@@ -73,6 +73,10 @@ public EthMiningTransaction mining() {
     return new EthMiningTransaction();
   }
 
+  public EthSyncingTransaction syncing() {
+    return new EthSyncingTransaction();
+  }
+
   public EthNewPendingTransactionFilterTransaction newPendingTransactionsFilter() {
     return new EthNewPendingTransactionFilterTransaction();
   }

acceptance-tests/tests/build.gradle

@@ -55,7 +55,6 @@ dependencies {
   testImplementation project(':testutil')
   testImplementation project(':util')
 
-  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone'
   testImplementation 'commons-io:commons-io'
   testImplementation 'io.grpc:grpc-all'
   testImplementation 'io.grpc:grpc-core'
@@ -84,6 +83,7 @@ dependencies {
   testImplementation 'org.web3j:abi'
   testImplementation 'org.web3j:besu'
   testImplementation 'org.web3j:core'
+  testImplementation 'org.wiremock:wiremock'
 
   testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
 }

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningAcceptanceTest.java

@@ -49,6 +49,11 @@ public void setUp() {
 
     permissionedNode.verify(admin.addPeer(bootnode));
     permissionedNode.verify(admin.addPeer(allowedNode));
+
+    allowedNode.verify(eth.syncingStatus(false));
+    bootnode.verify(eth.syncingStatus(false));
+    permissionedNode.verify(eth.syncingStatus(false));
+    forbiddenNode.verify(eth.syncingStatus(false));
   }
 
   @Test

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/permissioning/NodeSmartContractPermissioningV2AcceptanceTest.java

@@ -46,6 +46,11 @@ public void setUp() {
 
     permissionedNode.execute(allowNode(permissionedNode));
     permissionedNode.verify(connectionIsAllowed(permissionedNode));
+
+    allowedNode.verify(eth.syncingStatus(false));
+    bootnode.verify(eth.syncingStatus(false));
+    permissionedNode.verify(eth.syncingStatus(false));
+    forbiddenNode.verify(eth.syncingStatus(false));
   }
 
   @Test

acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_genesis.json

@@ -4,7 +4,7 @@
     "londonBlock": 0,
     "zeroBaseFee": true,
     "ethash": {
-      "fixeddifficulty": 100
+      "fixeddifficulty": 500
     }
   },
   "nonce": "0x42",

acceptance-tests/tests/src/test/resources/permissioning/simple_permissioning_v2_genesis.json

@@ -4,7 +4,7 @@
     "londonBlock": 0,
     "zeroBaseFee": true,
     "ethash": {
-      "fixeddifficulty": 100
+      "fixeddifficulty": 500
     }
   },
   "nonce": "0x42",

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -23,6 +23,7 @@
 import static org.hyperledger.besu.cli.config.NetworkName.MAINNET;
 import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPENDENCY_WARNING_MSG;
 import static org.hyperledger.besu.cli.util.CommandLineUtils.DEPRECATION_WARNING_MSG;
+import static org.hyperledger.besu.cli.util.CommandLineUtils.isOptionSet;
 import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
 import static org.hyperledger.besu.ethereum.api.graphql.GraphQLConfiguration.DEFAULT_GRAPHQL_HTTP_PORT;
 import static org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration.DEFAULT_ENGINE_JSON_RPC_PORT;
@@ -521,11 +522,11 @@ private InetAddress autoDiscoverDefaultIP() {
   private SyncMode syncMode = null;
 
   @Option(
-      names = {"--fast-sync-min-peers"},
+      names = {"--sync-min-peers", "--fast-sync-min-peers"},
       paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
       description =
-          "Minimum number of peers required before starting fast sync. Has only effect on PoW networks. (default: ${DEFAULT-VALUE})")
-  private final Integer fastSyncMinPeerCount = FAST_SYNC_MIN_PEER_COUNT;
+          "Minimum number of peers required before starting sync. Has effect only on non-PoS networks. (default: ${DEFAULT-VALUE})")
+  private final Integer syncMinPeerCount = SYNC_MIN_PEER_COUNT;
 
   @Option(
       names = {"--network"},
@@ -2028,11 +2029,10 @@ private void issueOptionWarnings() {
             "--p2p-port",
             "--remote-connections-max-percentage"));
 
-    CommandLineUtils.failIfOptionDoesntMeetRequirement(
-        commandLine,
-        "--fast-sync-min-peers can't be used with FULL sync-mode",
-        !SyncMode.isFullSync(getDefaultSyncModeIfNotSet()),
-        singletonList("--fast-sync-min-peers"));
+    if (SyncMode.isFullSync(getDefaultSyncModeIfNotSet())
+        && isOptionSet(commandLine, "--sync-min-peers")) {
+      logger.warn("--sync-min-peers is ignored in FULL sync-mode");
+    }
 
     CommandLineUtils.failIfOptionDoesntMeetRequirement(
         commandLine,
@@ -2867,7 +2867,7 @@ private SynchronizerConfiguration buildSyncConfig() {
     return unstableSynchronizerOptions
         .toDomainObject()
         .syncMode(syncMode)
-        .fastSyncMinimumPeerCount(fastSyncMinPeerCount)
+        .fastSyncMinimumPeerCount(syncMinPeerCount)
         .build();
   }
 

besu/src/main/java/org/hyperledger/besu/cli/DefaultCommandValues.java

@@ -66,8 +66,8 @@ public interface DefaultCommandValues {
   NatMethod DEFAULT_NAT_METHOD = NatMethod.AUTO;
   /** The constant DEFAULT_JWT_ALGORITHM. */
   JwtAlgorithm DEFAULT_JWT_ALGORITHM = JwtAlgorithm.RS256;
-  /** The constant FAST_SYNC_MIN_PEER_COUNT. */
-  int FAST_SYNC_MIN_PEER_COUNT = 5;
+  /** The constant SYNC_MIN_PEER_COUNT. */
+  int SYNC_MIN_PEER_COUNT = 5;
   /** The constant DEFAULT_MAX_PEERS. */
   int DEFAULT_MAX_PEERS = 25;
   /** The constant DEFAULT_P2P_PEER_LOWER_BOUND. */

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/RocksDbUsageHelper.java

@@ -34,17 +34,17 @@ static void printUsageForColumnFamily(
       final RocksDB rocksdb, final ColumnFamilyHandle cfHandle, final PrintWriter out)
       throws RocksDBException, NumberFormatException {
     final String size = rocksdb.getProperty(cfHandle, "rocksdb.estimate-live-data-size");
+    final String numberOfKeys = rocksdb.getProperty(cfHandle, "rocksdb.estimate-num-keys");
     boolean emptyColumnFamily = false;
-    if (!size.isEmpty() && !size.isBlank()) {
+    if (!size.isBlank() && !numberOfKeys.isBlank()) {
       try {
         final long sizeLong = Long.parseLong(size);
+        final long numberOfKeysLong = Long.parseLong(numberOfKeys);
         final String totalSstFilesSize =
             rocksdb.getProperty(cfHandle, "rocksdb.total-sst-files-size");
         final long totalSstFilesSizeLong =
-            !totalSstFilesSize.isEmpty() && !totalSstFilesSize.isBlank()
-                ? Long.parseLong(totalSstFilesSize)
-                : 0;
-        if (sizeLong == 0) {
+            !totalSstFilesSize.isBlank() ? Long.parseLong(totalSstFilesSize) : 0;
+        if (sizeLong == 0 && numberOfKeysLong == 0) {
           emptyColumnFamily = true;
         }
 

besu/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java

@@ -45,7 +45,11 @@
     description = "This command provides storage related actions.",
     mixinStandardHelpOptions = true,
     versionProvider = VersionProvider.class,
-    subcommands = {StorageSubCommand.RevertVariablesStorage.class, RocksDbSubCommand.class})
+    subcommands = {
+      StorageSubCommand.RevertVariablesStorage.class,
+      RocksDbSubCommand.class,
+      TrieLogSubCommand.class
+    })
 public class StorageSubCommand implements Runnable {
 
   /** The constant COMMAND_NAME. */