Merge branch 'main' of https://github.com/Citi/besu into hyperledger#…

…3810-qbft-no-empty-block
Citi · Apr 16, 2024 · 3f51dd2 · 3f51dd2
2 parents 98e3e3a + 941ab01
commit 3f51dd2
Show file tree

Hide file tree

Showing 15 changed files with 283 additions and 70 deletions.
diff --git a/.github/workflows/acceptance-tests.yml b/.github/workflows/acceptance-tests.yml
@@ -35,6 +35,8 @@ jobs:
         with:
           distribution: temurin
           java-version: 17
+      - name: Install required packages
+        run:  sudo apt-get install -y xmlstarlet
       - name: get acceptance test report
         uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
         with:
@@ -48,19 +50,16 @@ jobs:
         with:
           cache-disabled: true
       - name: Split tests
-        id: split-tests
-        uses: r7kamura/split-tests-by-timings@9322bd292d9423e2bc5a65bec548901801341e3f
+        run: .github/workflows/splitTestsByTime.sh tmp/junit-xml-reports-downloaded ${{env.total-runners}} ${{ matrix.runner_index }} > testList.txt
+      - name: Upload Timing
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3
+        if: matrix.runner_index == 0
         with:
-          reports: tmp/junit-xml-reports-downloaded
-          glob: 'acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/**/*Test.java'
-          total: ${{env.total-runners}}
-          index: ${{ matrix.runner_index }}
-      - name: write out test list
-        run: echo "${{ steps.split-tests.outputs.paths }}" >> testList.txt
+          name: acceptance-tests-timing
+          path: 'tmp/timing.tsv'
       - name: format gradle args
-        #regex means: first truncate file paths to align with package name, then swap path delimiter with package delimiter,
-        #then drop file extension, then insert --tests option between each.
-        run: cat testList.txt | sed -e 's@acceptance-tests/tests/src/test/java/@--tests\ @g;s@/@.@g;s/\.java//g'  > gradleArgs.txt
+        # insert --tests option between each.
+        run:  cat testList.txt | sed -e 's/^\| / --tests /g' | tee gradleArgs.txt
       - name: run acceptance tests
         run: ./gradlew acceptanceTestNotPrivacy `cat gradleArgs.txt` -Dorg.gradle.parallel=true -Dorg.gradle.caching=true
       - name: cleanup tempfiles

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -0,0 +1,121 @@
+name: docker develop
+
+on:
+  push:
+    branches:
+      - main
+env:
+  registry: docker.io
+
+jobs:
+  hadolint:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+
+      - name: hadoLint
+        run: docker run --rm -i hadolint/hadolint < docker/Dockerfile
+  buildDocker:
+    needs: hadolint
+    permissions:
+      contents: read
+      packages: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - ubuntu-22.04
+          - [self-hosted, ARM64]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Prepare
+        id: prep
+        run: |
+          platform=${{ matrix.platform }}
+          if [ "$platform" = 'ubuntu-22.04' ]; then 
+            echo "PLATFORM_PAIR=linux-amd64" >> $GITHUB_OUTPUT
+            echo "ARCH=amd64" >> $GITHUB_OUTPUT
+          else
+            echo "PLATFORM_PAIR=linux-arm64" >> $GITHUB_OUTPUT
+            echo "ARCH=arm64" >> $GITHUB_OUTPUT
+          fi
+
+          # Get the current date and time in the format YY.MM
+          DATE_TIME=$(date +"%y.%-m")
+          # Get the short SHA of the merge commit
+          SHORT_SHA=${GITHUB_SHA::7}
+          # Construct the build target name
+          BUILD_TARGET_NAME="${DATE_TIME}-develop-${SHORT_SHA}"
+          echo "Build Target Name: $BUILD_TARGET_NAME"
+          # Set the build target name as an environment variable
+          echo "BUILD_TARGET_NAME=${BUILD_TARGET_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: install goss
+        run: |
+          mkdir -p docker/reports
+          curl -L https://github.com/aelsabbahy/goss/releases/download/v0.4.4/goss-${{ steps.prep.outputs.PLATFORM_PAIR }} -o ./docker/tests/goss-${{ steps.prep.outputs.PLATFORM_PAIR }}
+      - name: login to ${{ env.registry }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ secrets.DOCKER_USER_RW }}
+          password: ${{ secrets.DOCKER_PASSWORD_RW }}
+      - name: build and test docker
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        with:
+          cache-disabled: true
+          arguments: testDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME}} -Prelease.releaseVersion=develop
+      - name: publish
+        env:
+          architecture: ${{ steps.prep.outputs.ARCH }}
+        run: ./gradlew --no-daemon dockerUpload -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME }} -Prelease.releaseVersion=develop
+  multiArch:
+    needs: buildDocker
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+      - name: Set up Java
+        uses: actions/setup-java@387ac29b308b003ca37ba93a6cab5eb57c8f5f93
+        with:
+          distribution: temurin
+          java-version: 17
+      - name: setup gradle
+        uses: gradle/actions/setup-gradle@9e899d11ad247ec76be7a60bc1cf9d3abbb9e7f1
+        with:
+          cache-disabled: true
+      - name: login to ${{ env.registry }}
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        with:
+          registry: ${{ env.registry }}
+          username: ${{ secrets.DOCKER_USER_RW }}
+          password: ${{ secrets.DOCKER_PASSWORD_RW }}
+      - name: multi-arch docker
+        run: ./gradlew manifestDocker -PdockerOrgName=${{ env.registry }}/${{ secrets.DOCKER_ORG }} -Pversion=${{ env.BUILD_TARGET_NAME }} -Prelease.releaseVersion=develop
diff --git a/.github/workflows/splitTestsByTime.sh b/.github/workflows/splitTestsByTime.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+REPORTS_DIR="$1"
+SPLIT_COUNT=$2
+SPLIT_INDEX=$3
+
+# extract tests time from Junit XML reports
+find "$REPORTS_DIR" -type f -name TEST-*.xml | xargs -I{} bash -c "xmlstarlet sel -t -v 'sum(//testcase/@time)' '{}'; echo '{}' | sed 's/.*TEST\-\(.*\)\.xml/ \1/'" > tmp/timing.tsv
+
+# Sort times in descending order
+IFS=$'\n' sorted=($(sort -nr tmp/timing.tsv))
+unset IFS
+
+sums=()
+tests=()
+
+# Initialize sums
+for ((i=0; i<SPLIT_COUNT; i++))
+do
+	sums[$i]=0
+done
+
+# add tests to groups trying to balance the sum of execution time of each group
+for line in "${sorted[@]}"; do
+	line_parts=( $line )
+	test_time=${line_parts[0]//./} # convert to millis
+	test_time=${test_time##0} # remove leading zeros
+	test_name=${line_parts[1]}
+
+	# Find index of min sum
+	idx_min_sum=0
+	min_sum=${sums[0]}
+	for ((i=0; i<SPLIT_COUNT; i++))
+	do		
+		if [[ ${sums[$i]} -lt $min_sum ]]
+		then
+			idx_min_sum=$i		
+			min_sum=${sums[$i]}
+		fi
+	done
+
+	# Add the test to the min sum list
+	min_sum_tests=${tests[$idx_min_sum]}
+	tests[$idx_min_sum]="${min_sum_tests}${test_name},"
+
+	# Update the sums
+	((sums[idx_min_sum]+=test_time))
+
+done
+
+# return the requests index, without quotes to drop the last trailing space
+echo ${tests[$SPLIT_INDEX]//,/ }
diff --git a/...tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java b/...tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java
@@ -56,12 +56,15 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
-import org.junit.jupiter.api.AfterEach;
+import org.junit.After;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/** Superclass for acceptance tests. */
+/**
+ * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports
+ * junit4 format.
+ */
 @ExtendWith(AcceptanceTestBaseTestWatcher.class)
 public class AcceptanceTestBase {
 
@@ -128,7 +131,7 @@ protected AcceptanceTestBase() {
     exitedSuccessfully = new ExitedWithCode(0);
   }
 
-  @AfterEach
+  @After
   public void tearDownAcceptanceTestBase() {
     reportMemory();
     cluster.close();

diff --git a/...ance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java b/...ance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java
@@ -789,21 +789,6 @@ public void stop() {
       nodeRequests.shutdown();
       nodeRequests = null;
     }
-
-    deleteRuntimeFiles();
-  }
-
-  private void deleteRuntimeFiles() {
-    try {
-      Files.deleteIfExists(homeDirectory.resolve("besu.networks"));
-    } catch (IOException e) {
-      LOG.error("Failed to clean up besu.networks file in {}", homeDirectory, e);
-    }
-    try {
-      Files.deleteIfExists(homeDirectory.resolve("besu.ports"));
-    } catch (IOException e) {
-      LOG.error("Failed to clean up besu.ports file in {}", homeDirectory, e);
-    }
   }
 
   @Override

diff --git a/...l/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/...l/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java
@@ -52,7 +52,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import org.apache.commons.lang3.SystemUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
@@ -78,15 +77,8 @@ public void startNode(final BesuNode node) {
 
     final Path dataDir = node.homeDirectory();
 
-    final var workingDir =
-        new File(System.getProperty("user.dir")).getParentFile().getParentFile().toPath();
-
     final List<String> params = new ArrayList<>();
-    if (SystemUtils.IS_OS_WINDOWS) {
-      params.add(workingDir.resolve("build\\install\\besu\\bin\\besu.bat").toString());
-    } else {
-      params.add("build/install/besu/bin/besu");
-    }
+    params.add("build/install/besu/bin/besu");
 
     params.add("--data-path");
     params.add(dataDir.toAbsolutePath().toString());
@@ -430,13 +422,15 @@ public void startNode(final BesuNode node) {
     LOG.info("Creating besu process with params {}", params);
     final ProcessBuilder processBuilder =
         new ProcessBuilder(params)
-            .directory(workingDir.toFile())
+            .directory(new File(System.getProperty("user.dir")).getParentFile().getParentFile())
             .redirectErrorStream(true)
             .redirectInput(Redirect.INHERIT);
     if (!node.getPlugins().isEmpty()) {
       processBuilder
           .environment()
-          .put("BESU_OPTS", "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath());
+          .put(
+              "BESU_OPTS",
+              "-Dbesu.plugins.dir=" + dataDir.resolve("plugins").toAbsolutePath().toString());
     }
     // Use non-blocking randomness for acceptance tests
     processBuilder
@@ -578,7 +572,7 @@ private void killBesuProcess(final String name) {
 
     LOG.info("Killing {} process, pid {}", name, process.pid());
 
-    process.descendants().forEach(ProcessHandle::destroy);
+    process.destroy();
     try {
       process.waitFor(30, TimeUnit.SECONDS);
     } catch (final InterruptedException e) {

diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
@@ -2679,7 +2679,8 @@ private SyncMode getDefaultSyncModeIfNotSet() {
   private Boolean getDefaultVersionCompatibilityProtectionIfNotSet() {
     // Version compatibility protection is enabled by default for non-named networks
     return Optional.ofNullable(versionCompatibilityProtection)
-        .orElse(commandLine.getParseResult().hasMatchedOption("network") ? false : true);
+        // if we have a specific genesis file or custom network id, we are not using a named network
+        .orElse(genesisFile != null || networkId != null);
   }
 
   private String generateConfigurationOverview() {

diff --git a/build.gradle b/build.gradle
@@ -979,6 +979,7 @@ def getGitCommitDetails(length = 8) {
 def isInterimBuild(dockerBuildVersion) {
   return (dockerBuildVersion ==~ /.*-SNAPSHOT/) || (dockerBuildVersion ==~ /.*-alpha/)
   || (dockerBuildVersion ==~ /.*-beta/) || (dockerBuildVersion ==~ /.*-RC.*/)
+  || (dockerBuildVersion ==~ /.*develop.*/)
 }
 
 tasks.register("verifyDistributions") {

diff --git a/...m/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockMiner.java b/...m/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockMiner.java
@@ -170,16 +170,15 @@ protected boolean mineBlock() throws InterruptedException {
   private void logProducedBlock(final Block block, final BlockCreationTiming blockCreationTiming) {
     LOG.info(
         String.format(
-            "Produced #%,d / %d tx / %d om / %,d (%01.1f%%) gas / (%s) in %01.3fs",
+            "Produced #%,d / %d tx / %d om / %,d (%01.1f%%) gas / (%s) in %01.3fs / Timing(%s)",
             block.getHeader().getNumber(),
             block.getBody().getTransactions().size(),
             block.getBody().getOmmers().size(),
             block.getHeader().getGasUsed(),
             (block.getHeader().getGasUsed() * 100.0) / block.getHeader().getGasLimit(),
             block.getHash(),
-            blockCreationTiming.end("log").toMillis() / 1000.0));
-
-    LOG.debug("Timing #{} / {}", block.getHeader().getNumber(), blockCreationTiming);
+            blockCreationTiming.end("log").toMillis() / 1000.0,
+            blockCreationTiming));
   }
 
   public void cancel() {