diff --git a/.github/workflows/code-freeze.yml b/.github/workflows/code-freeze.yml index 6f354c4ea..865bc6dc9 100644 --- a/.github/workflows/code-freeze.yml +++ b/.github/workflows/code-freeze.yml @@ -1,9 +1,21 @@ +# ******************************************************************************** +# Copyright (c) 2023, 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + name: Code Freeze Bot # Controls when the workflow will run on: pull_request_target: - branches: [ "v[0-9]+.[0-9]+.[0-9]+" ] issue_comment: types: [created] @@ -12,6 +24,15 @@ permissions: pull-requests: write jobs: - codefreeze: + # Check if the pull request target branch matches the required branch-regex? + codefreeze_branch_check: + uses: adoptium/.github/.github/workflows/code-freeze-regex-branch.yml@main + with: + branch-regex: "^v[0-9]+.[0-9]+.[0-9]+$" + + # Code freeze if branch-regex matches + codefreeze_if_branch_match: + needs: codefreeze_branch_check uses: adoptium/.github/.github/workflows/code-freeze.yml@main + if: (github.event_name == 'pull_request_target' || (github.event_name == 'issue_comment' && github.event.issue.pull_request)) && needs.codefreeze_branch_check.outputs.regex-matches == 'true' secrets: inherit diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 07f6b286b..e2c54fd33 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -42,7 +42,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 diff --git a/.github/workflows/semgrep_diff.yml b/.github/workflows/semgrep_diff.yml new file mode 100644 index 000000000..4b03d1ccb --- /dev/null +++ b/.github/workflows/semgrep_diff.yml @@ -0,0 +1,20 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** +--- +name: Semgrep Differential Scan +on: + pull_request: + +jobs: + semgrep-diff: + uses: adoptium/.github/.github/workflows/semgrep_diff.yml@main diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7d07852bf..1a1387812 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 - uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # v4.2.1 with: diff --git a/pipelines/build/common/build_base_file.groovy b/pipelines/build/common/build_base_file.groovy index de0469e77..1475e8e28 100644 --- a/pipelines/build/common/build_base_file.groovy +++ b/pipelines/build/common/build_base_file.groovy @@ -996,6 +996,17 @@ class Builder implements Serializable { } } context.parallel jobs + context.node('worker') { + // Archive tap files as a single tar file + context.sh "find . -type f -name '*.tap' -exec tar -czf AQAvitTapFiles.tar.gz {} + " + try { + context.timeout(time: pipelineTimeouts.ARCHIVE_ARTIFACTS_TIMEOUT, unit: 'HOURS') { + context.archiveArtifacts artifacts: "AQAvitTapFiles.tar.gz" + } + } catch (FlowInterruptedException e) { + throw new Exception("[ERROR] Archive AQAvitTapFiles.tar.gz timeout Exiting...") + } + } // publish to github if needed // Don't publish release automatically if (publish || release) { diff --git a/pipelines/build/common/openjdk_build_pipeline.groovy b/pipelines/build/common/openjdk_build_pipeline.groovy index 78f12aabf..495552dde 100644 --- a/pipelines/build/common/openjdk_build_pipeline.groovy +++ b/pipelines/build/common/openjdk_build_pipeline.groovy @@ -363,8 +363,6 @@ class Build { def vendorTestBranches = '' def vendorTestDirs = '' List testList = buildConfig.TEST_LIST - List dynamicList = buildConfig.DYNAMIC_LIST - List numMachines = buildConfig.NUM_MACHINES def enableTestDynamicParallel = Boolean.valueOf(buildConfig.ENABLE_TESTDYNAMICPARALLEL) def aqaBranch = 'master' def useTestEnvProperties = false @@ -374,6 +372,14 @@ class Build { } def aqaAutoGen = buildConfig.AQA_AUTO_GEN ?: false + def parallel = 'None' + def numMachinesPerTest = '' + def testTime = '' + // Enable time based parallel. Set expected completion time to 120 mins + if (enableTestDynamicParallel) { + testTime = '120' + parallel = 'Dynamic' + } testList.each { testType -> // For each requested test, i.e 'sanity.openjdk', 'sanity.system', 'sanity.perf', 'sanity.external', call test job @@ -416,20 +422,6 @@ class Build { } def jobParams = getAQATestJobParams(testType) - def parallel = 'None' - def numMachinesPerTest = '' - - if (enableTestDynamicParallel && dynamicList.contains(testType)) { - numMachinesPerTest = numMachines[(dynamicList.indexOf(testType))] - if (!numMachinesPerTest) { - // see build configuration in jdk*_pipeline_config.groovy - // when numMachines is an array, its size should match the testLists size - throw new Exception("No number of machines provided for running ${testType} tests in parallel, numMachines: ${numMachines}!") - } - context.println "Number of machines for running parallel tests: ${numMachinesPerTest}" - - parallel = 'Dynamic' - } def jobName = jobParams.TEST_JOB_NAME String helperRef = buildConfig.HELPER_REF ?: DEFAULTS_JSON['repository']['helper_ref'] @@ -478,6 +470,7 @@ class Build { context.booleanParam(name: 'KEEP_REPORTDIR', value: keep_test_reportdir), context.string(name: 'PARALLEL', value: parallel), context.string(name: 'NUM_MACHINES', value: "${numMachinesPerTest}"), + context.string(name: 'TEST_TIME', value: testTime), context.booleanParam(name: 'USE_TESTENV_PROPERTIES', value: useTestEnvProperties), context.booleanParam(name: 'GENERATE_JOBS', value: aqaAutoGen), context.string(name: 'ADOPTOPENJDK_BRANCH', value: aqaBranch), @@ -796,8 +789,6 @@ class Build { We run two jobs if we have a JRE (see https://github.com/adoptium/temurin-build/issues/1751). */ private void buildWindowsInstaller(VersionInfo versionData, String filter, String category) { - def nodeFilter = "${buildConfig.TARGET_OS}&&wix" - def buildNumber = versionData.build if (versionData.major == 8) { @@ -813,6 +804,12 @@ class Build { // Get version patch number if one is present def patch_version = versionData.patch ?: 0 + def INSTALLER_JVM = "${buildConfig.VARIANT}" + // if variant is temurin set param as hotpot + if (buildConfig.VARIANT == 'temurin') { + INSTALLER_JVM = 'hotspot' + } + // Execute installer job def installerJob = context.build job: 'build-scripts/release/create_installer_windows', propagate: true, @@ -827,9 +824,8 @@ class Build { context.string(name: 'PRODUCT_BUILD_NUMBER', value: "${buildNumber}"), context.string(name: 'MSI_PRODUCT_VERSION', value: "${versionData.msi_product_version}"), context.string(name: 'PRODUCT_CATEGORY', value: "${category}"), - context.string(name: 'JVM', value: "${buildConfig.VARIANT}"), + context.string(name: 'JVM', value: "${INSTALLER_JVM}"), context.string(name: 'ARCH', value: "${INSTALLER_ARCH}"), - ['$class': 'LabelParameterValue', name: 'NODE_LABEL', label: "${nodeFilter}"] ] context.copyArtifacts( projectName: 'build-scripts/release/create_installer_windows', @@ -1530,6 +1526,24 @@ class Build { } } + // Always clean any previous "openjdk_build_dir" output, possibly from any previous aborted build.. + try { + try { + context.timeout(time: buildTimeouts.NODE_CLEAN_TIMEOUT, unit: 'HOURS') { + if (context.WORKSPACE != null && !context.WORKSPACE.isEmpty()) { + context.println 'Removing workspace openjdk build directory: ' + openjdk_build_dir + context.sh(script: 'rm -rf ' + openjdk_build_dir) + } else { + context.println 'Warning: Unable to remove workspace openjdk build directory as context.WORKSPACE is null/empty' + } + } + } catch (FlowInterruptedException e) { + throw new Exception("[ERROR] Remove workspace openjdk build directory timeout (${buildTimeouts.NODE_CLEAN_TIMEOUT} HOURS) has been reached. Exiting...") + } + } catch (e) { + context.println "[WARNING] Failed to remove workspace openjdk build directory: ${e}" + } + try { context.timeout(time: buildTimeouts.NODE_CHECKOUT_TIMEOUT, unit: 'HOURS') { if (useAdoptShellScripts) { @@ -2009,13 +2023,13 @@ class Build { context.docker.image(buildConfig.DOCKER_IMAGE).pull() } } - // Store the pulled docker image digest as 'buildinfo' - dockerImageDigest = context.sh(script: "docker inspect --format='{{.RepoDigests}}' ${buildConfig.DOCKER_IMAGE}", returnStdout:true) } } catch (FlowInterruptedException e) { throw new Exception("[ERROR] Controller docker image pull timeout (${buildTimeouts.DOCKER_PULL_TIMEOUT} HOURS) has been reached. Exiting...") } } + // Store the pulled docker image digest as 'buildinfo' + dockerImageDigest = context.sh(script: "docker inspect --format='{{.RepoDigests}}' ${buildConfig.DOCKER_IMAGE}", returnStdout:true) // Use our dockerfile if DOCKER_FILE is defined if (buildConfig.DOCKER_FILE) { diff --git a/pipelines/build/common/trigger_beta_build.groovy b/pipelines/build/common/trigger_beta_build.groovy index a6e6e592e..3ee02a810 100644 --- a/pipelines/build/common/trigger_beta_build.groovy +++ b/pipelines/build/common/trigger_beta_build.groovy @@ -68,9 +68,9 @@ def isDuringReleasePeriod() { releaseTuesday = day17th.with(TemporalAdjusters.previous(DayOfWeek.TUESDAY)) } - // Release period no trigger from previous Saturday to following Sunday + // Release period no trigger from prior week previous Saturday to following Sunday def days = ChronoUnit.DAYS.between(releaseTuesday, now) - if (days >= -3 && days <= 5) { + if (days >= -10 && days <= 5) { releasePeriod = true } } diff --git a/pipelines/build/devkit/make_devkit.sh b/pipelines/build/devkit/make_devkit.sh index 5de64b128..d7672838e 100755 --- a/pipelines/build/devkit/make_devkit.sh +++ b/pipelines/build/devkit/make_devkit.sh @@ -59,9 +59,24 @@ if [ "${BASE_OS}" = "rhel" ]; then BASE_OS=Centos fi -# Perform devkit build +# Perform "bootstrap" devkit build +echo "Building 'bootstrap' DevKit toolchain, to be used to build the final DevKit..." cd make/devkit && pwd && make TARGETS=${devkit_target} BASE_OS=${BASE_OS} BASE_OS_VERSION=${BASE_OS_VERSION} -find ../../build/devkit -type f -print + +# Move "bootstrap" devkit toolchain to a new folder and setup gcc toolchain to point at it +cd ../.. +mv build/devkit/result/${devkit_target}-to-${devkit_target} build/bootstrap_${devkit_target}-to-${devkit_target} +export CC=$(pwd)/build/bootstrap_${devkit_target}-to-${devkit_target}/bin/gcc +export CXX=$(pwd)/build/bootstrap_${devkit_target}-to-${devkit_target}/bin/g++ +export LD_LIBRARY_PATH=$(pwd)/build/bootstrap_${devkit_target}-to-${devkit_target}/lib64 +export PATH=$(pwd)/build/bootstrap_${devkit_target}-to-${devkit_target}/bin:$PATH +gcc --version + +# Make final "DevKit" using the bootstrap devkit +rm -rf build/devkit +echo "Building 'final' DevKit toolchain, using 'bootstrap' toolchain in $(pwd)/build/bootstrap_${devkit_target}-to-${devkit_target}" +cd make/devkit && pwd && make TARGETS=${devkit_target} BASE_OS=${BASE_OS} BASE_OS_VERSION=${BASE_OS_VERSION} + # Back to original folder cd ../../.. diff --git a/pipelines/jobs/configurations/jdk17u_release.groovy b/pipelines/jobs/configurations/jdk17u_release.groovy index ac63150a0..e2b980400 100644 --- a/pipelines/jobs/configurations/jdk17u_release.groovy +++ b/pipelines/jobs/configurations/jdk17u_release.groovy @@ -31,6 +31,9 @@ targetConfigurations = [ ], 'arm32Linux' : [ 'temurin' + ], + 'riscv64Linux': [ + 'temurin' ] ] diff --git a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy index 4ffd01607..079e37e95 100644 --- a/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk21u_pipeline_config.groovy @@ -26,7 +26,7 @@ class Config21 { openj9 : 'pipelines/build/dockerFiles/cuda.dockerfile' ], test: [ - weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'sanity.external', 'dev.openjdk', 'dev.functional', 'dev.system'] + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'sanity.external', 'dev.openjdk', 'dev.functional', 'dev.system', 'special.system'] ], reproducibleCompare : [ 'temurin' : true @@ -70,7 +70,9 @@ class Config21 { os : 'windows', arch : 'x64', additionalNodeLabels: 'win2022&&vs2022', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], reproducibleCompare : [ 'temurin' : true ], diff --git a/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy b/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy index 9860c486c..e555d4c47 100644 --- a/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk22u_pipeline_config.groovy @@ -24,7 +24,7 @@ class Config22 { openj9 : 'pipelines/build/dockerFiles/cuda.dockerfile' ], test: [ - weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'dev.openjdk', 'dev.functional', 'dev.system'] + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'dev.openjdk', 'dev.functional', 'dev.system', 'special.system'] ], additionalTestLabels: [ openj9 : '!(centos6||rhel6)', @@ -65,7 +65,9 @@ class Config22 { os : 'windows', arch : 'x64', additionalNodeLabels: 'win2022&&vs2022', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22621.0/ucrt/DLLs/x64'", buildArgs : [ 'temurin' : '--create-jre-image --create-sbom' diff --git a/pipelines/jobs/configurations/jdk23_pipeline_config.groovy b/pipelines/jobs/configurations/jdk23_pipeline_config.groovy index 557e982a7..52cba20c9 100644 --- a/pipelines/jobs/configurations/jdk23_pipeline_config.groovy +++ b/pipelines/jobs/configurations/jdk23_pipeline_config.groovy @@ -24,7 +24,7 @@ class Config23 { openj9 : 'pipelines/build/dockerFiles/cuda.dockerfile' ], test: [ - weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'dev.openjdk', 'dev.functional', 'dev.system'] + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'dev.openjdk', 'dev.functional', 'dev.system', 'special.system'] ], additionalTestLabels: [ openj9 : '!(centos6||rhel6)', @@ -65,7 +65,9 @@ class Config23 { os : 'windows', arch : 'x64', additionalNodeLabels: 'win2022&&vs2022', - test : 'default', + test: [ + weekly : ['sanity.openjdk', 'sanity.system', 'extended.system', 'sanity.perf', 'sanity.functional', 'extended.functional', 'extended.openjdk', 'extended.perf', 'special.functional', 'special.openjdk', 'dev.functional', 'dev.system'] + ], configureArgs : "--with-ucrt-dll-dir='C:/progra~2/wi3cf2~1/10/Redist/10.0.22621.0/ucrt/DLLs/x64'", buildArgs : [ 'temurin' : '--create-jre-image --create-sbom'