From 3ed9424d60b68c323c89f26bba4ca92d909f44b1 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Wed, 4 Dec 2024 15:25:20 +0000 Subject: [PATCH 01/19] Adding secure mode option to build scripting This mode will eventually be used to disable a range of build script functionality that raises the potential security risk level during the build process. After this commit is merged, we will need to add the flag to the pipeline job configurations in order to pass it into build jobs. Signed-off-by: Adam Farley --- sbin/common/config_init.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 1a649a805..b4b9ab194 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -63,6 +63,7 @@ DISABLE_ADOPT_BRANCH_SAFETY DOCKER_FILE_PATH DOCKER_SOURCE_VOLUME_NAME ENABLE_SBOM_STRACE +ENABLE_SECURE_MODE FREETYPE FREETYPE_DIRECTORY FREETYPE_FONT_BUILD_TYPE_PARAM @@ -299,6 +300,9 @@ function parseConfigurationArguments() { "--enable-sbom-strace" ) BUILD_CONFIG[ENABLE_SBOM_STRACE]=true;; + "--enable-secure-mode" ) + BUILD_CONFIG[ENABLE_SECURE_MODE]=true;; + "--freetype-dir" | "-f" ) BUILD_CONFIG[FREETYPE_DIRECTORY]="$1"; shift;; @@ -558,6 +562,9 @@ function configDefaults() { BUILD_CONFIG[ENABLE_SBOM_STRACE]="false" + # Set default value to "false", for maximum user convenience. "false" enables potentially-insecure functionality, like the dynamic download of boot JDKs. + BUILD_CONFIG[ENABLE_SECURE_MODE]="false" + # The default behavior of whether we want to create a separate source archive BUILD_CONFIG[CREATE_SOURCE_ARCHIVE]="false" From 8d7d369ea47af6cca3202ff16e37ac61cadb7e1a Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 13 Dec 2024 15:45:21 +0000 Subject: [PATCH 02/19] Adding a download function library plus basic testing This is so that all downloads can take place through a single function, regardless of OS or scenario. Signed-off-by: Adam Farley --- sbin/common/lib/functionLibrary.sh | 219 +++++++++++++++++++++++ sbin/common/lib/functionLibraryTests.sh | 70 ++++++++ sbin/common/lib/sampleFileForTesting.txt | 3 + 3 files changed, 292 insertions(+) create mode 100644 sbin/common/lib/functionLibrary.sh create mode 100644 sbin/common/lib/functionLibraryTests.sh create mode 100644 sbin/common/lib/sampleFileForTesting.txt diff --git a/sbin/common/lib/functionLibrary.sh b/sbin/common/lib/functionLibrary.sh new file mode 100644 index 000000000..6d70fb452 --- /dev/null +++ b/sbin/common/lib/functionLibrary.sh @@ -0,0 +1,219 @@ +#!/usr/bin/bash +# This script contains a library of useful functions +# Functions list: +# downloadFile - Downloads a file and optionally checks it against a sha. +# info - echoes any string sent to it, but only if it is enabled first. +# checkFileSha - Checks whether a named file matches a supplied sha. +# doesThisURLExist - Checks if a given URL exists. + +# This simple function takes a string, and echoes it if info-level logging is enabled. +# Info-level logging can be enabled by simply passing the word "enable" to it as the sole argument. +# This is handy for debug messages, as it can simply not be enabled for quiet runs. +enableInfoLoggingBool=1 + +function info() { + [ $# == 0 ] && echo "Warning: The library function \"info\" was called without an argument." && return 1 + [[ $enableInfoLoggingBool == 0 ]] && echo $1 && return 0 + [[ "${1}" == "enable" ]] && enableInfoLoggingBool=0 && return 0 +} + +# This function takes the supplied sha and compares it against the supplied file. +# Example: checkFileSha 123456 /usr/bin/etc/exampleFile +# Caution: I strongly advise against using relative paths. +function checkFileSha() { + info "Checking if a file matches the sha256 checksum. Fails if there is no checksum." + if [ $# != 2 ]; then + echo "Error: checkFileSha() function was not supplied with exactly 2 arguments." + exit 1 + fi + + if [[ -z $1 ]]; then + info "No sha256 checksum found." + info "Check declared failed." + return 1 + fi + + if [[ ! -x $2 ]]; then + info "The file we're trying to check does not exist: ${2}" + return 1 + fi + + info "Checking if a file matches the checksum." + shaReturnCode=1 + if command -v sha256sum &> /dev/null; then + echo "$1 $2" | sha256sum -c --quiet + shaReturnCode=$? + elif command -v shasum &> /dev/null; then + echo "$1 $2" | shasum -a 256 -c &> /dev/null - + shaReturnCode=$? + else + echo "Error: Neither sha256sum nor shasum is available on this machine." + exit 1 + fi + + if [ $shaReturnCode != 0 ]; then + echo "Warning: File ${2} does not match the supplied checksum." + return 1 + else + info "File matches the checksum." + return 0 + fi +} + +# This function checks if a given URL (string argument) exists. +function doesThisURLExist() { + info "Checking if a given URL exists." + if [ $# == 0 ]; then + echo "Error: doesThisURLExist() function was not supplied with a URL." + exit 1 + fi + + spiderOutput=1 + if command -v wget &> /dev/null; then + wget --spider -q ${source} 2> /dev/null + spiderOutput=$? + elif command -v curl &> /dev/null; then + curl -I ${source} -s | grep "200 OK" -q + spiderOutput=$? + else + echo "Error: Neither wget nor curl could be found when downloading this file: ${source}" + exit 1 + fi + + return $spiderOutput +} + +# This function downloads files +# The accepted arguments are: +# -source (mandatory: a web URL where the file is located) +# -destination (mandatory: an existent folder where the file will be put) +# -filename (optional: the new name of the file post-download) +# -sha (optional: the anticipated sha of the downloaded file) +function downloadFile() { + source="" + destination="" + filename="" + sha="" + + arrayOfArgs=( "$@" ) + x=0 + while [[ ${#arrayOfArgs[@]} -gt $((x+1)) ]]; do + arg="${arrayOfArgs[x]}" + x="$((x+1))" + value="${arrayOfArgs[$x]}" + case "$arg" in + --source | -s ) + source="${value}" + ;; + + --destination | -d ) + destination="${value}" + ;; + + --filename | -f ) + filename="${value}" + ;; + + --sha | -sha ) + sha="${value}" + ;; + + *) echo >&2 "Invalid downloadFile argument: ${arg} ${value}"; exit 1;; + esac + x="$((x+1))" + done + + info "File download requested." + + if [[ "${source}" == "" || "${destination}" == "" ]]; then + echo "Error: function downloadFile requires both a source and a destination." + echo "Source detected: ${source}" + echo "Destination detected: ${destination}" + exit 1 + fi + + info "File details: " + info "- source: ${source}" + info "- destination: ${destination}" + info "- file name: ${filename}" + info "- sha256 checksum: ${sha}" + + if [ -z ${filename} ]; then + filename="${source##*/}" + fi + + info "Checking if source exists." + doesThisURLExist "${source}" + [[ $? != 0 ]] && echo "Error: File could not be found at source." && exit 1 + info "Source exists." + + info "Checking if destination folder exists." + [ ! -x ${destination} ] && echo "Error: Destination folder could not be found." && exit 1 + + info "Destination folder exists. Checking if file is already present." + if [ -x "${destination}/${filename}" ]; then + info "Warning: File already exists." + checkFileSha "${sha}" "${destination}/${filename}" + if [[ $? != 0 ]]; then + info "Warning: A file was found with the same name, and it does not match the supplied checksum. Removing file." + rm "${destination}/${filename}" + if [ $? != 0 ]; then + echo "Error: Could not remove file." + exit 1 + fi + else + info "A file was found with the same name, and it matches the supplied checksum. Skipping download." + exit 0 + fi + fi + if [ -x "${destination}/${source##*/}" ]; then + info "Warning: File already exists with the default file name: ${source##*/}" + checkFileSha "${sha}" "${destination}/${source##*/}" + if [[ $? != 0 ]]; then + info "Warning: A file was found with the same name, and it does not match the supplied checksum. Removing file." + rm "${destination}/${source##*/}" + if [ $? != 0 ]; then + echo "Error: Could not remove file." + exit 1 + fi + else + info "A file was found with the same name, and it matches the supplied checksum. Skipping download." + mv "${destination}/${source##*/}" "${destination}/${filename}" + exit 0 + fi + fi + + info "File not already downloaded. Attempting file download." + if command -v wget &> /dev/null; then + info "Found wget. Using wget to download the file." + wget -q -P "${destination}" "${source}" + elif command -v curl &> /dev/null; then + info "Found curl. Using curl to download the file." + curl -s "${source}" -o "${destination}" + fi + if [ ! -x "${destination}/${filename}" ]; then + mv "${destination}/${source##*/}" "${destination}/${filename}" + fi + + info "File download is complete." + + if [[ -n $sha ]]; then + checkFileSha "${destination}/${filename}" + if [[ $? != 0 ]]; then + echo "Error: Checksum does not match the downloaded file. Removing file." + rm "${destination}/${filename}" + exit 1 + fi + fi + + info "File has been downloaded successfully." + + info "Setting file permissions to 770." + chmod 770 "${destination}/${filename}" + [ $? != 0 ] && echo "Error: Checksum does not match the downloaded file. Removing file." && rm "${destination}/${filename}" && exit 1 + + info "File permissions set successfully." + info "File download script complete" + + return 0 +} diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh new file mode 100644 index 000000000..51616dc67 --- /dev/null +++ b/sbin/common/lib/functionLibraryTests.sh @@ -0,0 +1,70 @@ +#!/usr/bin/bash +# A set of tests for the functionLibrary script + +source functionLibrary.sh + +sampleFileURL=https://github.com/adamfarley/temurin-build +sampleFileName=sampleFileForTesting.txt +sampleFileSha="12345" + +successTotal=0 +failureTotal=0 + +# takes the name of the test and a boolean indicating whether it passed. +function testResults() { + if [[ $2 == 0 ]]; then + echo "Success: $1 has passed." + successTotal=$((successTotal+1)) + else + echo "Failure: $1 has failed." + failureTotal=$((failureTotal+1)) + fi +} + +# info +function infoTests(){ + # Does it work when it shouldn't? + [[ "$(info Test)" == "" ]] + testResults "infoTest 1" "$?" + + # Does it work when it should? + info "enable" + [[ "$(info 123)" == "123" ]] + testResults "infoTest 2" "$?" +} + +# checkFileSha +function checkFileShaTests(){ + return 0 +} + +# doesThisURLExist +function doesThisURLExistTests(){ + return 0 +} + +# downloadFile +function downloadFileTests(){ + return 0 +} + +echo "Test script start." +echo "" + +# Test execution +infoTests +checkFileShaTests +doesThisURLExistTests +downloadFileTests + +echo "" +echo "${successTotal} tests have passed." +echo "${failureTotal} tests have failed." +echo "" +if [[ $failureTotal -eq 0 ]]; then + echo "This test script has passed." + exit 0 +else + echo "This test script has failed." + exit 1 +fi diff --git a/sbin/common/lib/sampleFileForTesting.txt b/sbin/common/lib/sampleFileForTesting.txt new file mode 100644 index 000000000..5692ed2cc --- /dev/null +++ b/sbin/common/lib/sampleFileForTesting.txt @@ -0,0 +1,3 @@ +# A sample file which does not change. Used for download and sha256 testing. + +Sample file contents. \ No newline at end of file From 56a5566c1c91f52d9f5d40a8cc8cdfa2f708be0d Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 13 Dec 2024 16:26:35 +0000 Subject: [PATCH 03/19] Adding info tests and restructuring info Signed-off-by: Adam Farley --- sbin/common/lib/functionLibrary.sh | 23 +++++++++++++++++++---- sbin/common/lib/functionLibraryTests.sh | 20 +++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/sbin/common/lib/functionLibrary.sh b/sbin/common/lib/functionLibrary.sh index 6d70fb452..f42b21f78 100644 --- a/sbin/common/lib/functionLibrary.sh +++ b/sbin/common/lib/functionLibrary.sh @@ -7,14 +7,29 @@ # doesThisURLExist - Checks if a given URL exists. # This simple function takes a string, and echoes it if info-level logging is enabled. -# Info-level logging can be enabled by simply passing the word "enable" to it as the sole argument. +# Info-level logging can be enabled/disabled simply passing "enable/disable" "logging" to it as 2 arguments. # This is handy for debug messages, as it can simply not be enabled for quiet runs. enableInfoLoggingBool=1 function info() { - [ $# == 0 ] && echo "Warning: The library function \"info\" was called without an argument." && return 1 - [[ $enableInfoLoggingBool == 0 ]] && echo $1 && return 0 - [[ "${1}" == "enable" ]] && enableInfoLoggingBool=0 && return 0 + if [ $# == 0 ]; then + echo "Warning: The library function \"info\" was called without an argument." + return 1 + elif [[ $# == 1 ]]; then + if [[ $enableInfoLoggingBool == 0 ]]; then + echo $1 + return 0 + fi + else + if [[ "${2}" == "logging" ]]; then + if [[ "${1}" == "enable" ]]; then + enableInfoLoggingBool=0 + elif [[ "${1}" == "disable" ]]; then + enableInfoLoggingBool=1 + fi + return 0 + fi + fi } # This function takes the supplied sha and compares it against the supplied file. diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index 51616dc67..10b2ff2b8 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -3,9 +3,9 @@ source functionLibrary.sh -sampleFileURL=https://github.com/adamfarley/temurin-build -sampleFileName=sampleFileForTesting.txt -sampleFileSha="12345" +sampleFileURL="https://github.com/adamfarley/temurin-build/tree/build_scripts_secure_mode/sbin/common/lib" +sampleFileName="sampleFileForTesting.txt" +sampleFileSha="539446c23c650f24bb4061dc3ee50ee4a8ba68456c3fe19b86f8630f1df74465" successTotal=0 failureTotal=0 @@ -28,14 +28,24 @@ function infoTests(){ testResults "infoTest 1" "$?" # Does it work when it should? - info "enable" + info "enable" "logging" [[ "$(info 123)" == "123" ]] testResults "infoTest 2" "$?" + + # Clean up + info "disable" "logging" } # checkFileSha function checkFileShaTests(){ - return 0 + # Does it work when it should? + checkFileSha "${sampleFileSha}" "$(pwd)/${sampleFileName}" + testResults "checkFileShaTest 1" "$?" + + # Does it fail when we have the wrong sha? + checkFileSha "12345" "$(pwd)/${sampleFileName}" &> /dev/null + [[ "$?" != "0" ]] + testResults "checkFileShaTest 2" "$?" } # doesThisURLExist From c3d2bebbf916f8ff418bba53c08e33343b27c0bf Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Fri, 13 Dec 2024 16:35:05 +0000 Subject: [PATCH 04/19] Adding URL existence tests Signed-off-by: Adam Farley --- sbin/common/lib/functionLibrary.sh | 6 ++++-- sbin/common/lib/functionLibraryTests.sh | 14 +++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sbin/common/lib/functionLibrary.sh b/sbin/common/lib/functionLibrary.sh index f42b21f78..6fc8b3eac 100644 --- a/sbin/common/lib/functionLibrary.sh +++ b/sbin/common/lib/functionLibrary.sh @@ -85,10 +85,12 @@ function doesThisURLExist() { spiderOutput=1 if command -v wget &> /dev/null; then - wget --spider -q ${source} 2> /dev/null + info "Using wget to verify URL exists." + wget --spider -q ${1} 2> /dev/null spiderOutput=$? elif command -v curl &> /dev/null; then - curl -I ${source} -s | grep "200 OK" -q + info "Using curl to verify URL exists." + curl -I ${1} -s | grep "200 OK" -q spiderOutput=$? else echo "Error: Neither wget nor curl could be found when downloading this file: ${source}" diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index 10b2ff2b8..3d32e7fbe 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -50,7 +50,19 @@ function checkFileShaTests(){ # doesThisURLExist function doesThisURLExistTests(){ - return 0 + # Does it pass when it should? + doesThisURLExist "https://adoptium.net/index.html" + testResults "doesThisURLExistTest 1" "$?" + + # Does it fail when it should? + doesThisURLExist "https://thisurlshouldneverexist123456gibberish.com" &> /dev/null + [[ "$?" != "0" ]] + testResults "doesThisURLExistTest 2" "$?" + + # And does it fail when it's not even a URL? + doesThisURLExist "thisnonurlshouldneverexist123456gibberish" &> /dev/null + [[ "$?" != "0" ]] + testResults "doesThisURLExistTest 3" "$?" } # downloadFile From 3d03e27993d0825432237e81c73314e9474ffb46 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 12:51:28 +0000 Subject: [PATCH 05/19] Adding licenses and the first draft of the download test Signed-off-by: Adam Farley --- sbin/common/lib/functionLibrary.sh | 13 +++++++++++++ sbin/common/lib/functionLibraryTests.sh | 24 ++++++++++++++++++++++-- sbin/common/lib/sampleFileForTesting.txt | 13 +++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/sbin/common/lib/functionLibrary.sh b/sbin/common/lib/functionLibrary.sh index 6fc8b3eac..898f22bd1 100644 --- a/sbin/common/lib/functionLibrary.sh +++ b/sbin/common/lib/functionLibrary.sh @@ -1,4 +1,17 @@ #!/usr/bin/bash +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + # This script contains a library of useful functions # Functions list: # downloadFile - Downloads a file and optionally checks it against a sha. diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index 3d32e7fbe..f5e27735f 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -1,4 +1,17 @@ #!/usr/bin/bash +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + # A set of tests for the functionLibrary script source functionLibrary.sh @@ -66,8 +79,15 @@ function doesThisURLExistTests(){ } # downloadFile -function downloadFileTests(){ - return 0 +function downloadFileTests() { + workdir="$(pwd)/tmp_test_work_dir" + # Setup + [[ -x "${workdir}" ]] && echo "Error: Temporary test work directory exists and shouldn't." && exit 1 + mkdir "${workdir}" + [[ ! -x "${workdir}" ]] && echo "Error: Temporary test work directory could not be created." && exit 1 + + # Clean up + [[ ! (rm -rf "${workdir}") ]] && echo "Error: Temporary test work directory could not be deleted." && exit 1 } echo "Test script start." diff --git a/sbin/common/lib/sampleFileForTesting.txt b/sbin/common/lib/sampleFileForTesting.txt index 5692ed2cc..61e58c609 100644 --- a/sbin/common/lib/sampleFileForTesting.txt +++ b/sbin/common/lib/sampleFileForTesting.txt @@ -1,3 +1,16 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + # A sample file which does not change. Used for download and sha256 testing. Sample file contents. \ No newline at end of file From 7ca982d8a4aad3783ccf5cbeb0a7213470508962 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 12:57:08 +0000 Subject: [PATCH 06/19] Updating the sample file sha Signed-off-by: Adam Farley --- sbin/common/lib/functionLibraryTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index f5e27735f..4d4cd0e8e 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -18,7 +18,7 @@ source functionLibrary.sh sampleFileURL="https://github.com/adamfarley/temurin-build/tree/build_scripts_secure_mode/sbin/common/lib" sampleFileName="sampleFileForTesting.txt" -sampleFileSha="539446c23c650f24bb4061dc3ee50ee4a8ba68456c3fe19b86f8630f1df74465" +sampleFileSha="7eb664568090f0ac7f573b25e4ac7929a48f3fb39fb34e6b21421959acdf94b4" successTotal=0 failureTotal=0 From d887e78f044296f401934f4c3005635078af4141 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 14:54:27 +0000 Subject: [PATCH 07/19] More tests, secure mode, and line endings Signed-off-by: Adam Farley --- sbin/common/lib/functionLibrary.sh | 44 ++++++++++++-------- sbin/common/lib/functionLibraryTests.sh | 54 +++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/sbin/common/lib/functionLibrary.sh b/sbin/common/lib/functionLibrary.sh index 898f22bd1..dffaaa17f 100644 --- a/sbin/common/lib/functionLibrary.sh +++ b/sbin/common/lib/functionLibrary.sh @@ -52,7 +52,7 @@ function checkFileSha() { info "Checking if a file matches the sha256 checksum. Fails if there is no checksum." if [ $# != 2 ]; then echo "Error: checkFileSha() function was not supplied with exactly 2 arguments." - exit 1 + return 1 fi if [[ -z $1 ]]; then @@ -76,7 +76,7 @@ function checkFileSha() { shaReturnCode=$? else echo "Error: Neither sha256sum nor shasum is available on this machine." - exit 1 + return 1 fi if [ $shaReturnCode != 0 ]; then @@ -93,7 +93,7 @@ function doesThisURLExist() { info "Checking if a given URL exists." if [ $# == 0 ]; then echo "Error: doesThisURLExist() function was not supplied with a URL." - exit 1 + return 1 fi spiderOutput=1 @@ -107,7 +107,7 @@ function doesThisURLExist() { spiderOutput=$? else echo "Error: Neither wget nor curl could be found when downloading this file: ${source}" - exit 1 + return 1 fi return $spiderOutput @@ -119,11 +119,13 @@ function doesThisURLExist() { # -destination (mandatory: an existent folder where the file will be put) # -filename (optional: the new name of the file post-download) # -sha (optional: the anticipated sha of the downloaded file) +# -secure (optional: true/false - should this download be automatically failed?) function downloadFile() { source="" destination="" filename="" sha="" + secure="false" arrayOfArgs=( "$@" ) x=0 @@ -147,8 +149,12 @@ function downloadFile() { --sha | -sha ) sha="${value}" ;; - - *) echo >&2 "Invalid downloadFile argument: ${arg} ${value}"; exit 1;; + + --secure | -secure ) + [[ "${value}" == "true" ]] && secure="true" + ;; + + *) echo >&2 "Invalid downloadFile argument: ${arg} ${value}"; return 1;; esac x="$((x+1))" done @@ -159,7 +165,7 @@ function downloadFile() { echo "Error: function downloadFile requires both a source and a destination." echo "Source detected: ${source}" echo "Destination detected: ${destination}" - exit 1 + return 1 fi info "File details: " @@ -167,18 +173,21 @@ function downloadFile() { info "- destination: ${destination}" info "- file name: ${filename}" info "- sha256 checksum: ${sha}" + info "- secure: ${secure}" if [ -z ${filename} ]; then filename="${source##*/}" fi + [[ ${secure} == "true" ]] && echo "The attempted download of file ${filename} was blocked because secure mode is active." && return 1 + info "Checking if source exists." doesThisURLExist "${source}" - [[ $? != 0 ]] && echo "Error: File could not be found at source." && exit 1 + [[ $? != 0 ]] && echo "Error: File could not be found at source." && return 1 info "Source exists." info "Checking if destination folder exists." - [ ! -x ${destination} ] && echo "Error: Destination folder could not be found." && exit 1 + [ ! -x ${destination} ] && echo "Error: Destination folder could not be found." && return 1 info "Destination folder exists. Checking if file is already present." if [ -x "${destination}/${filename}" ]; then @@ -189,11 +198,11 @@ function downloadFile() { rm "${destination}/${filename}" if [ $? != 0 ]; then echo "Error: Could not remove file." - exit 1 + return 1 fi else info "A file was found with the same name, and it matches the supplied checksum. Skipping download." - exit 0 + return 0 fi fi if [ -x "${destination}/${source##*/}" ]; then @@ -204,12 +213,12 @@ function downloadFile() { rm "${destination}/${source##*/}" if [ $? != 0 ]; then echo "Error: Could not remove file." - exit 1 + return 0 fi else info "A file was found with the same name, and it matches the supplied checksum. Skipping download." mv "${destination}/${source##*/}" "${destination}/${filename}" - exit 0 + return 0 fi fi @@ -232,7 +241,7 @@ function downloadFile() { if [[ $? != 0 ]]; then echo "Error: Checksum does not match the downloaded file. Removing file." rm "${destination}/${filename}" - exit 1 + return 1 fi fi @@ -240,8 +249,11 @@ function downloadFile() { info "Setting file permissions to 770." chmod 770 "${destination}/${filename}" - [ $? != 0 ] && echo "Error: Checksum does not match the downloaded file. Removing file." && rm "${destination}/${filename}" && exit 1 - + if [ $? != 0 ]; then + echo "Error: Chmod has failed. Attempting to remove file." + rm "${destination}/${filename}" + return 1 + fi info "File permissions set successfully." info "File download script complete" diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index 4d4cd0e8e..961de84bd 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -78,16 +78,62 @@ function doesThisURLExistTests(){ testResults "doesThisURLExistTest 3" "$?" } + +sampleFileURL="https://raw.githubusercontent.com/adamfarley/temurin-build/refs/heads/build_scripts_secure_mode/sbin/common/lib" +sampleFileName="sampleFileForTesting.txt" +sampleFileSha="7eb664568090f0ac7f573b25e4ac7929a48f3fb39fb34e6b21421959acdf94b4" + + # downloadFile function downloadFileTests() { workdir="$(pwd)/tmp_test_work_dir" # Setup - [[ -x "${workdir}" ]] && echo "Error: Temporary test work directory exists and shouldn't." && exit 1 + [[ -x "${workdir}" ]] && echo "Error: Temporary test work directory exists and shouldn't: ${workdir}" && exit 1 mkdir "${workdir}" - [[ ! -x "${workdir}" ]] && echo "Error: Temporary test work directory could not be created." && exit 1 - + [[ ! -x "${workdir}" ]] && echo "Error: Temporary test work directory could not be created: ${workdir}" && exit 1 + + # Does it pass when it should (no sha)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" + [[ $? == 0 && -x "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 1" "$?" + rm -rf "${workdir}/*" + + # Does it pass when it should (sha)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "${sampleFileSha}" + [[ $? == 0 && -x "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 2" "$?" + exit 1 + rm -rf "${workdir}/*" + + # Does it correctly rename the downloaded file? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "${sampleFileSha}" -f "newfilename" + [[ $? == 0 && -x "${workdir}/newfilename" ]] + testResults "downloadFileTest 3" "$?" + rm -rf "${workdir}/*" + + # Does it fail when it should (no sha, source does not exist)? + downloadFile -s "${sampleFileURL}/thisFileDoesNotExist" -d "${workdir}" &> /dev/null + [[ $? != 0 && ! -x "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 4" "$?" + + # Does it fail when it should (with sha, source does not exist)? + downloadFile -s "${sampleFileURL}/thisFileDoesNotExist" -d "${workdir}" -sha "${sampleFileSha}" &> /dev/null + [[ $? != 0 && ! -x "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 5" "$?" + + # Does it fail when it should (with invalid sha, source exists)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "thisisaninvalidsha12345" -f "newfilename" &> /dev/null + [[ $? != 0 && ! -x "${workdir}/newfilename" ]] + testResults "downloadFileTest 6" "$?" + + # Does it fail when it should (secure mode)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -secure "true" &> /dev/null + [[ $? != 0 && ! -x "${workdir}/newfilename" ]] + testResults "downloadFileTest 7" "$?" + # Clean up - [[ ! (rm -rf "${workdir}") ]] && echo "Error: Temporary test work directory could not be deleted." && exit 1 + rm -rf "${workdir}" + [[ $? != 0 ]] && echo "Error: Temporary test work directory could not be deleted." && exit 1 } echo "Test script start." From ed8f429b8c1fc4e23b1945aee968974e7410a36d Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 14:56:15 +0000 Subject: [PATCH 08/19] Removing debug exit Signed-off-by: Adam Farley --- sbin/common/lib/functionLibraryTests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sbin/common/lib/functionLibraryTests.sh b/sbin/common/lib/functionLibraryTests.sh index 961de84bd..0f0396c1e 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/sbin/common/lib/functionLibraryTests.sh @@ -81,7 +81,7 @@ function doesThisURLExistTests(){ sampleFileURL="https://raw.githubusercontent.com/adamfarley/temurin-build/refs/heads/build_scripts_secure_mode/sbin/common/lib" sampleFileName="sampleFileForTesting.txt" -sampleFileSha="7eb664568090f0ac7f573b25e4ac7929a48f3fb39fb34e6b21421959acdf94b4" +sampleFileSha="041bef0ff1e6d44a0464a06131d20ea21e47da9359f485f3f59c9bdb92255379" # downloadFile @@ -102,7 +102,6 @@ function downloadFileTests() { downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "${sampleFileSha}" [[ $? == 0 && -x "${workdir}/${sampleFileName}" ]] testResults "downloadFileTest 2" "$?" - exit 1 rm -rf "${workdir}/*" # Does it correctly rename the downloaded file? From b57f9a97f55eb71ddd1bc34673eb497716af488d Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:39:58 +0000 Subject: [PATCH 09/19] Reorganising files and adding git action to run tests Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 41 +++++++++++++++++++ lib/README.md | 12 ++++++ {sbin/common/lib => lib}/functionLibrary.sh | 0 .../lib => lib/tests}/functionLibraryTests.sh | 4 +- .../tests}/sampleFileForTesting.txt | 0 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/function-lib-checker.yml create mode 100644 lib/README.md rename {sbin/common/lib => lib}/functionLibrary.sh (100%) rename {sbin/common/lib => lib/tests}/functionLibraryTests.sh (98%) rename {sbin/common/lib => lib/tests}/sampleFileForTesting.txt (100%) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml new file mode 100644 index 000000000..98ddfff08 --- /dev/null +++ b/.github/workflows/function-lib-checker.yml @@ -0,0 +1,41 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +# If a PR includes a change to any lib file, we run the function library tests. + +name: Check function library functions all work +on: + pull_request: + branches: [ master ] + paths: + - 'lib/**' + +env: + TEST_SCRIPT: "lib/tests/functionLibraryTests.sh" + +permissions: + contents: read + +jobs: + libtests: + permissions: + contents: read + issues: write + runs-on: ubuntu-latest + name: Run Build Function Library Tests + if: ${{ (github.repository == 'adoptium/temurin-build') || (github.event_name == 'workflow_dispatch') }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: "Run Tests" + run: bash "${PWD}/${TRIAGE_SCRIPT}" \ No newline at end of file diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 000000000..6f41c0d26 --- /dev/null +++ b/lib/README.md @@ -0,0 +1,12 @@ +## Build Library + +This folder contains the function library for the build repository. + +This includes a functionLibrary.sh that can be included in your scripts, +giving people the ability to download files, compare shas, etc, without +wasting the time needed to write code tocover all the edge cases +(can the file be downloaded, does it match the sha, etc). + +The tests folder contains testing for the function library, and will be +run against the function library script whenever any file in lib is changed +(see the github action \"function-lib-checker.yml\" for details) \ No newline at end of file diff --git a/sbin/common/lib/functionLibrary.sh b/lib/functionLibrary.sh similarity index 100% rename from sbin/common/lib/functionLibrary.sh rename to lib/functionLibrary.sh diff --git a/sbin/common/lib/functionLibraryTests.sh b/lib/tests/functionLibraryTests.sh similarity index 98% rename from sbin/common/lib/functionLibraryTests.sh rename to lib/tests/functionLibraryTests.sh index 0f0396c1e..e1339eeae 100644 --- a/sbin/common/lib/functionLibraryTests.sh +++ b/lib/tests/functionLibraryTests.sh @@ -14,9 +14,9 @@ # A set of tests for the functionLibrary script -source functionLibrary.sh +source ../functionLibrary.sh -sampleFileURL="https://github.com/adamfarley/temurin-build/tree/build_scripts_secure_mode/sbin/common/lib" +sampleFileURL="https://github.com/adamfarley/temurin-build/tree/build_scripts_secure_mode/lib" sampleFileName="sampleFileForTesting.txt" sampleFileSha="7eb664568090f0ac7f573b25e4ac7929a48f3fb39fb34e6b21421959acdf94b4" diff --git a/sbin/common/lib/sampleFileForTesting.txt b/lib/tests/sampleFileForTesting.txt similarity index 100% rename from sbin/common/lib/sampleFileForTesting.txt rename to lib/tests/sampleFileForTesting.txt From 205f41cf19f27c2d12da31c9fc8b387101b1e007 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:43:47 +0000 Subject: [PATCH 10/19] Excluding lib from build testing because it has its own tests Signed-off-by: Adam Farley --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5fb32a3f..68756ede6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,7 @@ on: - "sbin/**" - "**.sh" - "!tooling/build_autotriage/**" + - "!lib/**" - ".github/workflows/build.yml" - "security/**" - "cyclonedx-lib/**" From ade2985c857b07ecbecdd2547adaafa9392b027b Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:45:02 +0000 Subject: [PATCH 11/19] Temporary exclusion Signed-off-by: Adam Farley --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68756ede6..a0943cb3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ on: - "**.sh" - "!tooling/build_autotriage/**" - "!lib/**" - - ".github/workflows/build.yml" + - "!.github/workflows/build.yml" - "security/**" - "cyclonedx-lib/**" From 4b532df13a171a4280af8c8e89e3754f1122f905 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:47:03 +0000 Subject: [PATCH 12/19] undoing build.yml change Signed-off-by: Adam Farley --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0943cb3a..b5fb32a3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,8 +22,7 @@ on: - "sbin/**" - "**.sh" - "!tooling/build_autotriage/**" - - "!lib/**" - - "!.github/workflows/build.yml" + - ".github/workflows/build.yml" - "security/**" - "cyclonedx-lib/**" From b5c65827ac8189519e14799337bb64a320781e1f Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:48:38 +0000 Subject: [PATCH 13/19] Changing action name Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 98ddfff08..41acba30a 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -13,7 +13,8 @@ # If a PR includes a change to any lib file, we run the function library tests. -name: Check function library functions all work +name: Build Function Library Checker + on: pull_request: branches: [ master ] From 33680731459de5271e9f879fd8e1bca8a4e88621 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:50:17 +0000 Subject: [PATCH 14/19] Fixes for git action Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 41acba30a..87fbdce2c 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -14,7 +14,6 @@ # If a PR includes a change to any lib file, we run the function library tests. name: Build Function Library Checker - on: pull_request: branches: [ master ] @@ -39,4 +38,4 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: "Run Tests" - run: bash "${PWD}/${TRIAGE_SCRIPT}" \ No newline at end of file + run: bash "${PWD}/${TEST_SCRIPT}" \ No newline at end of file From 71e0ca28ac8930d483c2bf539bfc9c96b10ef40d Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:57:28 +0000 Subject: [PATCH 15/19] Removing concurrency from github action Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 87fbdce2c..66b6742f9 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -13,6 +13,7 @@ # If a PR includes a change to any lib file, we run the function library tests. +---- name: Build Function Library Checker on: pull_request: @@ -23,6 +24,11 @@ on: env: TEST_SCRIPT: "lib/tests/functionLibraryTests.sh" +# Cancel existing runs if user makes another push. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read @@ -30,7 +36,6 @@ jobs: libtests: permissions: contents: read - issues: write runs-on: ubuntu-latest name: Run Build Function Library Tests if: ${{ (github.repository == 'adoptium/temurin-build') || (github.event_name == 'workflow_dispatch') }} From 941d220151a39ec512ebd1947921a53b0ac30285 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:58:24 +0000 Subject: [PATCH 16/19] Fixing typo Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 66b6742f9..780f0a652 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -10,7 +10,6 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** - # If a PR includes a change to any lib file, we run the function library tests. ---- From 154106c03c9b4835b956090b0ed7d1bf8d5aa19c Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 15:59:16 +0000 Subject: [PATCH 17/19] Another typo Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 780f0a652..8524d014e 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -10,10 +10,10 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** -# If a PR includes a change to any lib file, we run the function library tests. ---- name: Build Function Library Checker + on: pull_request: branches: [ master ] From 338bf68ab083d363d36e1057930ad40f380be15b Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 16:00:35 +0000 Subject: [PATCH 18/19] typo --- .github/workflows/function-lib-checker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 8524d014e..01ce06e9d 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -11,7 +11,7 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************** ----- +--- name: Build Function Library Checker on: From fa64eb85e6ca4f68c95b0a47244a30de771b12b5 Mon Sep 17 00:00:00 2001 From: Adam Farley Date: Mon, 16 Dec 2024 16:02:28 +0000 Subject: [PATCH 19/19] Indentation fix Signed-off-by: Adam Farley --- .github/workflows/function-lib-checker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml index 01ce06e9d..010499f6e 100644 --- a/.github/workflows/function-lib-checker.yml +++ b/.github/workflows/function-lib-checker.yml @@ -17,8 +17,8 @@ name: Build Function Library Checker on: pull_request: branches: [ master ] - paths: - - 'lib/**' + paths: + - 'lib/**' env: TEST_SCRIPT: "lib/tests/functionLibraryTests.sh"