JenkinsfileBase

#!groovy

def getBuildList() {
	// This map is created in case of BUILD_LIST is not provided. Once we match all sub-folders with top-level test targets, this map can be simplified.
	def TESTPROJECTS = [system:'system', perf:'perf', jck:'jck', external:'external', functional: 'functional', openjdk:'openjdk', jdk:'openjdk', runtest:'', sanity:'', extended:'']
	String fullTarget="${TARGET}"
	String[] levelTargets = fullTarget.split('\\.')
	String groupTarget = levelTargets[-1]
	String[] partsTarget = groupTarget.split('_|-')
	String simpleTarget = partsTarget[0]
	def TESTPROJECT = TESTPROJECTS[simpleTarget]
	return TESTPROJECT
}

def makeTest(testParam) {
	String tearDownCmd = "$RESOLVED_MAKE; \$MAKE -f ./aqa-tests/TKG/testEnv.mk testEnvTeardown"
	String makeTestCmd = "$RESOLVED_MAKE; cd ./aqa-tests/TKG; \$MAKE $testParam"
	try {
		sh "$tearDownCmd"
		sh "$makeTestCmd"
	} finally {
		sh "$tearDownCmd"
	}
}

def setupEnv() {
	env.JOBSTARTTIME = sh(script: 'LC_TIME=C date +"%a, %d %b %Y %T %z"', returnStdout: true).trim()

	// Check what existing java processes are running
		echo "PROCESSCATCH: Java processes which are currently on the machine:"
	if (PLATFORM.contains("windows")) {
		sh(script:'powershell -c "Get-Process java | select id,processname,starttime"', returnStatus:true)
	} else {
		sh(script:'ps -fu '+"${env.USER}"+' | grep java | egrep -v "remoting.jar|agent.jar|grep"', returnStatus: true)
	}

	if ( params.JDK_VERSION ) {
		env.ORIGIN_JDK_VERSION = params.JDK_VERSION
		env.JDK_VERSION = params.JDK_VERSION.equalsIgnoreCase("next") ? "" : params.JDK_VERSION
	}

	env.SPEC = "${SPEC}"

	ADOPTOPENJDK_REPO = params.ADOPTOPENJDK_REPO ? params.ADOPTOPENJDK_REPO : "https://github.com/adoptium/aqa-tests.git"
	OPENJ9_REPO = params.OPENJ9_REPO ? params.OPENJ9_REPO : "https://github.com/eclipse-openj9/openj9.git"

	String[] tkg = getGitRepoBranch(params.TKG_OWNER_BRANCH, "adoptium:master", "TKG")
	TKG_REPO = tkg[0]
	TKG_BRANCH = tkg[1]

	//For Zos, the right repo should be something like: git@github.ibm.com:runtimes/openj9-openjdk-jdk**-zos.git and for now there is only jdk11
	env.JDK_REPO = params.JDK_REPO ? params.JDK_REPO : ""
	if (env.SPEC.startsWith('zos')) {
		ADOPTOPENJDK_REPO = ADOPTOPENJDK_REPO.replace("https://github.com/","git@github.com:")
		OPENJ9_REPO = OPENJ9_REPO.replace("https://github.com/","git@github.com:")
	}

	PLATFORM = params.PLATFORM ? params.PLATFORM : ""
	ADOPTOPENJDK_BRANCH = params.ADOPTOPENJDK_BRANCH ? params.ADOPTOPENJDK_BRANCH : "master"
	CLONE_OPENJ9 = params.CLONE_OPENJ9 ? params.CLONE_OPENJ9 : "true"
	OPENJ9_BRANCH = params.OPENJ9_BRANCH ? params.OPENJ9_BRANCH : "master"
	CUSTOM_TARGET = params.CUSTOM_TARGET ? params.CUSTOM_TARGET : ""
	UPSTREAM_JOB_NAME = params.UPSTREAM_JOB_NAME ? params.UPSTREAM_JOB_NAME : ""
	UPSTREAM_JOB_NUMBER = params.UPSTREAM_JOB_NUMBER ? params.UPSTREAM_JOB_NUMBER : ""
	SSH_AGENT_CREDENTIAL = params.SSH_AGENT_CREDENTIAL ? params.SSH_AGENT_CREDENTIAL : ""
	KEEP_WORKSPACE = params.KEEP_WORKSPACE ? params.KEEP_WORKSPACE : false
	OPENJ9_SHA = params.OPENJ9_SHA ? params.OPENJ9_SHA : ""
	CLOUD_PROVIDER = params.CLOUD_PROVIDER ? params.CLOUD_PROVIDER : ""
	env.JDK_BRANCH = params.JDK_BRANCH ? params.JDK_BRANCH : ""
	env.USER_CREDENTIALS_ID = params.USER_CREDENTIALS_ID ? params.USER_CREDENTIALS_ID : ""
	env.TEST_JDK_HOME = "$WORKSPACE/openjdkbinary/j2sdk-image"
	env.JVM_VERSION = params.JVM_VERSION ? params.JVM_VERSION : ""
	env.JVM_OPTIONS = params.JVM_OPTIONS ? params.JVM_OPTIONS : ""
	env.EXTRA_OPTIONS = params.EXTRA_OPTIONS ? params.EXTRA_OPTIONS : ""
	env.EXTRA_DOCKER_ARGS = params.EXTRA_DOCKER_ARGS ? params.EXTRA_DOCKER_ARGS : ""
	env.OPENJDK_SHA = params.OPENJDK_SHA ? params.OPENJDK_SHA : ""
	env.TEST_FLAG = params.TEST_FLAG ? params.TEST_FLAG : ''
	env.KEEP_REPORTDIR = params.KEEP_REPORTDIR ? params.KEEP_REPORTDIR : false
	SDK_RESOURCE = params.SDK_RESOURCE ? params.SDK_RESOURCE : "upstream"
	env.AUTO_DETECT = params.AUTO_DETECT
	env.EXTERNAL_CUSTOM_REPO=params.EXTERNAL_CUSTOM_REPO? params.EXTERNAL_CUSTOM_REPO : ""
	env.EXTERNAL_REPO_BRANCH=params.EXTERNAL_REPO_BRANCH ? params.EXTERNAL_REPO_BRANCH : "master"
	env.EXTERNAL_TEST_CMD=params.EXTERNAL_TEST_CMD ? params.EXTERNAL_TEST_CMD : "mvn clean install"

	ITERATIONS = params.ITERATIONS ? "${params.ITERATIONS}".toInteger() : 1

	if (params.JRE_IMAGE) {
		env.JRE_IMAGE = "${WORKSPACE}/${params.JRE_IMAGE}"
	}

	if (params.JDK_IMPL) {
		env.JDK_IMPL = params.JDK_IMPL
	} else if (params.JVM_VERSION) {
		env.JDK_IMPL = getJDKImpl(params.JVM_VERSION)
	}
	if( params.BUILD_LIST ) {
		env.BUILD_LIST = params.BUILD_LIST
	} else {
		env.BUILD_LIST = "${getBuildList()}"
	}

	if( params.PERF_ROOT ) {
		env.PERF_ROOT = params.PERF_ROOT
	} else {
		env.PERF_ROOT = "$WORKSPACE/../../benchmarks"
	}

	env.JCK_VERSION = params.JCK_VERSION ? params.JCK_VERSION : ""
	env.JCK_ROOT = params.JCK_ROOT ? params.JCK_ROOT : ""
	env.JCK_GIT_REPO = params.JCK_GIT_REPO ? params.JCK_GIT_REPO : ""

	env.OCP_SERVER = params.OCP_SERVER ? params.OCP_SERVER : ''
	env.OCP_TOKEN = params.OCP_TOKEN ? params.OCP_TOKEN : ''

	if (env.BUILD_LIST == 'openjdk' ||  env.BUILD_LIST.contains('external')) {
		env.DIAGNOSTICLEVEL ='noDetails'
	}

	if( params.DOCKERIMAGE_TAG ) {
		env.DOCKERIMAGE_TAG = params.DOCKERIMAGE_TAG
	}
	if ( env.SPEC.contains('sunos')) {
		sh 'env'
	} else {
		sh 'printenv'
	}
}

def setupParallelEnv() {
	stage('setupParallelEnv') {
		timestamps{
			def maxChildJobNum = 25
			def testSubDirs = []
			int childJobNum = 1
			def UPSTREAM_TEST_JOB_NAME = ""
			def UPSTREAM_TEST_JOB_NUMBER = ""

			if (params.PARALLEL == "NodesByIterations") {
				childJobNum = params.NUM_MACHINES ? params.NUM_MACHINES.toInteger() : 1
				// limit childJobNum
				if (childJobNum > 20) {
					echo "Due to the limited machines, NUM_MACHINES can only be set up to 20. Current NUM_MACHINES is ${NUM_MACHINES}."
					echo "Reset NUM_MACHINES to 20..."
					childJobNum = 20
				}
			} else if (params.PARALLEL == "Dynamic") {
				String PARALLEL_OPTIONS = "TEST=${TARGET}"
				if (params.NUM_MACHINES) {
					int numOfMachines = getNumMachines()
					PARALLEL_OPTIONS += " NUM_MACHINES=${numOfMachines} TEST_TIME="
				} else if (params.TEST_TIME) {
					PARALLEL_OPTIONS += " TEST_TIME=${params.TEST_TIME} NUM_MACHINES="
				} else {
					PARALLEL_OPTIONS += " TEST_TIME= NUM_MACHINES="
				}
				if (params.TRSS_URL) {
					PARALLEL_OPTIONS += " TRSS_URL=${params.TRSS_URL}"
				}

				try {
					//get cached TRSS JSON data
					timeout(time: 1, unit: 'HOURS') {
						copyArtifacts fingerprintArtifacts: true, projectName: "getTRSSOutput", selector: lastSuccessful(), target: 'aqa-tests/TKG/resources/TRSS'
						sh "cd ./aqa-tests/TKG/resources/TRSS; gzip -cd TRSSOutput.tar.gz | tar xof -; rm TRSSOutput.tar.gz"
					}
				} catch (Exception e) {
					echo 'Cannot get cached TRSS JSON data. Skipping copyArtifacts...'
				}

				try {
					//get pre-staged jars from test.getDependency build
					timeout(time: 1, unit: 'HOURS') {
						copyArtifacts fingerprintArtifacts: true, projectName: "test.getDependency", selector: lastSuccessful(), target: 'aqa-tests/TKG/lib'
					}
				} catch (Exception e) {
					echo 'Cannot run copyArtifacts from test.getDependency. Skipping copyArtifacts...'
				}

				sh "cd ./aqa-tests/TKG; make genParallelList ${PARALLEL_OPTIONS}"

				// get NUM_LIST from parallelList.mk. NUM_LIST can be different than numOfMachines
				def parallelList = "aqa-tests/TKG/parallelList.mk"
				int NUM_LIST = -1
				if (fileExists("${parallelList}")) {
					if (SPEC.startsWith('zos')) {
						echo 'Converting parallelList.mk file from ebcdic to ascii...'
						sh "iconv -f ibm-1047 -t iso8859-1 ${parallelList} > ${parallelList}.ascii; rm ${parallelList}; mv ${parallelList}.ascii ${parallelList}"
					}
					echo "read parallelList.mk file: ${parallelList}"
					def properties = readProperties file: "${parallelList}"
					if (properties.NUM_LIST) {
						NUM_LIST = properties.NUM_LIST.toInteger()
					}
				}
				if (!params.NUM_MACHINES && params.TEST_TIME && NUM_LIST > getMachineLimit()) {
					assert false : "Build failed. TEST_TIME (${params.TEST_TIME} minutes) is not possible as there are not enough slave machines. Please provide a larger TEST_TIME."
				} else if ( NUM_LIST > 0) {
					childJobNum = NUM_LIST
					echo "Saving parallelList.mk file on jenkins..."
					dir('aqa-tests/TKG') {
						archiveArtifacts artifacts: 'parallelList.mk', fingerprint: true, allowEmptyArchive: false
					}

				} else {
					assert false : "Build failed because cannot find NUM_LIST in parallelList.mk file."
				}
			} else if (params.PARALLEL == "Subdir") {
				dir("$WORKSPACE/aqa-tests/${env.BUILD_LIST}") {
					testSubDirs = sh(returnStdout: true, script: "ls -d */").trim().tokenize()
				}

				if (TARGET.contains('special.system')) {
					// In special.system, some subfolders do not have any system test in special level
					// In order to save machine resources, exclude the following system test subfolders in parallel mode
					def excludes = ["jlm/", "modularity/", "sharedClasses/"]
					echo "exclude the following system test subfolders: ${excludes}"
					testSubDirs = testSubDirs - excludes
				}

				childJobNum = testSubDirs.size()
				if ( childJobNum > maxChildJobNum) {
					assert false : "Build failed becuase childJobNum: ${childJobNum} > ${maxChildJobNum}."
				}
			}
			UPSTREAM_TEST_JOB_NAME = JOB_NAME
			UPSTREAM_TEST_JOB_NUMBER = BUILD_NUMBER
			echo "[PARALLEL: ${params.PARALLEL}] childJobNum is ${childJobNum}, creating jobs and running them in parallel..."
			parallel_tests = [:]
			create_jobs = [:]

			for (int i = 0; i < childJobNum; i++) {
				def buildListName = env.BUILD_LIST
				def childTest = ""
				def childTarget = TARGET
				if (params.PARALLEL == "NodesByIterations") {
					childTest = "iteration_${i}"
				} else if (params.PARALLEL == "Dynamic") {
					childTest = "testList_${i}"
					childTarget = "-f parallelList.mk ${childTest}"
				} else if (params.PARALLEL == "Subdir") {
					childTest = testSubDirs[i].trim().replace("/","");
					buildListName = "${env.BUILD_LIST}/${childTest}"
				}

				def TEST_JOB_NAME = "${JOB_NAME}_${childTest}"

				// If GENERATE_JOBS is set to true, force generate the child job. Otherwise, only generate the child job if it does not exist
				if (params.GENERATE_JOBS) {
					create_jobs[childTest] = {
						echo "GENERATE_JOBS is set to true, set test job ${TEST_JOB_NAME} params for generating the job"
						createJob( TEST_JOB_NAME, PLATFORM)
					}
				} else {
					def jobIsRunnable = false
					try {
						def JobHelper = library(identifier: 'openjdk-jenkins-helper@master').JobHelper
						jobIsRunnable = JobHelper.jobIsRunnable("${TEST_JOB_NAME}")
						echo "${TEST_JOB_NAME} jobIsRunnable: ${jobIsRunnable}"
					} catch (Exception e) {
						echo "Cannot call jobIsRunnable() from openjdk-jenkins-helper@master. Skipping..."
					}
					if (!jobIsRunnable) {
						create_jobs[childTest] = {
							echo "Test job ${TEST_JOB_NAME} doesn't exist, set test job ${TEST_JOB_NAME} params for generating the job"
							createJob( TEST_JOB_NAME, PLATFORM)
						}
					}
				}

				def childParams = []
				// loop through all the params and change the parameters if needed
				params.each { param ->
					// set PARALLEL, NUM_MACHINES and TEST_TIME to default values
					if (param.key == "BUILD_LIST") {
						childParams << string(name: param.key, value: "${buildListName}")
					} else if (param.key == "TARGET") {
						childParams << string(name: param.key, value: "${childTarget}")
					} else if (param.key == "PARALLEL") {
						childParams << string(name: param.key, value: "None")
					} else if (param.key == "NUM_MACHINES") {
						childParams << string(name: param.key, value: "")
					} else if (param.key == "TEST_TIME") {
						childParams << string(name: param.key, value: "")
					}else {
						def value = param.value.toString()
						if (value == "true" || value == "false") {
							childParams << booleanParam(name: param.key, value: value.toBoolean())
						} else {
							childParams << string(name: param.key, value: value)
						}
					}
				}

				childParams << string(name: 'UPSTREAM_TEST_JOB_NAME', value: UPSTREAM_TEST_JOB_NAME)
				childParams << string(name: 'UPSTREAM_TEST_JOB_NUMBER', value: UPSTREAM_TEST_JOB_NUMBER)

				parallel_tests[childTest] = {
					build job: TEST_JOB_NAME, parameters: childParams, propagate: false
				}
			}
			
			if (create_jobs) {
				parallel create_jobs
			}

			// return to top level pipeline file in order to exit node block before running tests in parallel
		}
	}
}


// Returns num
// num = params.NUM_MACHINES. If it is not provided, the default value is 1
// num cannot be greater than machines limit
def getNumMachines(){
	int num = params.NUM_MACHINES ? params.NUM_MACHINES.toInteger() : 1
	int limit = getMachineLimit()
	echo "machine limit is ${limit}"
	if (num > limit) {
		echo "Number of machines cannot be greater than ${limit}. Set num to ${limit}"
		num = limit
	}
	return num
}

def getMachineLimit(){
	int limit = nodesByLabel(LABEL).size()
	// Set limit for dynamic vm agents to 100
	if (LABEL.contains('ci.agent.dynamic')) {
		limit = 100
	}
	return limit
}

def createJob( TEST_JOB_NAME, ARCH_OS ) {

	def jobParams = [:]
	jobParams.put('TEST_JOB_NAME', TEST_JOB_NAME)
	jobParams.put('ARCH_OS_LIST', ARCH_OS)

	//get level and group if TEST_JOB_NAME matches the format of Test_openjdk11_j9_extended.functional_ppc64_aix
	//otherwise, use default level and group value in template
	if (TEST_JOB_NAME.startsWith("Test_openjdk")) {
		def level = ""
		def group = ""
		def tokens = TEST_JOB_NAME.split('_');
		if (tokens.size() > 3 && tokens[3].contains(".")) {
			level = tokens[3].split("\\.")[0]
			group = tokens[3].split("\\.")[1]
		}

		if (level && group) {
			jobParams.put('LEVELS', level)
			jobParams.put('GROUPS', group)
		}
	}

	def templatePath = 'aqa-tests/buildenv/jenkins/testJobTemplate'

	if (params.LIGHT_WEIGHT_CHECKOUT) {
		jobParams.put('LIGHT_WEIGHT_CHECKOUT', params.LIGHT_WEIGHT_CHECKOUT)
	}

	def create = jobDsl targets: templatePath, ignoreExisting: false, additionalParameters: jobParams
	return create
}

def setup() {
	stage('Setup') {
		timestamps{
			setupEnv()

			if (params.SDK_RESOURCE == 'nightly' && params.CUSTOMIZED_SDK_URL) {
				CUSTOMIZED_SDK_URL_OPTION = "-c '${params.CUSTOMIZED_SDK_URL}'"
			} else if (params.CUSTOMIZED_SDK_URL) {
				SDK_RESOURCE = "customized"
				CUSTOMIZED_SDK_URL_OPTION = "-c '${params.CUSTOMIZED_SDK_URL}'"
			} else {
				CUSTOMIZED_SDK_URL_OPTION = ""
			}
			if (params.CUSTOMIZED_SDK_SOURCE_URL) {
				SDK_RESOURCE = "customized"
				CUSTOMIZED_SDK_SOURCE_URL_OPTION = "-S '${params.CUSTOMIZED_SDK_SOURCE_URL}'"
			} else {
				CUSTOMIZED_SDK_SOURCE_URL_OPTION = ""
			}

			if (SDK_RESOURCE == 'upstream' && !params.CUSTOMIZED_SDK_URL) {
				timeout(time: 1, unit: 'HOURS') {
					dir('openjdkbinary') {
						step([$class: 'CopyArtifact',
							fingerprintArtifacts: true,
							flatten: true,
							filter: "**/*.tar.gz,**/*.tgz,**/*.zip,**/*.jar,**/*.Z",
							projectName: "${params.UPSTREAM_JOB_NAME}",
							selector: [$class: 'SpecificBuildSelector', buildNumber: "${params.UPSTREAM_JOB_NUMBER}"]])
					}
				}
			}
			CLONE_OPENJ9_OPTION = (params.CLONE_OPENJ9) ? "--clone_openj9 ${params.CLONE_OPENJ9}" : ""
			OPENJ9_REPO_OPTION = "--openj9_repo ${OPENJ9_REPO}"
			OPENJ9_BRANCH_OPTION = "--openj9_branch ${OPENJ9_BRANCH}"
			OPENJ9_SHA_OPTION = (params.OPENJ9_SHA) ? "--openj9_sha ${params.OPENJ9_SHA}" : ""
			TKG_REPO_OPTION = "--tkg_repo ${TKG_REPO}"
			TKG_BRANCH_OPTION = "--tkg_branch ${TKG_BRANCH}"
			JDK_VERSION_OPTION = env.JDK_VERSION ? "-j ${env.JDK_VERSION}" : ""
			JDK_IMPL_OPTION = env.JDK_IMPL ? "-i ${env.JDK_IMPL}" : ""

			// system test repository exports to be used by system/common.xml
			if (params.ADOPTOPENJDK_SYSTEMTEST_OWNER_BRANCH) {
				String[] adoptSystemTest = getGitRepoBranch(params.ADOPTOPENJDK_SYSTEMTEST_OWNER_BRANCH, "adoptium:master", "aqa-systemtest")
				env.ADOPTOPENJDK_SYSTEMTEST_REPO = adoptSystemTest[0]
				env.ADOPTOPENJDK_SYSTEMTEST_BRANCH = adoptSystemTest[1]
			}

			if (params.OPENJ9_SYSTEMTEST_OWNER_BRANCH) {
				String[] openj9SystemTest = getGitRepoBranch(params.OPENJ9_SYSTEMTEST_OWNER_BRANCH, "eclipse:master", "openj9-systemtest")
				env.OPENJ9_SYSTEMTEST_REPO = openj9SystemTest[0]
				env.OPENJ9_SYSTEMTEST_BRANCH = openj9SystemTest[1]
			}

			if (params.STF_OWNER_BRANCH) {
				String[] stf = getGitRepoBranch(params.STF_OWNER_BRANCH, "adoptium:master", "stf")
				env.STF_REPO = stf[0]
				env.STF_BRANCH = stf[1]
			}

			// vendor test
			// expect VENDOR_TEST_* to be comma separated string parameters
			VENDOR_TEST_REPOS = (params.VENDOR_TEST_REPOS) ? "--vendor_repos \"${params.VENDOR_TEST_REPOS}\"" : ""
			VENDOR_TEST_BRANCHES = (params.VENDOR_TEST_BRANCHES) ? "--vendor_branches \"${params.VENDOR_TEST_BRANCHES}\"" : ""
			VENDOR_TEST_DIRS = (params.VENDOR_TEST_DIRS) ? "--vendor_dirs \"${params.VENDOR_TEST_DIRS}\"" : ""
			VENDOR_TEST_SHAS = (params.VENDOR_TEST_SHAS) ? "--vendor_shas \"${params.VENDOR_TEST_SHAS}\"" : ""

			// handle three cases (true/false/null) in params.TEST_IMAGES_REQUIRED and params.DEBUG_IMAGES_REQUIRED
			// Only set image required to false if params is set to false. In get.sh, the default value is true
			TEST_IMAGES_REQUIRED = (params.TEST_IMAGES_REQUIRED == false) ? "--test_images_required false" : ""
			DEBUG_IMAGES_REQUIRED = (params.DEBUG_IMAGES_REQUIRED == false) ? "--debug_images_required false" : ""
			GET_SH_CMD = "./get.sh -s `pwd`/.. -p $PLATFORM -r ${SDK_RESOURCE} ${JDK_VERSION_OPTION} ${JDK_IMPL_OPTION} ${CUSTOMIZED_SDK_URL_OPTION} ${CUSTOMIZED_SDK_SOURCE_URL_OPTION} ${CLONE_OPENJ9_OPTION} ${OPENJ9_REPO_OPTION} ${OPENJ9_BRANCH_OPTION} ${OPENJ9_SHA_OPTION} ${TKG_REPO_OPTION} ${TKG_BRANCH_OPTION} ${VENDOR_TEST_REPOS} ${VENDOR_TEST_BRANCHES} ${VENDOR_TEST_DIRS} ${VENDOR_TEST_SHAS} ${TEST_IMAGES_REQUIRED} ${DEBUG_IMAGES_REQUIRED}"
			RESOLVED_MAKE = "if [ `uname` = AIX ] || [ `uname` = SunOS ] || [ `uname` = *BSD ]; then MAKE=gmake; else MAKE=make; fi"
			
			dir( WORKSPACE) {
				// use sshagent with Jenkins credentials ID for all platforms except zOS
				// on zOS use the user's ssh key
				if (!env.SPEC.startsWith('zos')) {
					get_sources_with_authentication()
				} else {
					get_sources()
				}
				getJobProperties()
			}
		}
	}
}

def getJobProperties() {
	def jobProperties = "./aqa-tests/job.properties"
	if (fileExists("${jobProperties}")) {
		echo "readProperties file: ${jobProperties}"
		def properties = readProperties file: "${jobProperties}"
		if (properties.TEST_JDK_HOME) {
			env.TEST_JDK_HOME = properties.TEST_JDK_HOME
			echo "Reset TEST_JDK_HOME to ${TEST_JDK_HOME}"
		}
	}
}

def get_sources_with_authentication() {
	sshagent(credentials:["${params.USER_CREDENTIALS_ID}"], ignoreMissing: true) {
		get_sources()
	}
}

def get_sources() {
	withEnv(['VERBOSE_CURL=SILENT']) {
		dir('aqa-tests') {
			if (params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID) {
				// USERNAME and PASSWORD reference with a withCredentials block will not be visible within job output
				withCredentials([usernamePassword(credentialsId: "${params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
					sh "$GET_SH_CMD"
				}
			} else {
				sh "$GET_SH_CMD"
			}
		}
	}
}

def makeCompileTest() {
	String makeTestCmd = "$RESOLVED_MAKE; cd ./aqa-tests/TKG; \$MAKE compile"
	// use sshagent with Jenkins credentials ID for all platforms except zOS
	if (!env.SPEC.startsWith('zos')) {
		sshagent (credentials: ["$params.SSH_AGENT_CREDENTIAL"], ignoreMissing: true) {
			sh "$makeTestCmd";
		}
	} else {
		sh "$makeTestCmd";
	}
}

def buildTest() {
	stage('Build') {
		timestamps{
			echo 'Building tests...'

			if ( params.PERF_CREDENTIALS_ID ) {
				withCredentials([usernamePassword(credentialsId: "$params.PERF_CREDENTIALS_ID",
					passwordVariable: "PASSWORD_VAR", usernameVariable: "USERNAME_VAR")]) {
						env.PERF_USERNAME = USERNAME_VAR
						env.PERF_PASSWORD = PASSWORD_VAR
				}
			}

			if (params.UPSTREAM_TEST_JOB_NAME && params.UPSTREAM_TEST_JOB_NUMBER) {
				try {
					timeout(time: 1, unit: 'HOURS') {
						copyArtifacts fingerprintArtifacts: true, projectName: params.UPSTREAM_TEST_JOB_NAME, selector: specific(params.UPSTREAM_TEST_JOB_NUMBER), target: './aqa-tests/TKG/'
					}
				} catch (Exception e) {
					echo "Cannot run copyArtifacts from ${params.UPSTREAM_TEST_JOB_NAME} ${params.UPSTREAM_TEST_JOB_NUMBER}. Skipping copyArtifacts..."
				}
			}

			try {
				//get pre-staged jars from test.getDependency build before test compilation
				timeout(time: 1, unit: 'HOURS') {
					copyArtifacts fingerprintArtifacts: true, projectName: "test.getDependency", selector: lastSuccessful(), target: 'aqa-tests/TKG/lib'
				}
			} catch (Exception e) {
				echo 'Cannot run copyArtifacts from test.getDependency. Skipping copyArtifacts...'
			}

			try {
				if (env.BUILD_LIST.startsWith('system') || env.BUILD_LIST.startsWith('jck')) {
					//get pre-staged test jars from systemtest.getDependency build before system test compilation
					timeout(time: 1, unit: 'HOURS') {
						copyArtifacts fingerprintArtifacts: true, projectName: "systemtest.getDependency", selector: lastSuccessful(), target: 'aqa-tests'
					}
				}
			} catch (Exception e) {
				echo 'Cannot run copyArtifacts from systemtest.getDependency. Skipping copyArtifacts...'
			}

			if (fileExists('openjdkbinary/openjdk-test-image')) {
				env.TESTIMAGE_PATH = "$WORKSPACE/openjdkbinary/openjdk-test-image"
			}

			if (fileExists('openjdkbinary/openjdk-test-image/openj9')) {
				env.NATIVE_TEST_LIBS = "$WORKSPACE/openjdkbinary/openjdk-test-image/openj9"
			}

			if(params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID) {
				withCredentials([usernamePassword(credentialsId: "${params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID}",
					usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
					makeCompileTest()
				}
			} else {
				makeCompileTest()
			}
		}
	}
}

def runTest( ) {
	stage('Test') {
		timestamps{
			echo 'Running tests...'
			def CUSTOM_OPTION = ''

			TARGET = "${params.TARGET}";
			if (TARGET.contains('custom') && CUSTOM_TARGET!='') {
				if (TARGET == 'system_custom') {
					env.SYSTEM_CUSTOM_TARGET=CUSTOM_TARGET
				} else {
					CUSTOM_OPTION = "${TARGET.toUpperCase()}_TARGET='${CUSTOM_TARGET}'"
				}
			}
			if (!TARGET.startsWith('-f')) {
				TARGET="_${params.TARGET}"
			} else if (TARGET.contains('-f parallelList.mk') && SPEC.startsWith('zos')) {
				def parallelList = "aqa-tests/TKG/parallelList.mk"
				if (fileExists("${parallelList}")) {
					echo 'Converting parallelList.mk file from ascii to ebcdic...'
					sh "iconv -f iso8859-1 -t ibm-1047 ${parallelList} > ${parallelList}.ebcdic; rm ${parallelList}; mv ${parallelList}.ebcdic ${parallelList}"
				}
			}
			RUNTEST_CMD = "${TARGET} ${CUSTOM_OPTION}"
			for (int i = 1; i <= ITERATIONS; i++) {
				echo "ITERATION: ${i}/${ITERATIONS}"
				if (env.SPEC.startsWith('aix')) {
					sh "nohup /usr/bin/X11/X -force -vfb -x abx -x dbe -x GLX :0 &"
					env.DISPLAY = ":0"
					echo "env.DISPLAY is ${env.DISPLAY}"
					makeTest("${RUNTEST_CMD}")
				}
				else if (env.SPEC.startsWith('sunos')) {
					sh "nohup /usr/X11/bin/Xvfb :2 -screen 0 1024x768x24 &"
					env.DISPLAY = ":2"
					echo "env.DISPLAY is ${env.DISPLAY}"
					makeTest("${RUNTEST_CMD}")
				}
				else if (env.SPEC.contains('linux') && !(LABEL.contains('ci.agent.dynamic') && CLOUD_PROVIDER == 'azure')) {
					// Add an additional 10 second timeout due to issue: https://github.com/adoptium/temurin-build/issues/2368#issuecomment-756683888
					wrap([$class: 'Xvfb', autoDisplayName: true, timeout:10]) {
						def DISPLAY = sh (
							script: 'ps -f  | grep \'[X]vfb\' | awk \'{print \$9}\'',
							returnStdout: true
						).trim()
						env.DISPLAY = "${DISPLAY}"
						echo "env.DISPLAY is ${env.DISPLAY}"
						makeTest("${RUNTEST_CMD}")
					}
				}
				else {
					makeTest("${RUNTEST_CMD}")
				}
			}
		}
	}
}

def post(output_name) {
	stage('Post') {
		timestamps{
			if (env.DISPLAY != null) {
				env.DISPLAY = ""
			}
			if (output_name.contains(',')) {
				output_name = "specifiedTarget"
			}
			else {
				output_name = output_name.replace("/","_")
			}
			def tar_cmd = "tar -cf"
			// Use pigz if we can as it is faster - 2> hides fallback message
			def tar_cmd_suffix = "| (pigz -9 2>/dev/null || gzip -9)"
			def suffix = ".tar.gz"
			def pax_opt = ""
			if (SPEC.startsWith('zos')) {
				echo 'Converting tap file from ebcdic to ascii...'
				sh 'cd ./aqa-tests/TKG'
				def tapFiles = findFiles(glob: "**/*.tap")
				for (String tapFile : tapFiles) {
					sh "iconv -f ibm-1047 -t iso8859-1 ${tapFile} > ${tapFile}.ascii; rm ${tapFile}; mv ${tapFile}.ascii ${tapFile}"
				}

				tar_cmd = "pax -wf"
				suffix = ".pax.Z"
				pax_opt = "-x pax"
				tar_cmd_suffix = ""
			}

			boolean failIfNoResultsValue = true
			if (params.PARALLEL && params.PARALLEL != "None") {
				failIfNoResultsValue = false
			}
			step([$class: "TapPublisher", testResults: "**/*.tap", outputTapToConsole: false, failIfNoResults: failIfNoResultsValue])

			// only archive children TAP result in parallel mode, the file will be copied into the parent job after parallel runs
			if (params.UPSTREAM_TEST_JOB_NAME && params.UPSTREAM_TEST_JOB_NUMBER) {
				archiveArtifacts artifacts: "**/*.tap", fingerprint: true, allowEmptyArchive: true
			}

			junit allowEmptyResults: true, keepLongStdio: true, testResults: '**/work/**/*.jtr.xml, **/junitreports/**/*.xml, **/external_test_reports/**/*.xml'
			archiveSHAFile()

			if (env.BUILD_LIST.startsWith('jck')) {
				xunit (
				tools: [Custom(customXSL: "$WORKSPACE/aqa-tests/jck/xUnit.xsl",
					deleteOutputFiles: true,
					failIfNotNew: true,
					pattern: "**/TKG/output_*/**/report.xml",
					skipNoTestFiles: true,
					stopProcessingIfError: true)]
				)
			}

			//for performance test, archive regardless the build result
			if (env.BUILD_LIST.startsWith('perf')) {
				def benchmark_output_tar_name = "benchmark_test_output${suffix}"
				sh "${tar_cmd} ${benchmark_output_tar_name} ${pax_opt} ./aqa-tests/TKG/output_*"
				if (!params.ARTIFACTORY_SERVER) {
					echo "ARTIFACTORY_SERVER is not set. Saving artifacts on jenkins."
					archiveArtifacts artifacts: benchmark_output_tar_name, fingerprint: true, allowEmptyArchive: true
				} else {
					def pattern = "${env.WORKSPACE}/*_output.*"
					uploadToArtifactory(pattern)
				}
			} else if ((currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE' || currentBuild.result == 'ABORTED') || params.ARCHIVE_TEST_RESULTS) {
				def test_output_tar_name = "${output_name}_test_output${suffix}"
				if (tar_cmd.startsWith('tar')) {
					sh "${tar_cmd} - ${pax_opt} ./aqa-tests/TKG/output_* ${tar_cmd_suffix} > ${test_output_tar_name}"
				} else {
					sh "${tar_cmd} ${test_output_tar_name} ${pax_opt} ./aqa-tests/TKG/output_* ${tar_cmd_suffix}"
				}

				if (!params.ARTIFACTORY_SERVER) {
					echo "ARTIFACTORY_SERVER is not set. Saving artifacts on jenkins."
					archiveArtifacts artifacts: test_output_tar_name, fingerprint: true, allowEmptyArchive: true
				} else {
					def pattern = "${env.WORKSPACE}/*_output.*"
					uploadToArtifactory(pattern)
				}
			}
		}
	}
}

def testBuild() {
	TIME_LIMIT =  params.TIME_LIMIT ? params.TIME_LIMIT.toInteger() : 10
	timeout(time: TIME_LIMIT, unit: 'HOURS') {
		try {
			addJobDescription()
			setup()
			addGrinderLink()
			// prepare environment and compile test projects
			if( params.PARALLEL && params.PARALLEL != "None" ) {
				setupParallelEnv()
			} else {
				try {
					//ToDo: temporary workaround for jck test parallel runs
					// until build.xml is added into each subfolder
					if( env.BUILD_LIST.startsWith('jck/')) {
						def temp = env.BUILD_LIST
						env.BUILD_LIST = "jck"
						buildTest()
						env.BUILD_LIST = temp
					} else {
						buildTest()
					}
					runTest()
				} finally {
					post("${env.BUILD_LIST}")
				}
			}
		} finally {
			// Terminate any left over java processes
			terminateJavaProcesses()

			if (!params.KEEP_WORKSPACE) {
				// cleanWs() does not work in some cases, so set opts below
				cleanWs notFailBuild: true, disableDeferredWipeout: true, deleteDirs: true
				// clean up remaining core files
				if (PLATFORM.contains("mac")) {
					sh "find /cores -name '*core*' -print 2>/dev/null -exec rm -f {} \\; || true"
				} else if (PLATFORM.contains("windows")) {
					sh "find /cygdrive/c/temp -name '*core*' -print 2>/dev/null -exec rm -f {} \\; || true"
				} else {
					sh "find /tmp -name '*core*' -print 2>/dev/null -exec rm -f {} \\; || true"
				}
			}
		}
	}
}

def terminateJavaProcesses() {
	// Attempt up to 30 times or until no more Java processes spawn...
	for(int i=0; i<30; i++) {
		echo 'terminateJavaProcesses iteration: '+(i+1)

		// Terminate Java processes
		if (PLATFORM.contains("windows")) {
			// Windows code based on https://github.com/adoptium/infrastructure/issues/1669#issuecomment-727863096
			echo "PROCESSCATCH: Showing java processes - one will be the jenkins agent - removing any of these created since ${env.JOBSTARTTIME}:"
			sh(script:'powershell -c "Get-Process java | select id,processname,starttime"', returnStatus:true)

			def C=sh(script:'powershell -c "(Get-Process java | select id,processname,starttime | where {\\$_.StartTime -gt (Get-Date -Date \\"'+"${env.JOBSTARTTIME}"+'\\")} | measure).count" | tr -d \\\\r', returnStdout:true)
			echo "Number of java processes = ${C}"
			if (!C.trim().equals("0")) {
				echo "PROCESSCATCH: Attempting to stop the following processes:"
				sh(script:'powershell -c "Get-Process java | select id,processname,starttime | where {\\$_.StartTime -gt (Get-Date -Date \\"'+"${env.JOBSTARTTIME}"+'\\")}"', returnStatus:true)
				def sh_rc=sh(script:'powershell -c "Get-Process java | select id,processname,starttime | where {\\$_.StartTime -gt (Get-Date -Date \\"'+"${env.JOBSTARTTIME}"+'\\")} | stop-process"', returnStatus:true)
				echo "stop-process rc = ${sh_rc}"
			} else {
				// No more spawned, we are done...
				break
						}
		} else {
			// Non-Windows code from https://ci.adoptopenjdk.net/job/SXA-processCheck
			echo "PROCESSCATCH: Checking for any leftover java processes on the machine"
			def sh_rc=sh(script:'ps -fu '+"${env.USER}"+' | grep java | egrep -v "remoting.jar|agent.jar|grep"', returnStatus: true)
			if (sh_rc == 0) {
				echo "PROCESSCATCH: There are rogue processes still on the machine as per the list above"
				// TODO: We could kill them, but SXA-processCheck covers this in the adoptopenjdk jenkins
				// And this is likely to be replaced as part of https://github.com/adoptium/TKG/issues/45
			}
			break
		}

		// Pause between iterations in case of spawning processes
		sleep(2)
		}

		echo "PROCESSCATCH: Removed processes - here is what is remaining:"
		if (PLATFORM.contains("windows")) {
		sh(script:'powershell -c "Get-Process java | select id,processname,starttime"', returnStatus:true)
		} else {
		sh(script:'ps -fu '+"${env.USER}"+' | grep java | egrep -v "remoting.jar|agent.jar|grep"', returnStatus:true)
		}
}

def getJDKImpl(jvm_version) {
	def jdk_impl = 'hotspot'
	if (jvm_version.contains('openj9')) {
		jdk_impl = 'openj9'
	} else if (JVM_VERSION.contains('sap')) {
		jdk_impl = 'sap'
	}
	return jdk_impl
}

def addJobDescription() {
	if (params.PERSONAL_BUILD) {
		// update build name if personal build
		wrap([$class: 'BuildUser']) {
			currentBuild.displayName = "#${BUILD_NUMBER} - ${BUILD_USER_EMAIL}"
		}
	}
	def description = (currentBuild.description) ? currentBuild.description + "<br>" : ""
	currentBuild.description = description \
		+ "TARGET: ${params.TARGET}<br/>" \
		+ "LABEL: <a href=${JENKINS_URL}label/${LABEL}>${LABEL}</a><br/>" \
		+ "<a href=${JENKINS_URL}computer/${NODE_NAME}>${NODE_NAME}</a>"
}

def getJenkinsDomain() {
	String regex = env.JENKINS_URL
	def m = regex =~ /:\/\/(.*)\/?/
	def match = m[0][1]
	def domainName = "undefined"
	if (match) {
		domainName = match
	}
	return domainName
}

def archiveSHAFile() {
	def shaFile = "aqa-tests/TKG/SHA.txt";
	if (!params.ARTIFACTORY_SERVER) {
		echo "ARTIFACTORY_SERVER is not set. Saving SHA file on jenkins."
		archiveArtifacts artifacts: shaFile, fingerprint: true, allowEmptyArchive: true
	} else {
		def pattern = "${env.WORKSPACE}/${shaFile}"
		uploadToArtifactory(pattern)
	}
}

def uploadToArtifactory(pattern) {
	if (params.ARTIFACTORY_SERVER) {
		def server = Artifactory.server params.ARTIFACTORY_SERVER
		def artifactoryRepo = params.ARTIFACTORY_REPO ? params.ARTIFACTORY_REPO : "sys-rt-generic-local"
		def artifactoryRoorDir = params.ARTIFACTORY_ROOT_DIR ? params.ARTIFACTORY_ROOT_DIR : getJenkinsDomain()
		def artifactoryUploadDir = "${artifactoryRepo}/${artifactoryRoorDir}/${JOB_NAME}/${BUILD_ID}/"

		def uploadSpec = """{
			"files":[
					{
						"pattern": "${pattern}",
						"target": "${artifactoryUploadDir}"
					}
				]
			}"""

		def buildInfo = Artifactory.newBuildInfo()
		// if ARTIFACTORY_NUM_ARTIFACTS is not set, set to keep 20 build artifacts
		def numArtifacts = params.ARTIFACTORY_NUM_ARTIFACTS ? params.ARTIFACTORY_NUM_ARTIFACTS : 20

		buildInfo.retention maxBuilds: numArtifacts, deleteBuildArtifacts: true
		server.upload spec: uploadSpec, buildInfo: buildInfo
		server.publishBuildInfo buildInfo

		def artifactoryUrl = server.getUrl()
		def uploadUrl = "${artifactoryUrl}/${artifactoryUploadDir}"
		echo "Test output artifactory URL:'${uploadUrl}'"
		if (!currentBuild.description.contains(uploadUrl)) {
			currentBuild.description += "<br><a href=${uploadUrl}>Artifacts</a>"
		}
	} else {
		echo "ARTIFACTORY_SERVER is not set. Artifacts are not uploaded onto artifactory server."
	}
}

def addGrinderLink() {
	String url = "${HUDSON_URL}job/Grinder/parambuild/?"
	int i = 1;
	def labelValue = ""
	params.each { key, value ->
		value = URLEncoder.encode(value.toString(), "UTF-8")
		url += "${key}=${value}"
		if (i != params.size()) {
			url += "&amp;"
		}
		i++;
		if ( key == "LABEL" ) {
			labelValue = "LABEL=${value}"
		}
	}
	echo "Rerun in Grinder: ${url}"
	currentBuild.description += "<br><a href=${url}>Rerun in Grinder</a> Change TARGET to run only the failed test targets."
	def sameMachineUrl = url.replace(labelValue,"LABEL=${NODE_NAME}")
	echo "Rerun in Grinder on same machine: ${sameMachineUrl}"
	currentBuild.description += "<br><a href=${sameMachineUrl}>Rerun in Grinder on same machine</a> Change TARGET to run only the failed test targets. For dynamic vm agents as they are created at runtime and quickly recycled, it may not be possible to rerun on the same agent."
	currentBuild.description += "<br><a href=\"https://github.com/adoptium/aqa-tests/wiki/How-to-Run-a-Grinder-Build-on-Jenkins\">Grinder Wiki</a>"
}

def run_parallel_tests() {
	if (params.PARALLEL && params.PARALLEL != "None") {
		stage ("Parallel Tests") {
			def childJobs = parallel parallel_tests
			node {
				def buildResult = ""
				childJobs.each {
					cjob ->
					def jobInvocation = cjob.value.getRawBuild()
					def buildId = jobInvocation.getNumber()
					def name = cjob.value.getProjectName()
					def childResult = cjob.value.getCurrentResult()
					try {
						echo "${name} #${buildId} completed with status ${childResult}"
						timeout(time: 1, unit: 'HOURS') {
							copyArtifacts (projectName: "${name}", selector: specific("${buildId}"), filter: "**/TestTargetResult.tap", target:"${name}/${buildId}")
						}
						step([$class: "TapPublisher", testResults: "${name}/${buildId}/**/TestTargetResult.tap", outputTapToConsole: false, failIfNoResults: false])
					} catch (Exception e) {
						echo "Cannot copy TestTargetResult.tap from ${name} with buildid ${buildId} . Skipping copyArtifacts..."
						buildResult = childResult
					}
				}
				if (buildResult) {
					echo "set build status to ${buildResult}"
					currentBuild.result = buildResult
				}
			}
		}
	}
}

def getGitRepoBranch(ownerBranch, defaultOwnerBranch, repo) {
	String[] actualRepoBranch = defaultOwnerBranch.split(":")
	if (ownerBranch) {
		String[] tokens = ownerBranch.split(":")
		if (tokens.size() == 2 ) {
			actualRepoBranch = tokens;
		}
	}

	String owner = actualRepoBranch[0].trim()
	String repoURL = "https://github.com/${owner}/${repo}.git"
	if (env.SPEC.startsWith('zos')) {
		repoURL = repoURL.replace("https://github.com/","git@github.com:")
	}
	actualRepoBranch[0] = repoURL
	actualRepoBranch[1] = actualRepoBranch[1].trim()
	return actualRepoBranch
}

return this