Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jenkins Job to run OSDInteg in Parallel #3465

Merged
merged 6 commits into from
May 4, 2023
Merged

Jenkins Job to run OSDInteg in Parallel #3465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7dfd266
Add jenkins file
Divyaasm Apr 12, 2023
964902f
Jenkins job for OSD Integtests
Divyaasm May 3, 2023
4169426
Update jenkinsfile.txt
Divyaasm May 3, 2023
f69b1f0
Update test cases
Divyaasm May 3, 2023
c5270eb
Update nits and suggestions
Divyaasm May 3, 2023
df9c68b
Update comment info
Divyaasm May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 83 additions & 26 deletions jenkins/opensearch-dashboards/integ-test.jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,19 @@ pipeline {
ARTIFACT_BUCKET_NAME = credentials('jenkins-artifact-bucket-name')
}
parameters {
string(
name: 'COMPONENT_NAME',
description: 'If this field contains one or more component names (e.g. notificationsDashboards indexManagementDashboards ...) separated by space, will test with "--component ...", else test everything in the TEST_MANIFEST..',
trim: true
)
string(
name: 'TEST_MANIFEST',
description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0-test.yml.',
trim: true
)
string(
name: 'BUILD_MANIFEST_URL',
description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.5.0/5367/linux/x64/tar/builds/opensearch-dashboards/manifest.yml',
description: 'The build manifest URL for OpenSearch Dashboards, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/2.5.0/5367/linux/x64/tar/builds/opensearch-dashboards/manifest.yml.',
trim: true
)
string(
Expand Down Expand Up @@ -98,29 +103,38 @@ pipeline {
currentBuild.result = 'ABORTED'
error("OSD Version $version does not match OS Version $versionOpenSearch")
}

}
}
}
stage('integ-test') {
agent {
docker {
label AGENT_LABEL
image docker_images["$distribution"]
args docker_args["$distribution"]
registryUrl 'https://public.ecr.aws/'
alwaysPull true
post {
always {
postCleanup()
}
}
}
stage('integ-test') {
// Need to run this directly on agent node here in order to trigger stages with docker container and avoid docker within docker situation
// Can only be run in runner that is at least 50GB per container
agent { label AGENT_LABEL }
steps {
script {
def buildManifestObj = downloadBuildManifest(

downloadBuildManifest(
url: BUILD_MANIFEST_URL,
path: BUILD_MANIFEST
)

def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
def componentDefaultList = buildManifestObj.getNames()
def componentList = COMPONENT_NAME ? COMPONENT_NAME.trim().split(" ") as List : componentDefaultList
String switch_user_non_root = (distribution.equals('rpm') || distribution.equals('deb')) ? 'true' : 'false'
echo "switch_user_non_root: ${switch_user_non_root}"
echo "componentList: ${componentList}"

for (component_check in componentList) {
if (!componentDefaultList.contains(component_check)) {
error("${component_check} is not in build manifest: ${componentDefaultList}, exit 1")
}
}

echo "Downloading from S3: ${artifactPathOpenSearch}"
downloadFromS3(
Expand All @@ -144,24 +158,67 @@ pipeline {
)
sh("cp -a $WORKSPACE/artifacts/${artifactPath} $WORKSPACE")

runIntegTestScript(
jobName: "$BUILD_JOB_NAME",
componentName: 'functionalTestDashboards',
buildManifest: "$BUILD_MANIFEST",
testManifest: "manifests/${TEST_MANIFEST}",
localPath: "${WORKSPACE}/${distribution}",
switchUserNonRoot: "${switch_user_non_root}"
)
// Stash the current working directory files, aka opensearch-build repo
// Unstash later in each triggered stage to run integTest
stash includes: "**", name: "integtest-opensearch-dashboards-$BUILD_NUMBER"

componentTests = [:]

for (component in componentList) {
// Must use local variable due to groovy for loop and closure scope
// Or the stage will be fixed to the last item in return when new stages are triggered here
// https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
def local_component = component.trim()
def local_component_index = componentList.indexOf(local_component)
def wait_seconds = local_component_index * 10

echo "Adding Component: ${local_component}"
componentTests["Run Integtest ${local_component}"] = {
// Using scripted pipelines to trigger dynamic parallel stages
timeout(time: 2, unit: 'HOURS') {
node(AGENT_LABEL) {
docker.withRegistry('https://public.ecr.aws/') {
docker.image(docker_images["$distribution"]).inside(docker_args["$distribution"]) {
try {
stage("${local_component}") {
// Jenkins tend to not clean up workspace at times even though ws clean is called
// Since docker is mounted on the agent node directly so it can communicated with the agent
// This sometimes causes the workspace to retain last run test-results and ends with build failures
// https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
sh("echo ${local_component} with index ${local_component_index} will sleep ${wait_seconds} seconds to reduce load && sleep ${wait_seconds}")
unstash "integtest-opensearch-dashboards-$BUILD_NUMBER"
sh("rm -rf test-results")
runIntegTestScript(
jobName: "$BUILD_JOB_NAME",
componentName: "${local_component}",
buildManifest: "$BUILD_MANIFEST",
testManifest: "manifests/${TEST_MANIFEST}",
localPath: "${WORKSPACE}/${distribution}",
switchUserNonRoot: "${switch_user_non_root}"
)
}
} catch (e) {
echo "Error running integtest for component ${local_component}"
peterzhuamazon marked this conversation as resolved.
Show resolved Hide resolved
throw e
} finally {
echo "Completed running integtest for component ${local_component}"
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
postCleanup()
}
}
}
}
}
}
}
parallel componentTests
}
}
post {
always {
script {
uploadTestResults(
buildManifestFileName: BUILD_MANIFEST,
jobName: JOB_NAME
)
}
postCleanup()
}
}
Expand Down
53 changes: 50 additions & 3 deletions tests/jenkins/TestOpenSearchDashboardsIntegTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ import org.junit.Before
import org.junit.Test
import org.yaml.snakeyaml.Yaml
import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
import static org.hamcrest.CoreMatchers.hasItem
import static org.hamcrest.CoreMatchers.hasItems
import static org.hamcrest.MatcherAssert.assertThat
import static org.junit.jupiter.api.Assertions.assertThrows

class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {

Expand All @@ -32,8 +37,9 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
def testManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml"
def buildId = 215
def buildManifest = "tests/jenkins/data/opensearch-dashboards-1.2.0-build.yml"
def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz"
def buildManifestUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/1.2.0/${buildId}/linux/x64/tar/builds/opensearch-dashboards/opensearch-dashboards-1.2.0-linux-x64.tar.gz"
def agentLabel = "Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host"
def bucketName = 'job-s3-bucket-name'

binding.setVariable('env', ['BUILD_NUMBER': '215'])
binding.setVariable('ARTIFACT_BUCKET_NAME', 'DUMMY_BUCKET_NAME')
Expand All @@ -53,7 +59,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
binding.setVariable('BUILD_MANIFEST', buildManifest)
binding.setVariable('BUILD_ID', "${buildId}")
def env = binding.getVariable('env')
env['DOCKER_AGENT'] = [image:'opensearchstaging/ci-runner:ci-runner-centos7-v1', args:'-e JAVA_HOME=/opt/java/openjdk-11']
env['DOCKER_AGENT'] = [image:'opensearchstaging/opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2', args:'-e JAVA_HOME=/opt/java/openjdk-11']
binding.getVariable('currentBuild').upstreamBuilds = [[fullProjectName: jobName]]

helper.registerAllowedMethod("s3Download", [Map])
Expand All @@ -73,6 +79,7 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.addFileExistsMock("manifests/${testManifest}", true)
helper.registerAllowedMethod("s3Upload", [Map])
helper.registerAllowedMethod('fileExists', [String.class], { args ->
return true;
Expand All @@ -83,8 +90,48 @@ class TestOpenSearchDashboardsIntegTest extends BuildPipelineTest {
}

@Test
void integTests_runs_consistently() {
void integTests_runs_for_all_components() {
super.testPipeline('jenkins/opensearch-dashboards/integ-test.jenkinsfile',
'tests/jenkins/jenkinsjob-regression-files/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('sh', 'test.sh'), hasItems(
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component ganttChartDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component indexManagementDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component anomalyDetectionDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component securityDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component functionalTestDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component OpenSearch-Dashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component alertingDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component queryWorkbenchDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component reportsDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString(),
'env PATH=$PATH ./test.sh integ-test manifests/tests/jenkins/data/opensearch-dashboards-1.2.0-test.yml --component observabilityDashboards --test-run-id 215 --paths opensearch=/tmp/workspace/tar opensearch-dashboards=/tmp/workspace/tar '.toString()
))
}

@Test
void checkUploadResults() {
runScript('jenkins/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('s3Upload', ''), hasItem('{file=test-results, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/1.2.0/215/linux/x64/tar/test-results}'))
}

@Test
void checkIfRunningInParallel(){
runScript('jenkins/opensearch-dashboards/integ-test.jenkinsfile')
assertThat(getCommandExecutions('parallel', ''), hasItem('{Run Integtest ganttChartDashboards=groovy.lang.Closure, Run Integtest indexManagementDashboards=groovy.lang.Closure, Run Integtest anomalyDetectionDashboards=groovy.lang.Closure, Run Integtest OpenSearch-Dashboards=groovy.lang.Closure, Run Integtest securityDashboards=groovy.lang.Closure, Run Integtest functionalTestDashboards=groovy.lang.Closure, Run Integtest alertingDashboards=groovy.lang.Closure, Run Integtest queryWorkbenchDashboards=groovy.lang.Closure, Run Integtest reportsDashboards=groovy.lang.Closure, Run Integtest observabilityDashboards=groovy.lang.Closure}'))
}

def getCommandExecutions(methodName, command) {
def shCommands = helper.callStack.findAll {
call ->
call.methodName == methodName
}.
collect {
call ->
callArgsToString(call)
}.findAll {
shCommand ->
shCommand.contains(command)
}

return shCommands
}
}
Loading