Add --component support for jenkins build pipelines (From PR [2095]) (#…

…2100) * Added tests Signed-off-by: pgodithi <[email protected]> * Add tests without component parameter and for rpm distribution. Signed-off-by: Zelin Hao <[email protected]> Co-authored-by: Zelin Hao <[email protected]>
opensearch-project · May 10, 2022 · fd028ed · fd028ed
1 parent c8a9b1f
commit fd028ed
Show file tree

Hide file tree

Showing 7 changed files with 104 additions and 7 deletions.
diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile
@@ -12,6 +12,11 @@ pipeline {
         AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
     }
     parameters {
+        string(
+            name: 'COMPONENT_NAME',
+            description: 'If this field contains a component name (e.g. OpenSearch-Dashboards), will build with "--component <COMPONENT_NAME>", else build everything in the INPUT_MANIFEST.',
+            trim: true
+        )
         string(
             name: 'INPUT_MANIFEST',
             description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-dashboards-2.0.0.yml.',
@@ -49,8 +54,8 @@ pipeline {
         stage('detect docker image + args') {
             agent {
                 docker {
-                    label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
+                    label AGENT_X64
+                    image 'alpine:3'
                     alwaysPull true
                 }
             }
@@ -75,6 +80,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 platform: 'linux',
                                 architecture: 'x64',
@@ -161,6 +167,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 platform: 'linux',
                                 architecture: 'x64',
@@ -221,6 +228,7 @@ pipeline {
                             steps {
                                 script {
                                     buildArchive(
+                                        componentName: "${COMPONENT_NAME}",
                                         inputManifest: "manifests/${INPUT_MANIFEST}",
                                         platform: 'linux',
                                         architecture: 'arm64',
@@ -246,6 +254,7 @@ pipeline {
                             steps {
                                 script {
                                     def buildManifestObj = archiveAssembleUpload(
+                                        componentName: "${COMPONENT_NAME}",
                                         inputManifest: "manifests/${INPUT_MANIFEST}",
                                         platform: 'linux',
                                         architecture: 'arm64',
@@ -336,6 +345,7 @@ pipeline {
                             steps {
                                 script {
                                     buildArchive(
+                                        componentName: "${COMPONENT_NAME}",
                                         inputManifest: "manifests/${INPUT_MANIFEST}",
                                         platform: 'linux',
                                         architecture: 'arm64',
@@ -361,6 +371,7 @@ pipeline {
                             steps {
                                 script {
                                     def buildManifestObj = archiveAssembleUpload(
+                                        componentName: "${COMPONENT_NAME}",
                                         inputManifest: "manifests/${INPUT_MANIFEST}",
                                         platform: 'linux',
                                         architecture: 'arm64',
@@ -498,3 +509,4 @@ pipeline {
         }
     }
 }
+
diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile
@@ -7,6 +7,11 @@ pipeline {
         AGENT_ARM64 = 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
     }
     parameters {
+        string(
+            name: 'COMPONENT_NAME',
+            description: 'If this field contains a component name (e.g. OpenSearch), will build with "--component <COMPONENT_NAME>", else build everything in the INPUT_MANIFEST.',
+            trim: true
+        )
         string(
             name: 'INPUT_MANIFEST',
             description: 'Input manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0.yml.',
@@ -44,8 +49,8 @@ pipeline {
         stage('detect docker image + args') {
             agent {
                 docker {
-                    label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-                    image 'opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028'
+                    label AGENT_X64
+                    image 'alpine:3'
                     alwaysPull true
                 }
             }
@@ -64,7 +69,7 @@ pipeline {
                     }
                     agent {
                         docker {
-                            label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
+                            label AGENT_X64
                             image dockerAgent.image
                             args dockerAgent.args
                             alwaysPull true
@@ -73,6 +78,7 @@ pipeline {
                     steps {
                         script {
                             buildManifest(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: 'tar',
                                 snapshot: true
@@ -103,7 +109,7 @@ pipeline {
                 stage('build-snapshot-arm64-tar') {
                     agent {
                         docker {
-                            label 'Jenkins-Agent-al2-arm64-c6g4xlarge-Docker-Host'
+                            label AGENT_ARM64
                             image dockerAgent.image
                             args dockerAgent.args
                             alwaysPull true
@@ -112,6 +118,7 @@ pipeline {
                     steps {
                         script {
                             buildManifest(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: 'tar',
                                 snapshot: true
@@ -141,6 +148,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: "tar"
                             )
@@ -225,6 +233,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: "rpm"
                             )
@@ -280,6 +289,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: "tar"
                             )
@@ -363,6 +373,7 @@ pipeline {
                     steps {
                         script {
                             def buildManifestObj = buildAssembleUpload(
+                                componentName: "${COMPONENT_NAME}",
                                 inputManifest: "manifests/${INPUT_MANIFEST}",
                                 distribution: "rpm"
                             )
@@ -497,3 +508,4 @@ pipeline {
         }
     }
 }
+
diff --git a/tests/jenkins/TestbuildManifestVar.groovy b/tests/jenkins/TestbuildManifestVar.groovy
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package jenkins.tests
+
+import org.junit.*
+
+class TestbuildManifestVar extends BuildPipelineTest {
+
+    @Test
+    void testbuildManifest() {
+        super.testPipeline("tests/jenkins/jobs/BuildShManifest_Jenkinsfile")
+    }
+}
+
diff --git a/tests/jenkins/data/opensearch-2.0.0.yml b/tests/jenkins/data/opensearch-2.0.0.yml
@@ -6,7 +6,7 @@ build:
   qualifier: alpha1
 ci:
   image:
-    name: opensearchstaging/ci-runner:ci-runner-centos7-v1
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
     args: -e JAVA_HOME=/opt/java/openjdk-17
 components:
   - name: OpenSearch
@@ -15,3 +15,15 @@ components:
     checks:
       - gradle:publish
       - gradle:properties:version
+  - name: common-utils
+    repository: https://github.com/opensearch-project/common-utils.git
+    ref: '2.0'
+    checks:
+      - gradle:publish
+      - gradle:properties:version
+  - name: job-scheduler
+    repository: https://github.com/opensearch-project/job-scheduler.git
+    ref: '2.0'
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
diff --git a/tests/jenkins/jobs/BuildShManifest_Jenkinsfile b/tests/jenkins/jobs/BuildShManifest_Jenkinsfile
@@ -0,0 +1,29 @@
+pipeline {
+    agent none
+    stages {
+        stage('Test build manifest jenkins var') {
+            steps {
+                script {
+                    buildManifest(
+                        inputManifest: "tests/jenkins/data/opensearch-2.0.0.yml",
+                        distribution: "tar",
+                        snapshot: true
+                    )
+                    buildManifest(
+                        componentName: "job-scheduler",
+                        inputManifest: "tests/jenkins/data/opensearch-2.0.0.yml",
+                        distribution: "tar",
+                        snapshot: true
+                    )
+                    buildManifest(
+                        componentName: "common-utils",
+                        inputManifest: "tests/jenkins/data/opensearch-2.0.0.yml",
+                        distribution: "rpm",
+                        snapshot: true
+                    )
+                }
+            }
+        }
+    }
+}
+
diff --git a/tests/jenkins/jobs/BuildShManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BuildShManifest_Jenkinsfile.txt
@@ -0,0 +1,11 @@
+   BuildShManifest_Jenkinsfile.run()
+      BuildShManifest_Jenkinsfile.pipeline(groovy.lang.Closure)
+         BuildShManifest_Jenkinsfile.echo(Executing on agent [label:none])
+         BuildShManifest_Jenkinsfile.stage(Test build manifest jenkins var, groovy.lang.Closure)
+            BuildShManifest_Jenkinsfile.script(groovy.lang.Closure)
+               BuildShManifest_Jenkinsfile.buildManifest({inputManifest=tests/jenkins/data/opensearch-2.0.0.yml, distribution=tar, snapshot=true})
+                  buildManifest.sh(./build.sh tests/jenkins/data/opensearch-2.0.0.yml -d tar --snapshot)
+               BuildShManifest_Jenkinsfile.buildManifest({componentName=job-scheduler, inputManifest=tests/jenkins/data/opensearch-2.0.0.yml, distribution=tar, snapshot=true})
+                  buildManifest.sh(./build.sh tests/jenkins/data/opensearch-2.0.0.yml -d tar --component job-scheduler --snapshot)
+               BuildShManifest_Jenkinsfile.buildManifest({componentName=common-utils, inputManifest=tests/jenkins/data/opensearch-2.0.0.yml, distribution=rpm, snapshot=true})
+                  buildManifest.sh(./build.sh tests/jenkins/data/opensearch-2.0.0.yml -d rpm --component common-utils --snapshot)
diff --git a/vars/buildManifest.groovy b/vars/buildManifest.groovy
@@ -3,6 +3,7 @@ void call(Map args = [:]) {
         './build.sh',
         args.inputManifest ?: "manifests/${INPUT_MANIFEST}",
         args.distribution ? "-d ${args.distribution}" : null,
+        args.componentName ? "--component ${args.componentName}" : null,
         args.platform ? "-p ${args.platform}" : null,
         args.architecture ? "-a ${args.architecture}" : null,
         args.snapshot ? '--snapshot' : null,