Add gradle check jenkins workflow on public jenkins (#2249)

* Add gradle check jenkins workflow on public jenkins Signed-off-by: Peter Zhu <[email protected]> * Tweak jenkins pipeline Signed-off-by: Peter Zhu <[email protected]> * Remove bot token Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Jun 24, 2022 · 4b9ca32 · 4b9ca32
1 parent 7ff5f5b
commit 4b9ca32
Show file tree

Hide file tree

Showing 8 changed files with 372 additions and 0 deletions.
diff --git a/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile b/docker/ci/dockerfiles/current/docker-builder.ubuntu2004.x64.dockerfile
@@ -11,6 +11,8 @@
 # This can only be used on Ubuntu 2004 X64 version, as QEMU 5.0 is required to get buildx work properly without segfault
 # https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1928075
 
+# This image can be used with these arguments: -u root -v /var/run/docker.sock:/var/run/docker.sock
+
 FROM ubuntu:20.04
 
 # Import necessary repository for installing qemu 5.0

diff --git a/docker/ci/dockerfiles/legacy/test.centos7.gradle-check.x64.dockerfile b/docker/ci/dockerfiles/legacy/test.centos7.gradle-check.x64.dockerfile
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+# This is a docker image specifically for running OpenSearch core gradle check
+# It should be used on a Jenkins Agent node with only 1 executor, 16xCPU + 16/32GB Memory
+
+# This image can be used with these arguments: 
+# '-u root -v /var/run/docker.sock:/var/run/docker.sock -e JAVA_HOME=/opt/java/openjdk-17 -e JAVA11_HOME=/opt/java/openjdk-11 -e JAVA8_HOME=/opt/java/openjdk-8'
+
+
+FROM centos:7
+
+ARG MAVEN_DIR=/usr/local/apache-maven
+
+# Ensure localedef running correct with root permission
+USER 0
+
+# Setup ENV to prevent ASCII data issues with Python3
+RUN echo "export LC_ALL=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
+    echo "export LANG=en_US.utf-8" >> /etc/profile.d/python3_ascii.sh && \
+    localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || echo set locale
+
+# Add normal dependencies
+RUN yum clean all && \
+    yum update -y && \
+    yum install -y which curl git gnupg2 tar net-tools procps-ng python3 python3-devel python3-pip docker zip unzip jq
+
+# Create user group
+RUN groupadd -g 1000 opensearch && \
+    useradd -u 1000 -g 1000 -d /usr/share/opensearch opensearch && \
+    mkdir -p /usr/share/opensearch && \
+    chown -R 1000:1000 /usr/share/opensearch
+
+#JDK setup
+COPY --chown=0:0 config/jdk-setup.sh /tmp
+RUN /tmp/jdk-setup.sh
+
+# Install higher version of maven 3.8.x
+RUN export MAVEN_URL=`curl -s https://maven.apache.org/download.cgi | grep -Eo '["\047].*.bin.tar.gz["\047]' | tr -d '"'`  && \
+    mkdir -p $MAVEN_DIR && (curl -s $MAVEN_URL | tar xzf - --strip-components=1 -C $MAVEN_DIR) && \
+    echo "export M2_HOME=$MAVEN_DIR" > /etc/profile.d/maven_path.sh && \
+    echo "export M2=\$M2_HOME/bin" >> /etc/profile.d/maven_path.sh && \
+    echo "export PATH=\$M2:\$PATH" >> /etc/profile.d/maven_path.sh && \
+    ln -sfn $MAVEN_DIR/bin/mvn /usr/local/bin/mvn
+
+# Change User
+USER 1000
+WORKDIR /usr/share/opensearch
+
diff --git a/jenkins/gradle/gradle-check.jenkinsfile b/jenkins/gradle/gradle-check.jenkinsfile
@@ -0,0 +1,123 @@
+lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
+
+pipeline {
+    options {
+            timeout(time: 2, unit: 'HOURS')
+    }
+    // gradle check have a lot of issues running on containers
+    // Therefore, we directly run it on the agent node
+    agent {
+        node {
+            // Must use Ubuntu agent with 1 executor or gradle check will show a lot of java-related errors
+            label 'Jenkins-Agent-Ubuntu2004-X64-c54xlarge-Single-Host'
+        }
+    }
+    parameters {
+        string(
+            name: 'GIT_REPO_URL',
+            description: 'OpenSearch core repository url on git, can be either the official upstream url or your fork url.',
+            defaultValue: 'https://github.com/opensearch-project/OpenSearch.git',
+            trim: true
+        )
+        string(
+            name: 'GIT_REFERENCE',
+            description: 'Git branch, tag, commitid for reference to checkout commit of OpenSearch core before running the gradle check.',
+            defaultValue: 'main',
+            trim: true
+        )
+    }
+    triggers {
+        GenericTrigger(
+            genericVariables: [
+                [key: 'pr_from_sha', value: '$.pr_from_sha'],
+                [key: 'pr_from_clone_url', value: '$.pr_from_clone_url'],
+                [key: 'pr_to_clone_url', value: '$.pr_to_clone_url'],
+                [key: 'pr_title', value: '$.pr_title'],
+                [key: 'pr_number', value: '$.pr_number']
+            ],
+            tokenCredentialId: 'jenkins-gradle-check-generic-webhook-token',
+            causeString: 'Triggered by PR on OpenSearch core repository',
+            printContributedVariables: false,
+            printPostContent: false
+        )
+    }
+    environment {
+        JAVA11_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-11"
+        JAVA17_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-17"
+        JAVA8_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-8"
+        JAVA_HOME="/var/jenkins/tools/hudson.model.JDK/jdk-17"
+        USER_BUILD_CAUSE = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')
+        TIMER_BUILD_CAUSE = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')
+    }
+    stages {
+        stage('Install jdk-8') {
+            tools {
+                jdk 'jdk-8'
+            }
+            steps {
+                echo "Install jdk-8"
+                sh("ls ${JAVA8_HOME}/.. | grep jdk-8")
+            }
+        }
+        stage('Install jdk-11') {
+           tools {
+               jdk 'jdk-11'
+           }
+            steps {
+                echo "Install jdk-11"
+                sh("ls ${JAVA11_HOME}/.. | grep jdk-11")
+            }
+        }
+        stage('Install jdk-17') {
+           tools {
+               jdk 'jdk-17'
+           }
+            steps {
+                echo "Install jdk-17"
+                sh("ls ${JAVA17_HOME}/.. | grep jdk-17")
+            }
+        }
+        stage('Run Gradle Check') {
+            steps {
+                script {
+                    sh("ls /var/jenkins/tools/hudson.model.JDK/ && env | grep JAVA")
+                    if (env.USER_BUILD_CAUSE.equals('[]') && env.TIMER_BUILD_CAUSE.equals('[]')) {
+                        def pr_url = "${pr_to_clone_url}".replace(".git", "/pull/${pr_number}")
+                        println("Triggered by PR: ${pr_url}")
+                        currentBuild.description = """<a href="${pr_url}">PR #${pr_number}</a>: ${pr_title}"""
+
+                        runGradleCheck(
+                            gitRepoUrl: "${pr_from_clone_url}",
+                            gitReference: "${pr_from_sha}"
+                        )
+                    }
+                    else {
+                        println("Triggered by User or Triggered by Timer")
+                        def repo_url = "${GIT_REPO_URL}".replace(".git", "/commit")
+                        currentBuild.description = """git: <a href="${GIT_REPO_URL}">${GIT_REPO_URL}</a><br>ref: <a href="${repo_url}/${GIT_REFERENCE}">${GIT_REFERENCE}</a>"""
+
+                        runGradleCheck(
+                            gitRepoUrl: "${GIT_REPO_URL}",
+                            gitReference: "${GIT_REFERENCE}"
+                        )
+                    }
+                }
+            }
+            post() {
+                always {
+                    script {
+                        sh("rm -rf *")
+                        postCleanup()
+                    }
+                }
+            }
+        }
+    }
+    post() {
+        always {
+            script {
+                postCleanup()
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/TestRunGradleCheck.groovy b/tests/jenkins/TestRunGradleCheck.groovy
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+
+class TestRunGradleCheck extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        this.registerLibTester(new RunGradleCheckLibTester(
+            'https://github.com/opensearch-project/OpenSearch',
+            'main',
+            )
+        )
+        super.setUp()
+    }
+
+    @Test
+    void testRunGradleCheck() {
+        super.testPipeline("tests/jenkins/jobs/RunGradleCheck_Jenkinsfile")
+    }
+}
diff --git a/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile b/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+pipeline {
+    agent none
+    stages {
+        stage('gradle-check') {
+            steps {
+                script {
+                    runGradleCheck(
+                        gitRepoUrl: "https://github.com/opensearch-project/OpenSearch",
+                        gitReference: "main"
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt b/tests/jenkins/jobs/RunGradleCheck_Jenkinsfile.txt
@@ -0,0 +1,49 @@
+   RunGradleCheck_Jenkinsfile.run()
+      RunGradleCheck_Jenkinsfile.pipeline(groovy.lang.Closure)
+         RunGradleCheck_Jenkinsfile.echo(Executing on agent [label:none])
+         RunGradleCheck_Jenkinsfile.stage(gradle-check, groovy.lang.Closure)
+            RunGradleCheck_Jenkinsfile.script(groovy.lang.Closure)
+               RunGradleCheck_Jenkinsfile.runGradleCheck({gitRepoUrl=https://github.com/opensearch-project/OpenSearch, gitReference=main})
+                  runGradleCheck.legacySCM(groovy.lang.Closure)
+                  runGradleCheck.library({identifier=jenkins@20211123, retriever=null})
+                  runGradleCheck.usernamePassword({credentialsId=jenkins-gradle-check-s3-aws-credentials, usernameVariable=amazon_s3_access_key, passwordVariable=amazon_s3_secret_key})
+                  runGradleCheck.usernamePassword({credentialsId=jenkins-gradle-check-s3-aws-resources, usernameVariable=amazon_s3_base_path, passwordVariable=amazon_s3_bucket})
+                  runGradleCheck.withCredentials([[amazon_s3_access_key, amazon_s3_secret_key], [amazon_s3_base_path, amazon_s3_bucket]], groovy.lang.Closure)
+                     runGradleCheck.sh(
+
+                set -e
+                set +x
+
+                env | grep JAVA | grep HOME
+
+                echo "Git clone: https://github.com/opensearch-project/OpenSearch with ref: main"
+                rm -rf search
+                git clone https://github.com/opensearch-project/OpenSearch search
+                cd search/
+                git checkout -f main
+                git rev-parse HEAD
+
+                echo "Stop existing gradledaemon"
+                ./gradlew --stop
+                find ~/.gradle -type f -name "*.lock" -delete
+
+                echo "Check existing dockercontainer"
+                docker ps -a
+                docker stop `docker ps -qa` > /dev/null 2>&1 || echo
+                docker rm `docker ps -qa` > /dev/null 2>&1 || echo
+                echo "Stop existing dockercontainer"
+                docker ps -a
+
+                echo "Check docker-compose version"
+                docker-compose version
+
+                echo "Start gradlecheck"
+                GRADLE_CHECK_STATUS=0
+                ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1
+
+                if [ "$GRADLE_CHECK_STATUS" != 0 ]; then
+                    echo Gradle Check Failed!
+                    exit 1
+                fi
+
+            )
diff --git a/tests/jenkins/lib-testers/RunGradleCheckLibTester.groovy b/tests/jenkins/lib-testers/RunGradleCheckLibTester.groovy
@@ -0,0 +1,32 @@
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+
+
+class RunGradleCheckLibTester extends LibFunctionTester {
+
+    private String gitRepoUrl
+    private String gitReference
+
+    public RunGradleCheckLibTester(gitRepoUrl, gitReference){
+        this.gitRepoUrl = gitRepoUrl
+        this.gitReference = gitReference
+    }
+
+    void configure(helper, binding) {
+        // N/A
+    }
+
+    void parameterInvariantsAssertions(call) {
+        assertThat(call.args.gitRepoUrl.first(), notNullValue())
+        assertThat(call.args.gitReference.first(), notNullValue())
+    }
+
+    boolean expectedParametersMatcher(call) {
+        return call.args.gitRepoUrl.first().toString().equals(this.gitRepoUrl)
+                && call.args.gitReference.first().toString().equals(this.gitReference)
+    }
+
+    String libFunctionName() {
+        return 'runGradleCheck'
+    }
+}
diff --git a/vars/runGradleCheck.groovy b/vars/runGradleCheck.groovy
@@ -0,0 +1,61 @@
+void call(Map args = [:]) {
+    def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
+    def git_repo_url = args.gitRepoUrl ?: 'null'
+    def git_reference = args.gitReference ?: 'null'
+
+    println("Git Repo: ${git_repo_url}")
+    println("Git Reference: ${git_reference}")
+
+    if (git_repo_url.equals('null') || git_reference.equals('null')) {
+        println("No git repo url or git reference to checkout the commit, exit 1")
+        System.exit(1)
+    }
+    else {
+        withCredentials([
+            usernamePassword(credentialsId: "jenkins-gradle-check-s3-aws-credentials", usernameVariable: 'amazon_s3_access_key', passwordVariable: 'amazon_s3_secret_key'),
+            usernamePassword(credentialsId: "jenkins-gradle-check-s3-aws-resources", usernameVariable: 'amazon_s3_base_path', passwordVariable: 'amazon_s3_bucket')]) {
+
+            sh """
+
+                set -e
+                set +x
+
+                env | grep JAVA | grep HOME
+
+                echo "Git clone: ${git_repo_url} with ref: ${git_reference}"
+                rm -rf search
+                git clone ${git_repo_url} search
+                cd search/
+                git checkout -f ${git_reference}
+                git rev-parse HEAD
+
+                echo "Stop existing gradledaemon"
+                ./gradlew --stop
+                find ~/.gradle -type f -name "*.lock" -delete
+
+                echo "Check existing dockercontainer"
+                docker ps -a
+                docker stop `docker ps -qa` > /dev/null 2>&1 || echo
+                docker rm `docker ps -qa` > /dev/null 2>&1 || echo
+                echo "Stop existing dockercontainer"
+                docker ps -a
+
+                echo "Check docker-compose version"
+                docker-compose version
+
+                echo "Start gradlecheck"
+                GRADLE_CHECK_STATUS=0
+                ./gradlew check -Dtests.coverage=true --no-daemon --no-scan || GRADLE_CHECK_STATUS=1
+
+                if [ "\$GRADLE_CHECK_STATUS" != 0 ]; then
+                    echo Gradle Check Failed!
+                    exit 1
+                fi
+
+            """
+        }
+
+    }
+
+
+}