build.gradle

/*
 * This file was generated by the Gradle 'init' task.
 *
 * This is a general purpose Gradle build.
 * Learn more about Gradle by exploring our samples at https://docs.gradle.org/7.5.1/samples
 * This project uses @Incubating APIs which are subject to change.
 */

import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.testclusters.OpenSearchCluster

import java.nio.file.Paths
import java.util.concurrent.Callable

buildscript {
    ext {
        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        plugin_no_snapshot = opensearch_build
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
            plugin_no_snapshot += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        opensearch_group = "org.opensearch"
        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
    }

    repositories {
        mavenLocal()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
        classpath "io.freefair.gradle:lombok-plugin:8.4"

        configurations.all {
            resolutionStrategy {
                force("org.eclipse.platform:org.eclipse.core.runtime:4.29.0") // CVE for < 4.29
                force("org.eclipse.platform:org.eclipse.core.resources:4.20.0") // CVE for < 4.20
            }
        }
    }
}

plugins{
    id "de.undercouch.download" version "5.3.0"
}
apply plugin: 'java'
apply plugin: 'java-test-fixtures'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'jacoco'
apply plugin: "com.diffplug.spotless"
apply plugin: 'io.freefair.lombok'
apply from: 'gradle/formatting.gradle'

ext.opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

ext {
    default_bwc_version = System.getProperty("bwc.version")
    default_bwc_bundle_version= System.getProperty("bwc.bundle.version")
    bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null && project.properties['customDistributionDownloadType'] == "bundle")
    neural_search_bwc_version = bwcBundleTest ? System.getProperty("tests.bwc.bundle.version",rootProject.ext.default_bwc_bundle_version): System.getProperty("tests.bwc.version", rootProject.ext.default_bwc_version)
    currentBundleVersion = opensearch_version.replace("-SNAPSHOT","")

    // Config below including files are copied from security demo configuration
    runIntegTestWithSecurityPlugin= System.getProperty("security.enabled")
    if (runIntegTestWithSecurityPlugin == "true"){
        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem','kirk.pem','kirk-key.pem'].forEach { file ->
            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
            download.run {
                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
                dest local
                overwrite false
            }
        }
    }

    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
    projectSubstitutions = [:]

    configureSecurityPlugin = { OpenSearchCluster cluster ->
        configurations.secureIntegTestPluginArchive.asFileTree.each {
            if(it.name.contains("opensearch-security")){
                cluster.plugin(provider(new Callable<RegularFile>() {
                    @Override
                    RegularFile call() throws Exception {
                        return new RegularFile() {
                            @Override
                            File getAsFile() {
                                return it
                            }
                        }
                    }
                }))
            }
        }

        cluster.getNodes().forEach { node ->
            var creds = node.getCredentials()
            if (creds.isEmpty()) {
                creds.add(Map.of('username', 'admin', 'password', 'admin'))
            } else {
                creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
            }
        }

        // // Config below including files are copied from security demo configuration
        cluster.extraConfigFile("esnode.pem", file("$opensearch_tmp_dir/esnode.pem"))
        cluster.extraConfigFile("esnode-key.pem", file("$opensearch_tmp_dir/esnode-key.pem"))
        cluster.extraConfigFile("root-ca.pem", file("$opensearch_tmp_dir/root-ca.pem"))

        // This configuration is copied from the security plugins demo install:
        // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
        cluster.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
        cluster.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
        cluster.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
        cluster.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
        cluster.setting("plugins.security.ssl.http.enabled", "true")
        cluster.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
        cluster.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
        cluster.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
        cluster.setting("plugins.security.allow_unsafe_democertificates", "true")
        cluster.setting("plugins.security.allow_default_init_securityindex", "true")
        cluster.setting("plugins.security.unsupported.inject_user.enabled", "true")

        cluster.setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
        cluster.setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
        cluster.setting('plugins.security.system_indices.enabled', "true")
        cluster.setting('plugins.security.system_indices.indices', '[' +
                '".plugins-ml-config", ' +
                '".plugins-ml-connector", ' +
                '".plugins-ml-model-group", ' +
                '".plugins-ml-model", ".plugins-ml-task", ' +
                '".plugins-ml-conversation-meta", ' +
                '".plugins-ml-conversation-interactions", ' +
                ']'
        )
        cluster.setSecure(true)
    }
}

allprojects {
    group = opensearch_group
    version = "${opensearch_build}"
    targetCompatibility = JavaVersion.VERSION_21
    sourceCompatibility = JavaVersion.VERSION_21

    apply from: rootProject.file('repositories.gradle').absoluteFile
    plugins.withId('java') {
        sourceCompatibility = targetCompatibility = JavaVersion.VERSION_21
    }

    afterEvaluate {
        project.dependencyLicenses.enabled = false
        project.thirdPartyAudit.enabled = false
        project.loggerUsageCheck.enabled = false
        project.forbiddenApis.ignoreFailures = false
        project.forbiddenPatterns {
            setEnabled(false)
        }
        project.testingConventions.enabled = false
        project.validateNebulaPom.enabled = false
        project.licenseFile = rootProject.file('LICENSE.txt')
        project.noticeFile = rootProject.file('NOTICE.txt')
    }
}

configurations {
    zipArchive
    secureIntegTestPluginArchive
}

tasks.register("preparePluginPathDirs") {
    mustRunAfter clean
    doLast {
        def newPath = pathToPlugin.replace(".", "/")
        mkdir "src/main/java/org/opensearch/$newPath"
        mkdir "src/test/java/org/opensearch/$newPath"
    }
}

def pluginName = 'opensearch-neural-search'
def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem'
def projectPath = 'org.opensearch'
def pathToPlugin = 'neuralsearch.plugin'
def pluginClassName = 'NeuralSearch'

publishing {
    repositories {
        maven {
            name = "Snapshots"
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = pluginName
                description = pluginDescription
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/neural-search"
                    }
                }
            }
        }
    }
}
opensearchplugin {
    name pluginName
    description pluginDescription
    classname "${projectPath}.${pathToPlugin}.${pluginClassName}"
    licenseFile rootProject.file('LICENSE')
    noticeFile rootProject.file('NOTICE')
    extendedPlugins = ['opensearch-knn']
}

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

def knnJarDirectory = "$buildDir/dependencies/opensearch-knn"

dependencies {
    api "org.opensearch:opensearch:${opensearch_version}"
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
    secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"
    compileOnly fileTree(dir: knnJarDirectory, include: "opensearch-knn-${opensearch_build}.jar")
    compileOnly group: 'com.google.guava', name: 'guava', version:'32.1.3-jre'
    compileOnly group: 'commons-lang', name: 'commons-lang', version: '2.6'
    api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
    testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
    // ml-common excluded reflection for runtime so we need to add it by ourselves.
    // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
    // TODO: Remove following three lines of dependencies if ml-common include them in their jar
    runtimeOnly group: 'org.reflections', name: 'reflections', version: '0.9.12'
    runtimeOnly group: 'org.javassist', name: 'javassist', version: '3.29.2-GA'
    runtimeOnly group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}"
    runtimeOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
    runtimeOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
    runtimeOnly group: 'org.json', name: 'json', version: '20231013'
    runtimeOnly("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
    runtimeOnly("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
    testFixturesImplementation "org.opensearch:common-utils:${version}"
    testFixturesImplementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
    testFixturesCompileOnly group: 'com.google.guava', name: 'guava', version:'32.1.3-jre'
    testFixturesCompileOnly fileTree(dir: knnJarDirectory, include: "opensearch-knn-${opensearch_build}.jar")
}

// In order to add the jar to the classpath, we need to unzip the
// k-NN zip and then copy it into a directory that we specify as a dependency.
task extractKnnJar(type: Copy) {
    mustRunAfter()
    from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-knn")}))
    into knnJarDirectory
}

project.tasks.delombok.dependsOn(extractKnnJar)

compileJava {
    dependsOn extractKnnJar
    dependsOn spotlessApply
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}
compileTestJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}
compileTestFixturesJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

def _numNodes = findProperty('numNodes') as Integer ?: 1

test {
    include '**/*Tests.class'
    systemProperty 'tests.security.manager', 'false'
}

// Setting up Integration Tests
task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    boolean runCompleteAggsTestSuite = Boolean.parseBoolean(System.getProperty('test_aggs', "false"))
    if (!runCompleteAggsTestSuite) {
        filter {
            excludeTestsMatching "org.opensearch.neuralsearch.query.aggregation.*IT"
        }
    }
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    // allows integration test classes to access test resource from project root path
    systemProperty('project.root', project.rootDir.absolutePath)
    systemProperty 'security.enabled', System.getProperty('security.enabled')
    var is_https = System.getProperty("https")
    var user = System.getProperty("user")
    var password = System.getProperty("password")
    if (System.getProperty("security.enabled") != null) {
        // If security is enabled, set is_https/user/password defaults
        is_https = is_https == null ? "true" : is_https
        user = user == null ? "admin" : user
        password = password == null ? "admin" : password
    }
    systemProperty("https", is_https)
    systemProperty("user", user)
    systemProperty("password", password)

    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }
}

testClusters.integTest {
    testDistribution = "ARCHIVE"

    // Optionally install security
    if (System.getProperty("security.enabled") != null && System.getProperty("security.enabled") == "true") {
        configureSecurityPlugin(testClusters.integTest)
    }

    // Install K-NN/ml-commons plugins on the integTest cluster nodes except security
    configurations.zipArchive.asFileTree.each {
        plugin(provider(new Callable<RegularFile>(){
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            return it
                        }
                    }
                }
        }))
    }

    // This installs our neural-search plugin into the testClusters
    plugin(project.tasks.bundlePlugin.archiveFile)
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}")
            debugPort += 1
        }
    }

    // Increase heap size from default of 512mb to 1gb. When heap size is 512mb, our integ tests sporadically fail due
    // to ml-commons memory circuit breaker exception
    jvmArgs("-Xms1g", "-Xmx2g")
}

// Remote Integration Tests
task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    systemProperty 'cluster.number_of_nodes', "${_numNodes}"

    systemProperty 'tests.security.manager', 'false'

    // Run tests with remote cluster only if rest case is defined
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.neuralsearch.*IT"
        }
    }
}

run {
    useCluster testClusters.integTest
}

jacocoTestReport {
    dependsOn integTest, test
    reports {
        xml.required = true
        html.required = true
    }
}

check.dependsOn spotlessCheck
check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport

// updateVersion: Task to auto update version to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}