Add bwc tests for the distribution bundle

Signed-off-by: Zelin Hao <[email protected]>
opensearch-project · Dec 20, 2022 · a579602 · a579602
1 parent d39b9ba
commit a579602
Show file tree

Hide file tree

Showing 40 changed files with 2,705 additions and 1,038 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,33 @@
+name: Run AD benchmark
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Build-ad:
+    strategy:
+      matrix:
+        java: [17]
+      fail-fast: false
+
+    name: Run Anomaly detection model performance benchmark
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      # anomaly-detection
+      - name: Checkout AD
+        uses: actions/checkout@v2
+
+      - name: Build and Run Tests
+        run: |
+          ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true
+          ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -48,6 +48,8 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and
 8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler.
 9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler.
 10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
+11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output
+12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -9,7 +9,7 @@
 | Kaituo Li               | [kaituo](https://github.com/kaituo)                     | Amazon      |
 | Lai Jiang               | [wnbts](https://github.com/wnbts)                       | Amazon      |
 | Sarat Vemulapalli       | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
-| Tyler Ohlsen            | [ohtyler](https://github.com/ohltyler)                  | Amazon      |
+| Tyler Ohlsen            | [ohltyler](https://github.com/ohltyler)                 | Amazon      |
 | Vamshi Vijay Nakkirtha  | [vamshin](https://github.com/vamshin)                   | Amazon      |
 | Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB)                 | Amazon      |
 | Yaliang Wu              | [ylwu-amzn](https://github.com/ylwu-amzn)               | Amazon      |

diff --git a/build.gradle b/build.gradle
@@ -134,7 +134,7 @@ configurations.all {
     if (it.state != Configuration.State.UNRESOLVED) return
     resolutionStrategy {
         force "joda-time:joda-time:${versions.joda}"
-        force "com.fasterxml.jackson.core:jackson-core:2.13.4"
+        force "com.fasterxml.jackson.core:jackson-core:2.14.1"
         force "commons-logging:commons-logging:${versions.commonslogging}"
         force "org.apache.httpcomponents:httpcore:${versions.httpcore}"
         force "commons-codec:commons-codec:${versions.commonscodec}"
@@ -214,6 +214,12 @@ test {
     }
     include '**/*Tests.class'
     systemProperty 'tests.security.manager', 'false'
+
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.ml.HCADModelPerfTests"
+        }
+    }
 }
 
 task integTest(type: RestIntegTestTask) {
@@ -259,6 +265,12 @@ integTest {
         }
     }
 
+    if (System.getProperty("model-benchmark") == null || System.getProperty("model-benchmark") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.ad.e2e.SingleStreamModelPerfIT"
+        }
+    }
+
     // The 'doFirst' delays till execution time.
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
@@ -372,7 +384,11 @@ task integTestRemote(type: RestIntegTestTask) {
     }
 }
 
-String bwcVersion = "1.1.0.0"
+String bwcMinVersion = "1.1.0.0"
+String bwcBundleVersion = "1.3.2.0"
+Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
+        project.properties['customDistributionDownloadType'] == "bundle");
+String bwcVersion = bwcBundleTest ? bwcBundleVersion : bwcMinVersion
 String baseName = "adBwcCluster"
 String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
 String bwcJobSchedulerPath = bwcFilePath + "job-scheduler/"
@@ -382,44 +398,72 @@ String bwcAnomalyDetectionPath = bwcFilePath + "anomaly-detection/"
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["1.1.0", opensearch_version]
             numberOfNodes = 3
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
-                                project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
+            if (bwcBundleTest) {
+                versions = ["1.3.2", "2.5.0"]
+                nodes.each { node ->
+                    node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
+                    node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
+                    node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
+                    node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
+                    node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
+                    node.setting("plugins.security.disabled", "true")
+                    node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                    node.setting("plugins.security.ssl.http.enabled", "true")
+                    node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                    node.setting("plugins.security.allow_default_init_securityindex", "true")
+                    node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+                    node.setting("plugins.security.audit.type", "internal_elasticsearch")
+                    node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                    node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                    node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                    node.setting("plugins.security.system_indices.enabled", "true")
+                }
+            } else {
+                versions = ["1.1.0", opensearch_version]
+                plugin(provider(new Callable<RegularFile>(){
+                    @Override
+                    RegularFile call() throws Exception {
+                        return new RegularFile() {
+                            @Override
+                            File getAsFile() {
+                                if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) {
+                                    project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion"))
+                                }
+                                project.mkdir bwcJobSchedulerPath + bwcVersion
+                                ant.get(src: bwcOpenSearchJSDownload,
+                                        dest: bwcJobSchedulerPath + bwcVersion,
+                                        httpusecaches: false)
+                                return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
                             }
-                            project.mkdir bwcJobSchedulerPath + bwcVersion
-                            ant.get(src: bwcOpenSearchJSDownload,
-                                    dest: bwcJobSchedulerPath + bwcVersion,
-                                    httpusecaches: false)
-                            return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile()
                         }
                     }
-                }
-            }))
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            if (new File("$project.rootDir/$bwcFilePath/anomaly-detection/$bwcVersion").exists()) {
-                                project.delete(files("$project.rootDir/$bwcFilePath/anomaly-detection/$bwcVersion"))
+                }))
+                plugin(provider(new Callable<RegularFile>(){
+                    @Override
+                    RegularFile call() throws Exception {
+                        return new RegularFile() {
+                            @Override
+                            File getAsFile() {
+                                if (new File("$project.rootDir/$bwcFilePath/anomaly-detection/$bwcVersion").exists()) {
+                                    project.delete(files("$project.rootDir/$bwcFilePath/anomaly-detection/$bwcVersion"))
+                                }
+                                project.mkdir bwcAnomalyDetectionPath + bwcVersion
+                                ant.get(src: bwcOpenSearchADDownload,
+                                        dest: bwcAnomalyDetectionPath + bwcVersion,
+                                        httpusecaches: false)
+                                return fileTree(bwcAnomalyDetectionPath + bwcVersion).getSingleFile()
                             }
-                            project.mkdir bwcAnomalyDetectionPath + bwcVersion
-                            ant.get(src: bwcOpenSearchADDownload,
-                                    dest: bwcAnomalyDetectionPath + bwcVersion,
-                                    httpusecaches: false)
-                            return fileTree(bwcAnomalyDetectionPath + bwcVersion).getSingleFile()
                         }
                     }
-                }
-            }))
+                }))
+            }
             setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
             setting 'http.content_type.required', 'true'
         }
@@ -479,9 +523,16 @@ List<Provider<RegularFile>> plugins = [
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}0"
     dependsOn "${baseName}#oldVersionClusterTask0"
-    doFirst {
-      testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
+
     filter {
         includeTestsMatching "org.opensearch.ad.bwc.*IT"
     }
@@ -498,8 +549,14 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
     useCluster testClusters."${baseName}0"
-    doFirst {
-      testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.ad.bwc.*IT"
@@ -517,9 +574,16 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
     useCluster testClusters."${baseName}0"
-    doFirst {
-      testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
+
     filter {
         includeTestsMatching "org.opensearch.ad.bwc.*IT"
     }
@@ -536,8 +600,14 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
 task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#oldVersionClusterTask1"
     useCluster testClusters."${baseName}1"
-    doFirst {
-      testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}1".goToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.ad.bwc.*IT"
@@ -665,9 +735,9 @@ dependencies {
     implementation 'software.amazon.randomcutforest:randomcutforest-core:3.0-rc3'
 
     // force Jackson version to avoid version conflict issue
-    implementation "com.fasterxml.jackson.core:jackson-core:2.13.4"
-    implementation "com.fasterxml.jackson.core:jackson-databind:2.13.4.2"
-    implementation "com.fasterxml.jackson.core:jackson-annotations:2.13.4"
+    implementation "com.fasterxml.jackson.core:jackson-core:2.14.1"
+    implementation "com.fasterxml.jackson.core:jackson-databind:2.14.1"
+    implementation "com.fasterxml.jackson.core:jackson-annotations:2.14.1"
 
     // used for serializing/deserializing rcf models.
     implementation group: 'io.protostuff', name: 'protostuff-core', version: '1.8.0'