Merge remote-tracking branch 'upstream/main' into replace-master-int-…

…server-main

Signed-off-by: Tianli Feng <[email protected]>

.gitignore

@@ -56,3 +56,6 @@ testfixtures_shared/
 
 # These are generated from .ci/jobs.t
 .ci/jobs/
+
+# build files generated
+doc-tools/missing-doclet/bin/

.linelint.yml

@@ -12,7 +12,6 @@ ignore:
   - 'buildSrc/src/testKit/opensearch.build/NOTICE'
   - 'server/licenses/apache-log4j-extras-DEPENDENCIES'
   # Empty files
-  - 'doc-tools/missing-doclet/bin/main/org/opensearch/missingdoclet/MissingDoclet.class'
   - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/build.gradle'
   - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-tar/build.gradle'
   - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix/build.gradle'

buildSrc/build.gradle

@@ -115,7 +115,7 @@ dependencies {
   api 'de.thetaphi:forbiddenapis:3.3'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.68'
+  api 'com.networknt:json-schema-validator:1.0.69'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"

buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt

@@ -26,4 +26,4 @@ com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly te
 
 org.junit.Test @defaultMessage Just name your test method testFooBar
 
-java.lang.Math#random() @ Use one of the various randomization methods from LuceneTestCase or ESTestCase for reproducibility
+java.lang.Math#random() @ Use one of the various randomization methods from LuceneTestCase or OpenSearchTestCase for reproducibility

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.1.0
+lucene            = 9.2.0-snapshot-f4f1f70
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.2+8

client/benchmark/README.md

@@ -29,7 +29,7 @@ Example invocation:
 wget http://benchmarks.elasticsearch.org.s3.amazonaws.com/corpora/geonames/documents-2.json.bz2
 bzip2 -d documents-2.json.bz2
 mv documents-2.json client/benchmark/build
-gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames type 8647880 5000'
+gradlew -p client/benchmark run --args ' rest bulk localhost build/documents-2.json geonames 8647880 5000'
 ```
 
 The parameters are all in the `'`s and are in order:
@@ -39,7 +39,6 @@ The parameters are all in the `'`s and are in order:
 * Benchmark target host IP (the host where OpenSearch is running)
 * full path to the file that should be bulk indexed
 * name of the index
-* name of the (sole) type in the index
 * number of documents in the file
 * bulk size
 

client/benchmark/src/main/java/org/opensearch/client/benchmark/AbstractBenchmark.java

@@ -49,7 +49,7 @@ public abstract class AbstractBenchmark<T extends Closeable> {
 
     protected abstract T client(String benchmarkTargetHost) throws Exception;
 
-    protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName, String typeName);
+    protected abstract BulkRequestExecutor bulkRequestExecutor(T client, String indexName);
 
     protected abstract SearchRequestExecutor searchRequestExecutor(T client, String indexName);
 
@@ -76,16 +76,15 @@ public final void run(String[] args) throws Exception {
 
     @SuppressForbidden(reason = "system out is ok for a command line tool")
     private void runBulkIndexBenchmark(String[] args) throws Exception {
-        if (args.length != 7) {
-            System.err.println("usage: 'bulk' benchmarkTargetHostIp indexFilePath indexName typeName numberOfDocuments bulkSize");
+        if (args.length != 6) {
+            System.err.println("usage: 'bulk' benchmarkTargetHostIp indexFilePath indexName numberOfDocuments bulkSize");
             System.exit(1);
         }
         String benchmarkTargetHost = args[1];
         String indexFilePath = args[2];
         String indexName = args[3];
-        String typeName = args[4];
-        int totalDocs = Integer.valueOf(args[5]);
-        int bulkSize = Integer.valueOf(args[6]);
+        int totalDocs = Integer.valueOf(args[4]);
+        int bulkSize = Integer.valueOf(args[5]);
 
         int totalIterationCount = (int) Math.floor(totalDocs / bulkSize);
         // consider 40% of all iterations as warmup iterations
@@ -97,7 +96,7 @@ private void runBulkIndexBenchmark(String[] args) throws Exception {
         BenchmarkRunner benchmark = new BenchmarkRunner(
             warmupIterations,
             iterations,
-            new BulkBenchmarkTask(bulkRequestExecutor(client, indexName, typeName), indexFilePath, warmupIterations, iterations, bulkSize)
+            new BulkBenchmarkTask(bulkRequestExecutor(client, indexName), indexFilePath, warmupIterations, iterations, bulkSize)
         );
 
         try {

client/benchmark/src/main/java/org/opensearch/client/benchmark/rest/RestClientBenchmark.java

@@ -65,8 +65,8 @@ protected RestClient client(String benchmarkTargetHost) {
     }
 
     @Override
-    protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName, String typeName) {
-        return new RestBulkRequestExecutor(client, indexName, typeName);
+    protected BulkRequestExecutor bulkRequestExecutor(RestClient client, String indexName) {
+        return new RestBulkRequestExecutor(client, indexName);
     }
 
     @Override
@@ -78,9 +78,9 @@ private static final class RestBulkRequestExecutor implements BulkRequestExecuto
         private final RestClient client;
         private final String actionMetadata;
 
-        RestBulkRequestExecutor(RestClient client, String index, String type) {
+        RestBulkRequestExecutor(RestClient client, String index) {
             this.client = client;
-            this.actionMetadata = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
+            this.actionMetadata = String.format(Locale.ROOT, "{ \"index\" : { \"_index\" : \"%s\" } }%n", index);
         }
 
         @Override
@@ -91,7 +91,7 @@ public boolean bulkIndex(List<String> bulkData) {
                 bulkRequestBody.append(bulkItem);
                 bulkRequestBody.append("\n");
             }
-            Request request = new Request("POST", "/geonames/type/_noop_bulk");
+            Request request = new Request("POST", "/geonames/_noop_bulk");
             request.setJsonEntity(bulkRequestBody.toString());
             try {
                 Response response = client.performRequest(request);

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -139,7 +139,9 @@
 import org.opensearch.search.aggregations.bucket.sampler.InternalSampler;
 import org.opensearch.search.aggregations.bucket.sampler.ParsedSampler;
 import org.opensearch.search.aggregations.bucket.terms.LongRareTerms;
+import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.terms.ParsedLongRareTerms;
+import org.opensearch.search.aggregations.bucket.terms.ParsedMultiTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedSignificantLongTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedSignificantStringTerms;
 import org.opensearch.search.aggregations.bucket.terms.ParsedStringRareTerms;
@@ -1915,6 +1917,10 @@ private <Req, Resp> Cancellable internalPerformRequestAsync(
         ActionListener<Resp> listener,
         Set<Integer> ignores
     ) {
+        if (listener == null) {
+            throw new IllegalArgumentException("The listener is required and cannot be null");
+        }
+
         Request req;
         try {
             req = requestConverter.apply(request);
@@ -2140,6 +2146,7 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
         map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
         map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
+        map.put(MultiTermsAggregationBuilder.NAME, (p, c) -> ParsedMultiTerms.fromXContent(p, (String) c));
         List<NamedXContentRegistry.Entry> entries = map.entrySet()
             .stream()
             .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))

client/rest-high-level/src/test/java/org/opensearch/client/PingAndInfoIT.java

@@ -33,7 +33,6 @@
 package org.opensearch.client;
 
 import org.apache.http.client.methods.HttpGet;
-import org.opensearch.action.main.TransportMainAction;
 import org.opensearch.client.core.MainResponse;
 
 import java.io.IOException;
@@ -63,25 +62,4 @@ public void testInfo() throws IOException {
         assertTrue(versionMap.get("number").toString().startsWith(info.getVersion().getNumber()));
         assertEquals(versionMap.get("lucene_version"), info.getVersion().getLuceneVersion());
     }
-
-    public void testInfo_overrideResponseVersion() throws IOException {
-        Request overrideResponseVersionRequest = new Request("PUT", "/_cluster/settings");
-        overrideResponseVersionRequest.setOptions(expectWarnings(TransportMainAction.OVERRIDE_MAIN_RESPONSE_VERSION_DEPRECATION_MESSAGE));
-        overrideResponseVersionRequest.setJsonEntity("{\"persistent\":{\"compatibility\": {\"override_main_response_version\":true}}}");
-        client().performRequest(overrideResponseVersionRequest);
-
-        MainResponse info = highLevelClient().info(RequestOptions.DEFAULT);
-        assertEquals("7.10.2", info.getVersion().getNumber());
-
-        // Set back to default version.
-        Request resetResponseVersionRequest = new Request("PUT", "/_cluster/settings");
-        resetResponseVersionRequest.setJsonEntity("{\"persistent\":{\"compatibility\": {\"override_main_response_version\":null}}}");
-        client().performRequest(resetResponseVersionRequest);
-
-        Map<String, Object> infoAsMap = entityAsMap(adminClient().performRequest(new Request(HttpGet.METHOD_NAME, "/")));
-        @SuppressWarnings("unchecked")
-        Map<String, Object> versionMap = (Map<String, Object>) infoAsMap.get("version");
-        info = highLevelClient().info(RequestOptions.DEFAULT);
-        assertTrue(versionMap.get("number").toString().startsWith(info.getVersion().getNumber()));
-    }
 }

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -284,6 +284,20 @@ public ActionRequestValidationException validate() {
         }
     }
 
+    public void testNullableActionListener() {
+        ActionRequest request = new ActionRequest() {
+            @Override
+            public ActionRequestValidationException validate() {
+                return null;
+            }
+        };
+
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> restHighLevelClient.performRequestAsync(request, null, RequestOptions.DEFAULT, null, null, null)
+        );
+    }
+
     public void testParseEntity() throws IOException {
         {
             IllegalStateException ise = expectThrows(IllegalStateException.class, () -> restHighLevelClient.parseEntity(null, null));

distribution/tools/java-version-checker/src/main/java/org/opensearch/tools/java_version_checker/JavaVersionChecker.java

@@ -32,18 +32,19 @@
 
 package org.opensearch.tools.java_version_checker;
 
+import java.lang.Runtime.Version;
 import java.util.Arrays;
 import java.util.Locale;
 
 /**
- * Simple program that checks if the runtime Java version is at least 1.8.
+ * Simple program that checks if the runtime Java version is at least 11
  */
 final class JavaVersionChecker {
 
     private JavaVersionChecker() {}
 
     /**
-     * The main entry point. The exit code is 0 if the Java version is at least 1.8, otherwise the exit code is 1.
+     * The main entry point. The exit code is 0 if the Java version is at least 11, otherwise the exit code is 1.
      *
      * @param args the args to the program which are rejected if not empty
      */
@@ -52,23 +53,15 @@ public static void main(final String[] args) {
         if (args.length != 0) {
             throw new IllegalArgumentException("expected zero arguments but was " + Arrays.toString(args));
         }
-        if (JavaVersion.compare(JavaVersion.CURRENT, JavaVersion.JAVA_8) < 0) {
+        if (Runtime.version().compareTo(Version.parse("11")) < 0) {
             final String message = String.format(
                 Locale.ROOT,
-                "the minimum required Java version is 8; your Java version from [%s] does not meet this requirement",
+                "OpenSearch requires Java 11; your Java version from [%s] does not meet this requirement",
                 System.getProperty("java.home")
             );
             errPrintln(message);
             exit(1);
         }
-        if (JavaVersion.compare(JavaVersion.CURRENT, JavaVersion.JAVA_11) < 0) {
-            final String message = String.format(
-                Locale.ROOT,
-                "future versions of OpenSearch will require Java 11; your Java version from [%s] does not meet this requirement",
-                System.getProperty("java.home")
-            );
-            errPrintln(message);
-        }
         exit(0);
     }
 

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/JvmErgonomics.java

@@ -32,8 +32,6 @@
 
 package org.opensearch.tools.launchers;
 
-import org.opensearch.tools.java_version_checker.JavaVersion;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -73,19 +71,7 @@ static List<String> choose(final List<String> userDefinedJvmOptions) throws Inte
         final long heapSize = extractHeapSize(finalJvmOptions);
         final long maxDirectMemorySize = extractMaxDirectMemorySize(finalJvmOptions);
 
-        if (System.getProperty("os.name").startsWith("Windows") && JavaVersion.majorVersion(JavaVersion.CURRENT) == 8) {
-            Launchers.errPrintln("Warning: with JDK 8 on Windows, OpenSearch may be unable to derive correct");
-            Launchers.errPrintln("  ergonomic settings due to a JDK issue (JDK-8074459). Please use a newer");
-            Launchers.errPrintln("  version of Java.");
-        }
-
         if (maxDirectMemorySize == 0 && userDefinedJvmOptions.stream().noneMatch(s -> s.startsWith("-XX:MaxDirectMemorySize"))) {
-
-            if (System.getProperty("os.name").startsWith("Windows") && JavaVersion.majorVersion(JavaVersion.CURRENT) == 8) {
-                Launchers.errPrintln("Warning: MaxDirectMemorySize may have been miscalculated due to JDK-8074459.");
-                Launchers.errPrintln("  Please use a newer version of Java or set MaxDirectMemorySize explicitly.");
-            }
-
             ergonomicChoices.add("-XX:MaxDirectMemorySize=" + heapSize / 2);
         }
         return ergonomicChoices;

distribution/tools/launchers/src/main/java/org/opensearch/tools/launchers/JvmOptionsParser.java

@@ -32,8 +32,6 @@
 
 package org.opensearch.tools.launchers;
 
-import org.opensearch.tools.java_version_checker.JavaVersion;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -183,7 +181,7 @@ List<String> readJvmOptionsFiles(final Path config) throws IOException, JvmOptio
                 Reader reader = new InputStreamReader(is, StandardCharsets.UTF_8);
                 BufferedReader br = new BufferedReader(reader)
             ) {
-                parse(JavaVersion.majorVersion(JavaVersion.CURRENT), br, jvmOptions::add, invalidLines::put);
+                parse(Runtime.version().feature(), br, jvmOptions::add, invalidLines::put);
             }
             if (invalidLines.isEmpty() == false) {
                 throw new JvmOptionsFileParserException(jvmOptionsFile, invalidLines);