Merge branch 'opensearch-project:main' into bwc-tests-type-mapping

.ci/bwcVersions

@@ -40,3 +40,4 @@ BWC_VERSION:
   - "1.3.1"
   - "1.3.2"
   - "2.0.0"
+  - "2.1.0"

TESTING.md

@@ -245,7 +245,7 @@ The YAML REST tests support all the options provided by the randomized runner, p
 
 -   `tests.rest.suite`: comma separated paths of the test suites to be run (by default loaded from /rest-api-spec/test). It is possible to run only a subset of the tests providing a sub-folder or even a single yaml file (the default /rest-api-spec/test prefix is optional when files are loaded from classpath) e.g. `-Dtests.rest.suite=index,get,create/10_with_id`
 
--   `tests.rest.blacklist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.blacklist=index/**/Index document,get/10_basic/**`
+-   `tests.rest.denylist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.denylist=index/**/Index document,get/10_basic/**`
 
 Java REST tests can be run with the "javaRestTest" task.
 

build.gradle

@@ -48,7 +48,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.3.0" apply false
+  id "com.diffplug.spotless" version "6.4.1" apply false
   id "org.gradle.test-retry" version "1.3.1" apply false
 }
 

buildSrc/build.gradle

@@ -113,7 +113,7 @@ dependencies {
   api "net.java.dev.jna:jna:5.10.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'de.thetaphi:forbiddenapis:3.3'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.12'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api 'org.apache.maven:maven-model:3.6.2'
   api 'com.networknt:json-schema-validator:1.0.68'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"

buildSrc/src/main/java/org/opensearch/gradle/Version.java

@@ -77,7 +77,7 @@ public Version(int major, int minor, int revision) {
         // currently snapshot is not taken into account
         int id = major * 10000000 + minor * 100000 + revision * 1000;
         // identify if new OpenSearch version 1
-        this.id = major == 1 || major == 2 ? id ^ MASK : id;
+        this.id = major == 1 || major == 2 || major == 3 ? id ^ MASK : id;
     }
 
     private static int parseSuffixNumber(String substring) {

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -237,6 +237,7 @@ public void runThirdPartyAudit() throws IOException {
         Set<String> jdkJarHellClasses = runJdkJarHellCheck();
 
         if (missingClassExcludes != null) {
+            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             long bogousExcludesCount = Stream.concat(missingClassExcludes.stream(), violationsExcludes.stream())
                 .filter(each -> missingClasses.contains(each) == false)
                 .filter(each -> violationsClasses.contains(each) == false)
@@ -247,7 +248,6 @@ public void runThirdPartyAudit() throws IOException {
                     "All excluded classes seem to have no issues. " + "This is sometimes an indication that the check silently failed"
                 );
             }
-            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             missingClasses.removeAll(missingClassExcludes);
         }
         assertNoPointlessExclusions("have no violations", violationsExcludes, violationsClasses);

buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java

@@ -53,11 +53,11 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     private static Project packagesProject;
     private static Project bwcProject;
 
-    private static final Version BWC_MAJOR_VERSION = Version.fromString("5.0.0");
-    private static final Version BWC_MINOR_VERSION = Version.fromString("4.1.0");
-    private static final Version BWC_STAGED_VERSION = Version.fromString("4.0.0");
-    private static final Version BWC_BUGFIX_VERSION = Version.fromString("4.0.1");
-    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("3.90.1");
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("6.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("5.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("5.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("5.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("4.90.1");
     private static final BwcVersions BWC_MINOR = new BwcVersions(
         new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)),
         BWC_MAJOR_VERSION

buildSrc/src/test/java/org/opensearch/gradle/VersionTests.java

@@ -65,7 +65,7 @@ public void testRelaxedVersionParsing() {
 
     public void testCompareWithStringVersions() {
         // 1.10.2 is now rebased to OpenSearch version; so this needs to report
-        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 3.0.0", Version.fromString("1.10.20").after("3.0.0"));
+        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 6.0.0", Version.fromString("1.10.20").after("6.0.0"));
         assertTrue(
             "7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))

buildSrc/version.properties

@@ -1,4 +1,4 @@
-opensearch        = 2.1.0
+opensearch        = 3.0.0
 lucene            = 9.1.0
 
 bundled_jdk_vendor = adoptium

client/rest-high-level/src/test/java/org/opensearch/client/core/MainResponseTests.java

@@ -33,7 +33,6 @@
 package org.opensearch.client.core;
 
 import org.opensearch.Build;
-import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.client.AbstractResponseTestCase;
 import org.opensearch.cluster.ClusterName;
@@ -53,7 +52,7 @@ protected org.opensearch.action.main.MainResponse createServerTestInstance(XCont
         ClusterName clusterName = new ClusterName(randomAlphaOfLength(10));
         String nodeName = randomAlphaOfLength(10);
         final String date = new Date(randomNonNegativeLong()).toString();
-        Version version = VersionUtils.randomVersionBetween(random(), LegacyESVersion.V_7_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT);
         Build build = new Build(
             Build.Type.UNKNOWN,
             randomAlphaOfLength(8),

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/AddFileKeyStoreCommand.java

@@ -95,7 +95,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
             keyStore.setFile(setting, Files.readAllBytes(file));
         }
 
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 
     @SuppressForbidden(reason = "file arg for cli")

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/AddStringKeyStoreCommand.java

@@ -121,7 +121,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
             }
         }
 
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 
 }

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/CreateKeyStoreCommand.java

@@ -59,16 +59,16 @@ class CreateKeyStoreCommand extends KeyStoreAwareCommand {
     @Override
     protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
         try (SecureString password = options.has(passwordOption) ? readPassword(terminal, true) : new SecureString(new char[0])) {
-            Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+            Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir());
             if (Files.exists(keystoreFile)) {
                 if (terminal.promptYesNo("An opensearch keystore already exists. Overwrite?", false) == false) {
                     terminal.println("Exiting without creating keystore.");
                     return;
                 }
             }
             KeyStoreWrapper keystore = KeyStoreWrapper.create();
-            keystore.save(env.configFile(), password.getChars());
-            terminal.println("Created opensearch keystore in " + KeyStoreWrapper.keystorePath(env.configFile()));
+            keystore.save(env.configDir(), password.getChars());
+            terminal.println("Created opensearch keystore in " + KeyStoreWrapper.keystorePath(env.configDir()));
         } catch (SecurityException e) {
             throw new UserException(ExitCodes.IO_ERROR, "Error creating the opensearch keystore.");
         }

distribution/tools/keystore-cli/src/main/java/org/opensearch/common/settings/RemoveSettingKeyStoreCommand.java

@@ -66,6 +66,6 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
             }
             keyStore.remove(setting);
         }
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 }

distribution/tools/keystore-cli/src/test/java/org/opensearch/bootstrap/BootstrapTests.java

@@ -71,7 +71,7 @@ public void setupEnv() throws IOException {
     }
 
     public void testLoadSecureSettings() throws Exception {
-        final Path configPath = env.configFile();
+        final Path configPath = env.configDir();
         final SecureString seed;
         try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
             seed = KeyStoreWrapper.SEED_SETTING.get(Settings.builder().setSecureSettings(keyStoreWrapper).build());

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/AddFileKeyStoreCommandTests.java

@@ -66,14 +66,14 @@ private Path createRandomFile() throws IOException {
         for (int i = 0; i < length; ++i) {
             bytes[i] = randomByte();
         }
-        Path file = env.configFile().resolve(randomAlphaOfLength(16));
+        Path file = env.configDir().resolve(randomAlphaOfLength(16));
         Files.write(file, bytes);
         return file;
     }
 
     private void addFile(KeyStoreWrapper keystore, String setting, Path file, String password) throws Exception {
         keystore.setFile(setting, Files.readAllBytes(file));
-        keystore.save(env.configFile(), password.toCharArray());
+        keystore.save(env.configDir(), password.toCharArray());
     }
 
     public void testMissingCreateWithEmptyPasswordWhenPrompted() throws Exception {
@@ -95,7 +95,7 @@ public void testMissingNoCreate() throws Exception {
         terminal.addSecretInput(randomFrom("", "keystorepassword"));
         terminal.addTextInput("n"); // explicit no
         execute("foo");
-        assertNull(KeyStoreWrapper.load(env.configFile()));
+        assertNull(KeyStoreWrapper.load(env.configDir()));
     }
 
     public void testOverwritePromptDefault() throws Exception {

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/AddStringKeyStoreCommandTests.java

@@ -101,7 +101,7 @@ public void testMissingPromptCreateWithoutPasswordWithoutPromptIfForced() throws
     public void testMissingNoCreate() throws Exception {
         terminal.addTextInput("n"); // explicit no
         execute("foo");
-        assertNull(KeyStoreWrapper.load(env.configFile()));
+        assertNull(KeyStoreWrapper.load(env.configDir()));
     }
 
     public void testOverwritePromptDefault() throws Exception {
@@ -161,7 +161,7 @@ public void testForceNonExistent() throws Exception {
 
     public void testPromptForValue() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         terminal.addSecretInput("secret value");
         execute("foo");
@@ -170,7 +170,7 @@ public void testPromptForValue() throws Exception {
 
     public void testPromptForMultipleValues() throws Exception {
         final String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         terminal.addSecretInput("bar1");
         terminal.addSecretInput("bar2");
@@ -183,7 +183,7 @@ public void testPromptForMultipleValues() throws Exception {
 
     public void testStdinShort() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("secret value 1");
         execute("-x", "foo");
@@ -192,7 +192,7 @@ public void testStdinShort() throws Exception {
 
     public void testStdinLong() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("secret value 2");
         execute("--stdin", "foo");
@@ -201,7 +201,7 @@ public void testStdinLong() throws Exception {
 
     public void testStdinNoInput() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("");
         execute("-x", "foo");
@@ -210,7 +210,7 @@ public void testStdinNoInput() throws Exception {
 
     public void testStdinInputWithLineBreaks() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("Typedthisandhitenter\n");
         execute("-x", "foo");
@@ -219,7 +219,7 @@ public void testStdinInputWithLineBreaks() throws Exception {
 
     public void testStdinInputWithCarriageReturn() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("Typedthisandhitenter\r");
         execute("-x", "foo");
@@ -228,7 +228,7 @@ public void testStdinInputWithCarriageReturn() throws Exception {
 
     public void testStdinWithMultipleValues() throws Exception {
         final String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("bar1\nbar2\nbar3");
         execute(randomFrom("-x", "--stdin"), "foo1", "foo2", "foo3");
@@ -239,7 +239,7 @@ public void testStdinWithMultipleValues() throws Exception {
 
     public void testAddUtf8String() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         final int stringSize = randomIntBetween(8, 16);
         try (CharArrayWriter secretChars = new CharArrayWriter(stringSize)) {

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/CreateKeyStoreCommandTests.java

@@ -67,7 +67,7 @@ public void testNotMatchingPasswords() throws Exception {
 
     public void testDefaultNotPromptForPassword() throws Exception {
         execute();
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
@@ -76,7 +76,7 @@ public void testPosix() throws Exception {
         terminal.addSecretInput(password);
         terminal.addSecretInput(password);
         execute();
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
@@ -86,13 +86,13 @@ public void testNotPosix() throws Exception {
         terminal.addSecretInput(password);
         env = setupEnv(false, fileSystems);
         execute();
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
     public void testOverwrite() throws Exception {
         String password = randomFrom("", "keystorepassword");
-        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir());
         byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8);
         Files.write(keystoreFile, content);
 
@@ -108,6 +108,6 @@ public void testOverwrite() throws Exception {
         terminal.addSecretInput(password);
         terminal.addSecretInput(password);
         execute();
-        assertNotNull(KeyStoreWrapper.load(env.configFile()));
+        assertNotNull(KeyStoreWrapper.load(env.configDir()));
     }
 }

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java

@@ -92,12 +92,12 @@ KeyStoreWrapper createKeystore(String password, String... settings) throws Excep
         for (int i = 0; i < settings.length; i += 2) {
             keystore.setString(settings[i], settings[i + 1].toCharArray());
         }
-        keystore.save(env.configFile(), password.toCharArray());
+        keystore.save(env.configDir(), password.toCharArray());
         return keystore;
     }
 
     KeyStoreWrapper loadKeystore(String password) throws Exception {
-        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configDir());
         keystore.decrypt(password.toCharArray());
         return keystore;
     }