Merge branch 'main' into non-issue/graph_serverless_scope

javanna · Jan 27, 2025 · 3ac6258 · 3ac6258
2 parents cf0dfcd + 1f2824c
commit 3ac6258
Show file tree

Hide file tree

Showing 245 changed files with 5,843 additions and 2,432 deletions.
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -72,18 +72,19 @@ public void apply(Project project) {
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         var isIdeaSync = System.getProperty("idea.sync.active", "false").equals("true");
         var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
+        int minJavaVersion = Integer.parseInt(buildParams.getMinimumCompilerVersion().getMajorVersion());
 
         // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
         // Avoids an IntelliJ bug:
         // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
         if (isIdeaSync == false || ideaSourceSetsEnabled) {
-            List<Integer> mainVersions = findSourceVersions(project);
+            List<Integer> mainVersions = findSourceVersions(project, minJavaVersion);
             List<String> mainSourceSets = new ArrayList<>();
             mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
-            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), 21);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), minJavaVersion);
             List<String> testSourceSets = new ArrayList<>(mainSourceSets);
             testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
-            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), minJavaVersion);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
                 SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
@@ -103,6 +104,7 @@ public void apply(Project project) {
     }
 
     private void configureMrjar(Project project) {
+
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> { task.manifest(manifest -> { manifest.attributes(Map.of("Multi-Release", "true")); }); });
 
@@ -222,7 +224,7 @@ private void createTestTask(
         project.getTasks().named("check").configure(checkTask -> checkTask.dependsOn(testTaskProvider));
     }
 
-    private static List<Integer> findSourceVersions(Project project) {
+    private static List<Integer> findSourceVersions(Project project, int minJavaVersion) {
         var srcDir = project.getProjectDir().toPath().resolve("src");
         List<Integer> versions = new ArrayList<>();
         try (var subdirStream = Files.list(srcDir)) {
@@ -231,7 +233,23 @@ private static List<Integer> findSourceVersions(Project project) {
                 String sourcesetName = sourceSetPath.getFileName().toString();
                 Matcher sourcesetMatcher = MRJAR_SOURCESET_PATTERN.matcher(sourcesetName);
                 if (sourcesetMatcher.matches()) {
-                    versions.add(Integer.parseInt(sourcesetMatcher.group(1)));
+                    int version = Integer.parseInt(sourcesetMatcher.group(1));
+                    if (version < minJavaVersion) {
+                        // NOTE: We allow mainNN for the min java version so that incubating modules can be used without warnings.
+                        // It is a workaround for https://bugs.openjdk.org/browse/JDK-8187591. Once min java is 22, we
+                        // can use the SuppressWarnings("preview") in the code using incubating modules and this check
+                        // can change to <=
+                        throw new IllegalArgumentException(
+                            "Found src dir '"
+                                + sourcesetName
+                                + "' for Java "
+                                + version
+                                + " but multi-release jar sourceset should have version "
+                                + minJavaVersion
+                                + " or greater"
+                        );
+                    }
+                    versions.add(version);
                 }
             }
         } catch (IOException e) {

diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
@@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-                if (settingNames.contains(prefix + key)) {
-                    if (propertiesMap.containsKey(key)) {
-                        throw new IllegalStateException(
-                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
-                        );
-                    }
-
-                    try (SecureString token = secrets.getString(prefix + key)) {
-                        propertiesMap.put(key, token.toString());
-                    }
+            String prefix = "telemetry.";
+            if (settingNames.contains(prefix + key)) {
+                try (SecureString token = secrets.getString(prefix + key)) {
+                    propertiesMap.put(key, token.toString());
                 }
             }
-
         }
     }
 
@@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
-        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
-        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
-        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
-        for (String key : apmAgentSettings.keySet()) {
-            if (propertiesMap.containsKey(key)) {
-                throw new IllegalStateException(
-                    Strings.format(
-                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
-                        telemetryAgentPrefix,
-                        key,
-                        deprecatedTelemetryAgentPrefix,
-                        key
-                    )
-                );
-            }
-            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
-        }
-
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
-        if (globalLabels.length() == 0) {
-            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-        } else {
-            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-            if (tracingGlobalLabels.length() != 0) {
-                throw new IllegalArgumentException(
-                    "Cannot have global labels with tracing.agent prefix ["
-                        + globalLabels
-                        + "] and telemetry.apm.agent prefix ["
-                        + tracingGlobalLabels
-                        + "]"
-                );
-            }
-        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }

diff --git a/...ution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/...ution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java
@@ -25,18 +25,15 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException {
     }
 
     public void testExtractSecureSettings() {
-        MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("telemetry.secret_token", "token");
+        secureSettings.setString("telemetry.api_key", "key");
 
-        for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-            MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString(prefix + "secret_token", "token");
-            secureSettings.setString(prefix + "api_key", "key");
-
-            duplicateSecureSettings.setString(prefix + "api_key", "secret");
-
-            Map<String, String> propertiesMap = new HashMap<>();
-            APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
-
-            assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
-        }
-
-        Exception exception = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
-        );
-        assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
-        assertThat(exception.getMessage(), containsString("telemetry.api_key"));
-        assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
+        Map<String, String> propertiesMap = new HashMap<>();
+        APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
 
+        assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
     }
 
     public void testExtractSettings() throws UserException {
-        Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
-            .put(prefix + "server_url", "https://myurl:443")
-            .put(prefix + "service_node_name", "instance-0000000001");
-
-        for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
-            var name = "APM Tracing";
-            var deploy = "123";
-            var org = "456";
-            var extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-
-            assertThat(
-                extracted,
-                allOf(
-                    hasEntry("server_url", "https://myurl:443"),
-                    hasEntry("service_node_name", "instance-0000000001"),
-                    hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
-                    not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
-                )
-            );
-
-            List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(3));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
-
-            // test replacing with underscores and skipping empty
-            name = "APM=Tracing";
-            deploy = "";
-            org = ",456";
-            extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-            labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(2));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
-        }
-
-        IllegalStateException err = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractApmSettings(
-                Settings.builder()
-                    .put("tracing.apm.agent.server_url", "https://myurl:443")
-                    .put("telemetry.agent.server_url", "https://myurl-2:443")
-                    .build()
-            )
-        );
-        assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
-    }
-
-    public void testNoMixedLabels() {
-        String telemetryAgent = "telemetry.agent.";
-        String tracingAgent = "tracing.apm.agent.";
-        Settings settings = Settings.builder()
-            .put("tracing.apm.enabled", true)
-            .put(telemetryAgent + "server_url", "https://myurl:443")
-            .put(telemetryAgent + "service_node_name", "instance-0000000001")
-            .put(tracingAgent + "global_labels.deployment_id", "123")
-            .put(telemetryAgent + "global_labels.organization_id", "456")
+        Settings defaults = Settings.builder()
+            .put("telemetry.agent.server_url", "https://myurl:443")
+            .put("telemetry.agent.service_node_name", "instance-0000000001")
             .build();
 
-        IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
+        var name = "APM Tracing";
+        var deploy = "123";
+        var org = "456";
+        var extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+
         assertThat(
-            err.getMessage(),
-            is(
-                "Cannot have global labels with tracing.agent prefix [organization_id=456] and"
-                    + " telemetry.apm.agent prefix [deployment_id=123]"
+            extracted,
+            allOf(
+                hasEntry("server_url", "https://myurl:443"),
+                hasEntry("service_node_name", "instance-0000000001"),
+                hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
+                not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
             )
         );
+
+        List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(3));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
+
+        // test replacing with underscores and skipping empty
+        name = "APM=Tracing";
+        deploy = "";
+        org = ",456";
+        extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+        labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(2));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
     }
 
     private Path makeFakeAgentJar() throws IOException {

diff --git a/docs/build.gradle b/docs/build.gradle
@@ -120,6 +120,8 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
   systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+  systemProperty 'es.queryable_built_in_roles_enabled', 'false'
+
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 

diff --git a/docs/changelog/119926.yaml b/docs/changelog/119926.yaml
@@ -0,0 +1,11 @@
+pr: 119926
+summary: "Deprecated tracing.apm.* settings got removed."
+area: Infra/Metrics
+type: breaking
+issues: []
+breaking:
+  title: "Deprecated tracing.apm.* settings got removed."
+  area: Cluster and node setting
+  details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead.
+  impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present.
+  notable: false
diff --git a/docs/changelog/120267.yaml b/docs/changelog/120267.yaml
@@ -0,0 +1,16 @@
+pr: 120267
+summary: Set allow_partial_search_results=true by default
+area: EQL
+type: breaking
+issues: []
+breaking:
+  title: Set allow_partial_search_results=true by default
+  area: REST API
+  details:
+    Before this change, in case of shard failures, EQL queries always returned an error.
+    With this change, they will keep running and will return partial results.
+  impact:
+    EQL queries that would previously fail due to shard failures, will now succeed and return partial results.
+    The previous defaults can be restored by setting `xpack.eql.default_allow_partial_results` cluster setting to `false`
+    or setting with `allow_partial_search_results` to `false` in the query request.
+  notable: false
diff --git a/docs/changelog/120392.yaml b/docs/changelog/120392.yaml
@@ -0,0 +1,6 @@
+pr: 120392
+summary: Test/107515 restore template with match only text mapper it fail
+area: Search
+type: bug
+issues:
+ - 107515
diff --git a/docs/changelog/120487.yaml b/docs/changelog/120487.yaml
@@ -0,0 +1,5 @@
+pr: 120487
+summary: Fix cat_component_templates documentation
+area: CAT APIs
+type: bug
+issues: []
diff --git a/docs/changelog/120717.yaml b/docs/changelog/120717.yaml
@@ -0,0 +1,6 @@
+pr: 120717
+summary: Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled
+area: Ranking
+type: bug
+issues:
+ - 120739
diff --git a/docs/changelog/120727.yaml b/docs/changelog/120727.yaml
@@ -0,0 +1,6 @@
+pr: 120727
+summary: Esql - Support date nanos in date extract function
+area: ES|QL
+type: enhancement
+issues:
+ - 110000