Merge branch 'main' into feature/shard-changes-data-stream-alias

docs/changelog/117778.yaml

@@ -0,0 +1,5 @@
+pr: 117778
+summary: "[Connector APIs] Enforce index prefix for managed connectors"
+area: Extract&Transform
+type: feature
+issues: []

docs/changelog/118858.yaml

@@ -0,0 +1,5 @@
+pr: 118858
+summary: Lookup join on multiple join fields not yet supported
+area: ES|QL
+type: enhancement
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml

@@ -2012,3 +2012,42 @@ synthetic_source with copy_to pointing inside dynamic object:
       hits.hits.2.fields:
         c.copy.keyword: [ "hello", "zap" ]
 
+---
+create index with use_synthetic_source:
+  - requires:
+      cluster_features: ["mapper.synthetic_recovery_source"]
+      reason: requires synthetic recovery source
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              recovery:
+                use_synthetic_source: true
+              mapping:
+                source:
+                  mode: synthetic
+
+  - do:
+      indices.get_settings: {}
+  - match: { test.settings.index.mapping.source.mode: synthetic}
+  - is_true: test.settings.index.recovery.use_synthetic_source
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "field": "aaaa" }'
+          - '{ "create": { } }'
+          - '{ "field": "bbbb" }'
+
+  - do:
+      indices.disk_usage:
+        index: test
+        run_expensive_tasks: true
+  - gt: { test.fields.field.total_in_bytes: 0 }
+  - is_false: test.fields.field._recovery_source

server/src/main/java/org/elasticsearch/action/ResolvedIndices.java

@@ -150,10 +150,26 @@ public static ResolvedIndices resolveWithIndicesRequest(
         RemoteClusterService remoteClusterService,
         long startTimeInMillis
     ) {
-        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(
+        return resolveWithIndexNamesAndOptions(
+            request.indices(),
             request.indicesOptions(),
-            request.indices()
+            clusterState,
+            indexNameExpressionResolver,
+            remoteClusterService,
+            startTimeInMillis
         );
+    }
+
+    public static ResolvedIndices resolveWithIndexNamesAndOptions(
+        String[] indexNames,
+        IndicesOptions indicesOptions,
+        ClusterState clusterState,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        RemoteClusterService remoteClusterService,
+        long startTimeInMillis
+    ) {
+        final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(indicesOptions, indexNames);
+
         final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
 
         Index[] concreteLocalIndices = localIndices == null

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -685,29 +685,11 @@ public void validate(Boolean enabled, Map<Setting<?>, Object> settings) {
                         );
                     }
                 }
-
-                // Verify that all nodes can handle this setting
-                var version = (IndexVersion) settings.get(SETTING_INDEX_VERSION_CREATED);
-                if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)
-                    && version.between(
-                        IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT,
-                        IndexVersions.UPGRADE_TO_LUCENE_10_0_0
-                    ) == false) {
-                    throw new IllegalArgumentException(
-                        String.format(
-                            Locale.ROOT,
-                            "The setting [%s] is unavailable on this cluster because some nodes are running older "
-                                + "versions that do not support it. Please upgrade all nodes to the latest version "
-                                + "and try again.",
-                            RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                        )
-                    );
-                }
             }
 
             @Override
             public Iterator<Setting<?>> settings() {
-                List<Setting<?>> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, SETTING_INDEX_VERSION_CREATED, MODE);
+                List<Setting<?>> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE);
                 return res.iterator();
             }
         },
@@ -1050,6 +1032,24 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         indexMappingSourceMode = scopedSettings.get(INDEX_MAPPER_SOURCE_MODE_SETTING);
         recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
         recoverySourceSyntheticEnabled = scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
+        if (recoverySourceSyntheticEnabled) {
+            // Verify that all nodes can handle this setting
+            if (version.before(IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY)
+                && version.between(
+                    IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT,
+                    IndexVersions.UPGRADE_TO_LUCENE_10_0_0
+                ) == false) {
+                throw new IllegalArgumentException(
+                    String.format(
+                        Locale.ROOT,
+                        "The setting [%s] is unavailable on this cluster because some nodes are running older "
+                            + "versions that do not support it. Please upgrade all nodes to the latest version "
+                            + "and try again.",
+                        RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
+                    )
+                );
+            }
+        }
 
         scopedSettings.addSettingsUpdateConsumer(
             MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -77,7 +77,8 @@ public Set<NodeFeature> getTestFeatures() {
             DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE,
             CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX,
             META_FETCH_FIELDS_ERROR_CODE_CHANGED,
-            SPARSE_VECTOR_STORE_SUPPORT
+            SPARSE_VECTOR_STORE_SUPPORT,
+            SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -56,6 +56,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         "mapper.source.remove_synthetic_source_only_validation"
     );
     public static final NodeFeature SOURCE_MODE_FROM_INDEX_SETTING = new NodeFeature("mapper.source.mode_from_index_setting");
+    public static final NodeFeature SYNTHETIC_RECOVERY_SOURCE = new NodeFeature("mapper.synthetic_recovery_source");
 
     public static final String NAME = "_source";
     public static final String RECOVERY_SOURCE_NAME = "_recovery_source";

server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java

@@ -465,60 +465,6 @@ public void testRecoverySourceWitInvalidSettings() {
                 )
             );
         }
-        {
-            Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
-                .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
-                .build();
-            IllegalArgumentException exc = expectThrows(
-                IllegalArgumentException.class,
-                () -> createMapperService(
-                    IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT),
-                    settings,
-                    () -> false,
-                    topMapping(b -> {})
-                )
-            );
-            assertThat(
-                exc.getMessage(),
-                containsString(
-                    String.format(
-                        Locale.ROOT,
-                        "The setting [%s] is unavailable on this cluster",
-                        IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                    )
-                )
-            );
-        }
-        {
-            Settings settings = Settings.builder()
-                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString())
-                .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true)
-                .build();
-            IllegalArgumentException exc = expectThrows(
-                IllegalArgumentException.class,
-                () -> createMapperService(
-                    IndexVersionUtils.randomVersionBetween(
-                        random(),
-                        IndexVersions.UPGRADE_TO_LUCENE_10_0_0,
-                        IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER
-                    ),
-                    settings,
-                    () -> false,
-                    topMapping(b -> {})
-                )
-            );
-            assertThat(
-                exc.getMessage(),
-                containsString(
-                    String.format(
-                        Locale.ROOT,
-                        "The setting [%s] is unavailable on this cluster",
-                        IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey()
-                    )
-                )
-            );
-        }
     }
 
     public void testRecoverySourceWithSyntheticSource() throws IOException {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java

@@ -596,8 +596,6 @@ static Map<String, Settings> getSSLSettingsMap(Settings settings) {
         sslSettingsMap.put(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX, settings.getByPrefix(WatcherField.EMAIL_NOTIFICATION_SSL_PREFIX));
         sslSettingsMap.put(XPackSettings.TRANSPORT_SSL_PREFIX, settings.getByPrefix(XPackSettings.TRANSPORT_SSL_PREFIX));
         sslSettingsMap.putAll(getTransportProfileSSLSettings(settings));
-        // Mount Elastic Inference Service (part of the Inference plugin) configuration
-        sslSettingsMap.put("xpack.inference.elastic.http.ssl", settings.getByPrefix("xpack.inference.elastic.http.ssl."));
         // Only build remote cluster server SSL if the port is enabled
         if (REMOTE_CLUSTER_SERVER_ENABLED.get(settings)) {
             sslSettingsMap.put(XPackSettings.REMOTE_CLUSTER_SERVER_SSL_PREFIX, getRemoteClusterServerSslSettings(settings));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/LocalStateCompositeXPackPlugin.java

@@ -623,7 +623,7 @@ public Map<String, IndexStorePlugin.SnapshotCommitSupplier> getSnapshotCommitSup
     }
 
     @SuppressWarnings("unchecked")
-    protected <T> List<T> filterPlugins(Class<T> type) {
+    private <T> List<T> filterPlugins(Class<T> type) {
         return plugins.stream().filter(x -> type.isAssignableFrom(x.getClass())).map(p -> ((T) p)).collect(Collectors.toList());
     }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java

@@ -614,8 +614,7 @@ public void testGetConfigurationByContextName() throws Exception {
             "xpack.security.authc.realms.ldap.realm1.ssl",
             "xpack.security.authc.realms.saml.realm2.ssl",
             "xpack.monitoring.exporters.mon1.ssl",
-            "xpack.monitoring.exporters.mon2.ssl",
-            "xpack.inference.elastic.http.ssl" };
+            "xpack.monitoring.exporters.mon2.ssl" };
 
         assumeTrue("Not enough cipher suites are available to support this test", getCipherSuites.length >= contextNames.length);
 

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml

@@ -152,6 +152,19 @@ setup:
           service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+  - do:
+      catch: "bad_request"
+      connector.put:
+        connector_id: test-connector-test-managed
+        body:
+          index_name: wrong-prefix-index
+          name: my-connector
+          language: pl
+          is_native: true
+          service_type: super-connector
+
 ---
 'Create Connector - Id returned as part of response':
   - do:

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml

@@ -151,3 +151,18 @@ setup:
 
   - match: { index_name: content-search-2-test }
 
+---
+"Update Managed Connector Index Name - Bad Prefix":
+  - do:
+      connector.put:
+        connector_id: test-connector-2
+        body:
+          is_native: true
+          service_type: super-connector
+
+  - do:
+      catch: "bad_request"
+      connector.update_index_name:
+        connector_id: test-connector-2
+        body:
+          index_name: wrong-prefix-search-2-test

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml

@@ -73,3 +73,43 @@ setup:
             field_1: test
             field_2: something
 
+---
+"Update Connector Native - changing connector to Elastic-managed wrong index name":
+
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          is_native: false
+          index_name: super-connector
+
+  - do:
+      catch: "bad_request"
+      connector.update_native:
+        connector_id: test-connector-1
+        body:
+          is_native: true
+
+---
+"Update Connector Native - changing connector to Elastic-managed correct index name":
+
+  - do:
+      connector.put:
+        connector_id: test-connector-1
+        body:
+          is_native: false
+          index_name: content-super-connector
+
+  - do:
+      connector.update_native:
+        connector_id: test-connector-1
+        body:
+          is_native: true
+
+  - match: { result: updated }
+
+  - do:
+      connector.get:
+        connector_id: test-connector-1
+
+  - match: { is_native: true }

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml

@@ -103,6 +103,18 @@ setup:
           service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+  - do:
+      catch: "bad_request"
+      connector.post:
+        body:
+          index_name: wrong-prefix-index
+          name: my-connector
+          language: pl
+          is_native: true
+          service_type: super-connector
+
 ---
 'Create Connector - Index name used by another connector':
   - do: