Merge branch 'main' into feature/timestamp-and-hostname-sparse-index

docs/changelog/120617.yaml

@@ -0,0 +1,5 @@
+pr: 120617
+summary: Fix queries with document level security on lookup indexes
+area: ES|QL
+type: bug
+issues: [120509]

docs/changelog/120781.yaml

@@ -0,0 +1,5 @@
+pr: 120781
+summary: Add back `keep_alive` to `async_search.submit` rest-api-spec
+area: Search
+type: bug
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json

@@ -43,6 +43,11 @@
         "description":"Control whether the response should be stored in the cluster if it completed within the provided [wait_for_completion] time (default: false)",
         "default":false
       },
+      "keep_alive": {
+        "type": "time",
+        "description": "Update the time interval in which the results (partial or final) for this search will be available",
+        "default": "5d"
+      },
       "batched_reduce_size":{
         "type":"number",
         "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.",

x-pack/plugin/build.gradle

@@ -96,5 +96,11 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
   task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
   task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed")
+  task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now")
+  task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now")
+  task.skipTest("esql/190_lookup_join/alias-repeated-index", "LOOKUP JOIN does not support index aliases for now")
+  task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now")
+  task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now")
+
 })
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java

@@ -196,6 +196,9 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion(
     public static final String APM_ORIGIN = "apm";
     public static final String OTEL_ORIGIN = "otel";
     public static final String REINDEX_DATA_STREAM_ORIGIN = "reindex_data_stream";
+    // TODO consolidate the Kibana origin with the one defined in org/elasticsearch/kibana/KibanaPlugin.java
+    public static final String KIBANA_ORIGIN = "kibana";
+    public static final String CLOUD_ORIGIN = "cloud";
 
     private ClientHelper() {}
 

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-sync-jobs-mappings.json → x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors-sync-jobs.json

@@ -1,12 +1,16 @@
 {
-  "template": {
-    "aliases": {
-      ".elastic-connectors-sync-jobs":  {}
-    },
-    "mappings": {
-      "dynamic": "false",
+  "settings": {
+    "index": {
+      "number_of_shards": "1",
+      "auto_expand_replicas": "0-1"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": "strict",
       "_meta": {
-        "version": ${xpack.application.connector.template.version}
+        "version": "${elastic-connectors-sync-jobs.version}",
+        "managed_index_mappings_version": ${elastic-connectors-sync-jobs.managed.index.version}
       },
       "properties": {
         "cancelation_requested_at": {
@@ -21,9 +25,11 @@
         "connector": {
           "properties": {
             "configuration": {
+              "dynamic": "false",
               "type": "object"
             },
             "filtering": {
+              "dynamic": "false",
               "properties": {
                 "advanced_snippet": {
                   "properties": {
@@ -91,6 +97,7 @@
               "type": "keyword"
             },
             "pipeline": {
+              "dynamic": "false",
               "properties": {
                 "extract_binary_content": {
                   "type": "boolean"
@@ -110,6 +117,7 @@
               "type": "keyword"
             },
             "sync_cursor": {
+              "dynamic": "false",
               "type": "object"
             }
           }
@@ -136,6 +144,7 @@
           "type": "date"
         },
         "metadata": {
+          "dynamic": "false",
           "type": "object"
         },
         "started_at": {
@@ -155,10 +164,5 @@
         }
       }
     }
-  },
-  "_meta": {
-    "description": "Built-in mappings applied by default to elastic-connectors indices",
-    "managed": true
-  },
-  "version": ${xpack.application.connector.template.version}
+  }
 }

x-pack/plugin/core/template-resources/src/main/resources/entsearch/connector/elastic-connectors-mappings.json → x-pack/plugin/core/template-resources/src/main/resources/elastic-connectors.json

@@ -1,36 +1,43 @@
 {
-  "template": {
-    "aliases": {
-      ".elastic-connectors":  {}
-    },
-    "mappings": {
-      "dynamic": "false",
+  "settings": {
+    "index": {
+      "number_of_shards": "1",
+      "auto_expand_replicas": "0-1"
+    }
+  },
+  "mappings": {
+    "_doc": {
+      "dynamic": "strict",
       "_meta": {
-        "pipeline": {
-          "default_name": "search-default-ingestion",
-          "default_extract_binary_content": true,
-          "default_run_ml_inference": true,
-          "default_reduce_whitespace": true
-        },
-        "version": ${xpack.application.connector.template.version}
+        "version": "${elastic-connectors.version}",
+        "managed_index_mappings_version": ${elastic-connectors.managed.index.version}
       },
       "properties": {
         "api_key_id": {
           "type": "keyword"
         },
+        "api_key_secret_id": {
+          "type": "keyword"
+        },
         "configuration": {
+          "dynamic": "false",
           "type": "object"
         },
         "custom_scheduling": {
+          "dynamic": "false",
           "type": "object"
         },
+        "deleted": {
+          "type": "boolean"
+        },
         "description": {
           "type": "text"
         },
         "error": {
           "type": "keyword"
         },
         "features": {
+          "dynamic": "false",
           "properties": {
             "filtering_advanced_config": {
               "type": "boolean"
@@ -66,6 +73,7 @@
           }
         },
         "filtering": {
+          "dynamic": "false",
           "properties": {
             "active": {
               "properties": {
@@ -78,6 +86,7 @@
                       "type": "date"
                     },
                     "value": {
+                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -143,6 +152,7 @@
                       "type": "date"
                     },
                     "value": {
+                      "dynamic": "false",
                       "type": "object"
                     }
                   }
@@ -242,6 +252,7 @@
           "type": "keyword"
         },
         "pipeline": {
+          "dynamic": "false",
           "properties": {
             "extract_binary_content": {
               "type": "boolean"
@@ -258,6 +269,7 @@
           }
         },
         "scheduling": {
+          "dynamic": "false",
           "properties": {
             "access_control": {
               "properties": {
@@ -298,22 +310,13 @@
           "type": "keyword"
         },
         "sync_cursor": {
+          "dynamic": "false",
           "type": "object"
         },
         "sync_now": {
           "type": "boolean"
-        },
-        "deleted": {
-          "type": "boolean"
         }
       }
     }
-  },
-  "_meta": {
-    "description": "Built-in mappings applied by default to elastic-connectors indices",
-    "managed": true
-  },
-  "version": ${xpack.application.connector.template.version}
+  }
 }
-
-

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.xpack.application.analytics.action.TransportPutAnalyticsCollectionAction;
 import org.elasticsearch.xpack.application.analytics.ingest.AnalyticsEventIngestConfig;
 import org.elasticsearch.xpack.application.connector.ConnectorAPIFeature;
+import org.elasticsearch.xpack.application.connector.ConnectorIndexService;
 import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry;
 import org.elasticsearch.xpack.application.connector.action.DeleteConnectorAction;
 import org.elasticsearch.xpack.application.connector.action.GetConnectorAction;
@@ -124,6 +125,7 @@
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportGetConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPostConnectorSecretAction;
 import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction;
+import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction;
 import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction;
@@ -477,7 +479,12 @@ public Collection<?> createComponents(PluginServices services) {
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         Collection<SystemIndexDescriptor> systemIndices = new ArrayList<>(
-            List.of(SearchApplicationIndexService.getSystemIndexDescriptor(), QueryRulesIndexService.getSystemIndexDescriptor())
+            List.of(
+                SearchApplicationIndexService.getSystemIndexDescriptor(),
+                QueryRulesIndexService.getSystemIndexDescriptor(),
+                ConnectorSyncJobIndexService.getSystemIndexDescriptor(),
+                ConnectorIndexService.getSystemIndexDescriptor()
+            )
         );
 
         if (ConnectorSecretsFeature.isEnabled()) {