From b5157516fdbaba5d663f43a5bc05d0232fd48f21 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 23 Nov 2023 16:48:19 +0100 Subject: [PATCH 01/35] Remove explicit SearchResponse references from `org.elasticsearch.index` (#102534) --- .../elasticsearch/index/FinalPipelineIT.java | 30 ++++---- .../elasticsearch/index/HiddenIndexIT.java | 53 +++++++------ .../index/engine/MaxDocsLimitIT.java | 44 +++++------ .../mapper/CopyToMapperIntegrationIT.java | 27 +++---- .../mapper/MultiFieldsIntegrationIT.java | 25 +++---- .../index/shard/SearchIdleIT.java | 49 ++++++------ .../index/store/ExceptionRetryIT.java | 75 ++++++++++--------- .../index/suggest/stats/SuggestStatsIT.java | 20 +++-- 8 files changed, 167 insertions(+), 156 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java index dbf73b4ceeedb..443d0c384a058 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -39,6 +38,7 @@ import java.util.Objects; import java.util.function.BiConsumer; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; @@ -123,9 +123,10 @@ public void testFinalPipelineOfOldDestinationIsNotInvoked() { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); assertEquals(RestStatus.CREATED, indexResponse.status()); - SearchResponse target = prepareSearch("target").get(); - assertEquals(1, target.getHits().getTotalHits().value); - assertFalse(target.getHits().getAt(0).getSourceAsMap().containsKey("final")); + assertResponse(prepareSearch("target"), response -> { + assertEquals(1, response.getHits().getTotalHits().value); + assertFalse(response.getHits().getAt(0).getSourceAsMap().containsKey("final")); + }); } public void testFinalPipelineOfNewDestinationIsInvoked() { @@ -148,9 +149,10 @@ public void testFinalPipelineOfNewDestinationIsInvoked() { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); assertEquals(RestStatus.CREATED, indexResponse.status()); - SearchResponse target = prepareSearch("target").get(); - assertEquals(1, target.getHits().getTotalHits().value); - assertEquals(true, target.getHits().getAt(0).getSourceAsMap().get("final")); + assertResponse(prepareSearch("target"), response -> { + assertEquals(1, response.getHits().getTotalHits().value); + assertEquals(true, response.getHits().getAt(0).getSourceAsMap().get("final")); + }); } public void testDefaultPipelineOfNewDestinationIsNotInvoked() { @@ -173,9 +175,10 @@ public void testDefaultPipelineOfNewDestinationIsNotInvoked() { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); assertEquals(RestStatus.CREATED, indexResponse.status()); - SearchResponse target = prepareSearch("target").get(); - assertEquals(1, target.getHits().getTotalHits().value); - assertFalse(target.getHits().getAt(0).getSourceAsMap().containsKey("final")); + assertResponse(prepareSearch("target"), response -> { + assertEquals(1, response.getHits().getTotalHits().value); + assertFalse(response.getHits().getAt(0).getSourceAsMap().containsKey("final")); + }); } public void testDefaultPipelineOfRerouteDestinationIsInvoked() { @@ -198,9 +201,10 @@ public void testDefaultPipelineOfRerouteDestinationIsInvoked() { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); assertEquals(RestStatus.CREATED, indexResponse.status()); - SearchResponse target = prepareSearch("target").get(); - assertEquals(1, target.getHits().getTotalHits().value); - assertTrue(target.getHits().getAt(0).getSourceAsMap().containsKey("final")); + assertResponse(prepareSearch("target"), response -> { + assertEquals(1, response.getHits().getTotalHits().value); + assertTrue(response.getHits().getAt(0).getSourceAsMap().containsKey("final")); + }); } public void testAvoidIndexingLoop() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index a844e878d0ed7..14d9cf9e56eae 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.metadata.MappingMetadata; @@ -27,6 +26,7 @@ import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -40,37 +40,48 @@ public void testHiddenIndexSearch() { prepareIndex("hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); // default not visible to wildcard expansion - SearchResponse searchResponse = prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000) - .setQuery(QueryBuilders.matchAllQuery()) - .get(); - boolean matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); - assertFalse(matchedHidden); + assertResponse( + prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()), + response -> { + boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); + assertFalse(matchedHidden); + } + ); // direct access allowed - searchResponse = prepareSearch("hidden-index").setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get(); - matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); - assertTrue(matchedHidden); + assertResponse(prepareSearch("hidden-index").setSize(1000).setQuery(QueryBuilders.matchAllQuery()), response -> { + boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); + assertTrue(matchedHidden); + }); // with indices option to include hidden - searchResponse = prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000) - .setQuery(QueryBuilders.matchAllQuery()) - .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN) - .get(); - matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); - assertTrue(matchedHidden); + assertResponse( + prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000) + .setQuery(QueryBuilders.matchAllQuery()) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN), + response -> { + boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); + assertTrue(matchedHidden); + } + ); // implicit based on use of pattern starting with . and a wildcard assertAcked(indicesAdmin().prepareCreate(".hidden-index").setSettings(Settings.builder().put("index.hidden", true).build()).get()); prepareIndex(".hidden-index").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); - searchResponse = prepareSearch(randomFrom(".*", ".hidden-*")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get(); - matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> ".hidden-index".equals(hit.getIndex())); - assertTrue(matchedHidden); + assertResponse(prepareSearch(randomFrom(".*", ".hidden-*")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()), response -> { + boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> ".hidden-index".equals(hit.getIndex())); + assertTrue(matchedHidden); + }); // make index not hidden updateIndexSettings(Settings.builder().put("index.hidden", false), "hidden-index"); - searchResponse = prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()).get(); - matchedHidden = Arrays.stream(searchResponse.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); - assertTrue(matchedHidden); + assertResponse( + prepareSearch(randomFrom("*", "_all", "h*", "*index")).setSize(1000).setQuery(QueryBuilders.matchAllQuery()), + response -> { + boolean matchedHidden = Arrays.stream(response.getHits().getHits()).anyMatch(hit -> "hidden-index".equals(hit.getIndex())); + assertTrue(matchedHidden); + } + ); } public void testGlobalTemplatesDoNotApply() { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java index 1d256ddb3038a..ee165d1870571 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/engine/MaxDocsLimitIT.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.engine; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -21,7 +20,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.elasticsearch.xcontent.XContentType; import org.junit.After; import org.junit.Before; @@ -32,6 +30,8 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -108,24 +108,20 @@ public void testMaxDocsLimit() throws Exception { ); assertThat(deleteError.getMessage(), containsString("Number of documents in the index can't exceed [" + maxDocs.get() + "]")); indicesAdmin().prepareRefresh("test").get(); - SearchResponse searchResponse = prepareSearch("test").setQuery(new MatchAllQueryBuilder()) - .setTrackTotalHitsUpTo(Integer.MAX_VALUE) - .setSize(0) - .get(); - ElasticsearchAssertions.assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) maxDocs.get())); + assertNoFailuresAndResponse( + prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0), + response -> assertThat(response.getHits().getTotalHits().value, equalTo((long) maxDocs.get())) + ); if (randomBoolean()) { indicesAdmin().prepareFlush("test").get(); } internalCluster().fullRestart(); internalCluster().ensureAtLeastNumDataNodes(2); ensureGreen("test"); - searchResponse = prepareSearch("test").setQuery(new MatchAllQueryBuilder()) - .setTrackTotalHitsUpTo(Integer.MAX_VALUE) - .setSize(0) - .get(); - ElasticsearchAssertions.assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) maxDocs.get())); + assertNoFailuresAndResponse( + prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0), + response -> assertThat(response.getHits().getTotalHits().value, equalTo((long) maxDocs.get())) + ); } public void testMaxDocsLimitConcurrently() throws Exception { @@ -135,12 +131,10 @@ public void testMaxDocsLimitConcurrently() throws Exception { assertThat(indexingResult.numFailures, greaterThan(0)); assertThat(indexingResult.numSuccess, both(greaterThan(0)).and(lessThanOrEqualTo(maxDocs.get()))); indicesAdmin().prepareRefresh("test").get(); - SearchResponse searchResponse = prepareSearch("test").setQuery(new MatchAllQueryBuilder()) - .setTrackTotalHitsUpTo(Integer.MAX_VALUE) - .setSize(0) - .get(); - ElasticsearchAssertions.assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) indexingResult.numSuccess)); + assertHitCountAndNoFailures( + prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0), + indexingResult.numSuccess + ); int totalSuccess = indexingResult.numSuccess; while (totalSuccess < maxDocs.get()) { indexingResult = indexDocs(between(1, 10), between(1, 8)); @@ -152,12 +146,10 @@ public void testMaxDocsLimitConcurrently() throws Exception { assertThat(indexingResult.numSuccess, equalTo(0)); } indicesAdmin().prepareRefresh("test").get(); - searchResponse = prepareSearch("test").setQuery(new MatchAllQueryBuilder()) - .setTrackTotalHitsUpTo(Integer.MAX_VALUE) - .setSize(0) - .get(); - ElasticsearchAssertions.assertNoFailures(searchResponse); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo((long) totalSuccess)); + assertHitCountAndNoFailures( + prepareSearch("test").setQuery(new MatchAllQueryBuilder()).setTrackTotalHitsUpTo(Integer.MAX_VALUE).setSize(0), + totalSuccess + ); } record IndexingResult(int numSuccess, int numFailures) {} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java index 89bd60dc4683f..c1f06aeceebde 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/CopyToMapperIntegrationIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -22,6 +21,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; @@ -38,18 +38,19 @@ public void testDynamicTemplateCopyTo() throws Exception { SubAggCollectionMode aggCollectionMode = randomFrom(SubAggCollectionMode.values()); - SearchResponse response = prepareSearch("test-idx").setQuery(QueryBuilders.termQuery("even", true)) - .addAggregation(AggregationBuilders.terms("test").field("test_field").size(recordCount * 2).collectMode(aggCollectionMode)) - .addAggregation( - AggregationBuilders.terms("test_raw").field("test_field_raw").size(recordCount * 2).collectMode(aggCollectionMode) - ) - .get(); - - assertThat(response.getHits().getTotalHits().value, equalTo((long) recordCount)); - - assertThat(((Terms) response.getAggregations().get("test")).getBuckets().size(), equalTo(recordCount + 1)); - assertThat(((Terms) response.getAggregations().get("test_raw")).getBuckets().size(), equalTo(recordCount)); - + assertResponse( + prepareSearch("test-idx").setQuery(QueryBuilders.termQuery("even", true)) + .addAggregation(AggregationBuilders.terms("test").field("test_field").size(recordCount * 2).collectMode(aggCollectionMode)) + .addAggregation( + AggregationBuilders.terms("test_raw").field("test_field_raw").size(recordCount * 2).collectMode(aggCollectionMode) + ), + response -> { + assertThat(response.getHits().getTotalHits().value, equalTo((long) recordCount)); + + assertThat(((Terms) response.getAggregations().get("test")).getBuckets().size(), equalTo(recordCount + 1)); + assertThat(((Terms) response.getAggregations().get("test_raw")).getBuckets().size(), equalTo(recordCount)); + } + ); } public void testDynamicObjectCopyTo() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index 8e94502cfb63e..a22910ab9c4eb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.unit.DistanceUnit; @@ -48,8 +47,8 @@ public void testMultiFields() throws Exception { prepareIndex("my-index").setId("1").setSource("title", "Multi fields").setRefreshPolicy(IMMEDIATE).get(); - assertHitCount(client().prepareSearch("my-index").setQuery(matchQuery("title", "multi")), 1); - assertHitCount(client().prepareSearch("my-index").setQuery(matchQuery("title.not_analyzed", "Multi fields")), 1); + assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title", "multi")), 1); + assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title.not_analyzed", "Multi fields")), 1); assertAcked(indicesAdmin().preparePutMapping("my-index").setSource(createPutMappingSource())); @@ -67,7 +66,7 @@ public void testMultiFields() throws Exception { prepareIndex("my-index").setId("1").setSource("title", "Multi fields").setRefreshPolicy(IMMEDIATE).get(); - assertHitCount(client().prepareSearch("my-index").setQuery(matchQuery("title.uncased", "Multi")), 1); + assertHitCount(prepareSearch("my-index").setQuery(matchQuery("title.uncased", "Multi")), 1); } @SuppressWarnings("unchecked") @@ -90,12 +89,12 @@ public void testGeoPointMultiField() throws Exception { GeoPoint point = new GeoPoint(51, 19); prepareIndex("my-index").setId("1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get(); - SearchResponse countResponse = prepareSearch("my-index").setSize(0) - .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))) - .get(); - assertThat(countResponse.getHits().getTotalHits().value, equalTo(1L)); - countResponse = prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())).get(); - assertThat(countResponse.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount( + prepareSearch("my-index").setSize(0) + .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS))), + 1L + ); + assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", point.geohash())), 1L); } @SuppressWarnings("unchecked") @@ -116,8 +115,7 @@ public void testCompletionMultiField() throws Exception { assertThat(bField.get("type").toString(), equalTo("keyword")); prepareIndex("my-index").setId("1").setSource("a", "complete me").setRefreshPolicy(IMMEDIATE).get(); - SearchResponse countResponse = prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")).get(); - assertThat(countResponse.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")), 1L); } @SuppressWarnings("unchecked") @@ -138,8 +136,7 @@ public void testIpMultiField() throws Exception { assertThat(bField.get("type").toString(), equalTo("keyword")); prepareIndex("my-index").setId("1").setSource("a", "127.0.0.1").setRefreshPolicy(IMMEDIATE).get(); - SearchResponse countResponse = prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")).get(); - assertThat(countResponse.getHits().getTotalHits().value, equalTo(1L)); + assertHitCount(prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")), 1L); } private XContentBuilder createMappingSource(String fieldType) throws IOException { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java index 09b91813d6cb6..e1ab2bdc2369e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -43,6 +42,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.equalTo; public class SearchIdleIT extends ESSingleNodeTestCase { @@ -304,18 +304,20 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException { assertIdleShard(activeIndexStatsBefore); // WHEN - final SearchResponse searchResponse = client().prepareSearch("test*") - .setQuery(new RangeQueryBuilder("@timestamp").from("2021-05-12T20:00:00.000Z").to("2021-05-12T21:00:00.000Z")) - .setPreFilterShardSize(5) - .get(); - - // THEN - assertEquals(RestStatus.OK, searchResponse.status()); - assertEquals(idleIndexShardsCount + activeIndexShardsCount - 1, searchResponse.getSkippedShards()); - assertEquals(0, searchResponse.getFailedShards()); - Arrays.stream(searchResponse.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex())); - // NOTE: we need an empty result from at least one shard - assertEquals(1, searchResponse.getHits().getHits().length); + assertResponse( + client().prepareSearch("test*") + .setQuery(new RangeQueryBuilder("@timestamp").from("2021-05-12T20:00:00.000Z").to("2021-05-12T21:00:00.000Z")) + .setPreFilterShardSize(5), + response -> { + // THEN + assertEquals(RestStatus.OK, response.status()); + assertEquals(idleIndexShardsCount + activeIndexShardsCount - 1, response.getSkippedShards()); + assertEquals(0, response.getFailedShards()); + Arrays.stream(response.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex())); + // NOTE: we need an empty result from at least one shard + assertEquals(1, response.getHits().getHits().length); + } + ); final IndicesStatsResponse idleIndexStatsAfter = indicesAdmin().prepareStats(idleIndex).get(); assertIdleShardsRefreshStats(idleIndexStatsBefore, idleIndexStatsAfter); } @@ -366,18 +368,15 @@ public void testSearchIdleExistsQueryMatchOneIndex() throws InterruptedException assertIdleShard(activeIndexStatsBefore); // WHEN - final SearchResponse searchResponse = client().prepareSearch("test*") - .setQuery(new ExistsQueryBuilder("unmapped")) - .setPreFilterShardSize(5) - .get(); - - // THEN - assertEquals(RestStatus.OK, searchResponse.status()); - assertEquals(idleIndexShardsCount, searchResponse.getSkippedShards()); - assertEquals(0, searchResponse.getFailedShards()); - Arrays.stream(searchResponse.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex())); - // NOTE: we need an empty result from at least one shard - assertEquals(1, searchResponse.getHits().getHits().length); + assertResponse(client().prepareSearch("test*").setQuery(new ExistsQueryBuilder("unmapped")).setPreFilterShardSize(5), response -> { + // THEN + assertEquals(RestStatus.OK, response.status()); + assertEquals(idleIndexShardsCount, response.getSkippedShards()); + assertEquals(0, response.getFailedShards()); + Arrays.stream(response.getHits().getHits()).forEach(searchHit -> assertEquals("test2", searchHit.getIndex())); + // NOTE: we need an empty result from at least one shard + assertEquals(1, response.getHits().getHits().length); + }); final IndicesStatsResponse idleIndexStatsAfter = indicesAdmin().prepareStats(idleIndex).get(); assertIdleShardsRefreshStats(idleIndexStatsBefore, idleIndexStatsAfter); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java index 19efcd9e3f31f..423e5c14c472a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/ExceptionRetryIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.TransportShardBulkAction; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.plugins.Plugin; @@ -39,7 +38,8 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -94,9 +94,9 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I bulkBuilder.add(client.prepareIndex("index").setSource(doc)); } - BulkResponse response = bulkBuilder.get(); - if (response.hasFailures()) { - for (BulkItemResponse singleIndexRespons : response.getItems()) { + BulkResponse bulkResponse = bulkBuilder.get(); + if (bulkResponse.hasFailures()) { + for (BulkItemResponse singleIndexRespons : bulkResponse.getItems()) { if (singleIndexRespons.isFailed()) { fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage()); } @@ -104,41 +104,42 @@ public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, I } refresh(); - SearchResponse searchResponse = prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get(); - - Set uniqueIds = new HashSet<>(); - long dupCounter = 0; - boolean found_duplicate_already = false; - for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { - if (uniqueIds.add(searchResponse.getHits().getHits()[i].getId()) == false) { - if (found_duplicate_already == false) { - SearchResponse dupIdResponse = prepareSearch("index").setQuery( - termQuery("_id", searchResponse.getHits().getHits()[i].getId()) - ).setExplain(true).get(); - assertThat(dupIdResponse.getHits().getTotalHits().value, greaterThan(1L)); - logger.info("found a duplicate id:"); - for (SearchHit hit : dupIdResponse.getHits()) { - logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId()); + assertNoFailuresAndResponse(prepareSearch("index").setSize(numDocs * 2).addStoredField("_id"), response -> { + Set uniqueIds = new HashSet<>(); + long dupCounter = 0; + boolean found_duplicate_already = false; + for (int i = 0; i < response.getHits().getHits().length; i++) { + if (uniqueIds.add(response.getHits().getHits()[i].getId()) == false) { + if (found_duplicate_already == false) { + assertResponse( + prepareSearch("index").setQuery(termQuery("_id", response.getHits().getHits()[i].getId())).setExplain(true), + dupIdResponse -> { + assertThat(dupIdResponse.getHits().getTotalHits().value, greaterThan(1L)); + logger.info("found a duplicate id:"); + for (SearchHit hit : dupIdResponse.getHits()) { + logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId()); + } + logger.info("will not print anymore in case more duplicates are found."); + } + ); + found_duplicate_already = true; } - logger.info("will not print anymore in case more duplicates are found."); - found_duplicate_already = true; + dupCounter++; } - dupCounter++; } - } - assertNoFailures(searchResponse); - assertThat(dupCounter, equalTo(0L)); - assertHitCount(searchResponse, numDocs); - IndicesStatsResponse index = indicesAdmin().prepareStats("index").clear().setSegments(true).get(); - IndexStats indexStats = index.getIndex("index"); - long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE; - for (IndexShardStats indexShardStats : indexStats) { - for (ShardStats shardStats : indexShardStats) { - SegmentsStats segments = shardStats.getStats().getSegments(); - maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp()); + assertThat(dupCounter, equalTo(0L)); + assertHitCount(response, numDocs); + IndicesStatsResponse index = indicesAdmin().prepareStats("index").clear().setSegments(true).get(); + IndexStats indexStats = index.getIndex("index"); + long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE; + for (IndexShardStats indexShardStats : indexStats) { + for (ShardStats shardStats : indexShardStats) { + SegmentsStats segments = shardStats.getStats().getSegments(); + maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp()); + } } - } - assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get()); - assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1); + assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get()); + assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1); + }); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java index ec1bdf29991e6..143ffedeefc55 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; @@ -28,6 +27,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -66,16 +66,22 @@ public void testSimpleStats() throws Exception { long startTime = System.currentTimeMillis(); for (int i = 0; i < suggestAllIdx; i++) { - SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch(), i).get(); - assertAllSuccessful(suggestResponse); + assertResponse( + addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch(), i), + response -> assertAllSuccessful(response) + ); } for (int i = 0; i < suggestIdx1; i++) { - SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test1"), i).get(); - assertAllSuccessful(suggestResponse); + assertResponse( + addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test1"), i), + response -> assertAllSuccessful(response) + ); } for (int i = 0; i < suggestIdx2; i++) { - SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test2"), i).get(); - assertAllSuccessful(suggestResponse); + assertResponse( + addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test2"), i), + response -> assertAllSuccessful(response) + ); } long endTime = System.currentTimeMillis(); From 91d97144ce26d70f2e137cfc1947afbca17d1ebf Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Thu, 23 Nov 2023 16:56:16 +0100 Subject: [PATCH 02/35] Fix connector index pattern version name (#102543) --- .../application/connector/ConnectorTemplateRegistry.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index 54c7b9999bf8a..642295061d17a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -36,14 +36,14 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { public static final NodeFeature CONNECTOR_TEMPLATES_FEATURE = new NodeFeature("elastic-connectors.templates"); // This number must be incremented when we make changes to built-in templates. - static final int REGISTRY_VERSION = 2; + static final int REGISTRY_VERSION = 3; // Connector indices constants - public static final String CONNECTOR_INDEX_NAME_PATTERN = ".elastic-connectors-v" + REGISTRY_VERSION; + public static final String CONNECTOR_INDEX_NAME_PATTERN = ".elastic-connectors-v1"; public static final String CONNECTOR_TEMPLATE_NAME = "elastic-connectors"; - public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v" + REGISTRY_VERSION; + public static final String CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN = ".elastic-connectors-sync-jobs-v1"; public static final String CONNECTOR_SYNC_JOBS_TEMPLATE_NAME = "elastic-connectors-sync-jobs"; public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*"; From d835ebfc65bd1f5e4f7518d3987c09772de006a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Thu, 23 Nov 2023 16:56:57 +0100 Subject: [PATCH 03/35] [Transform] Implement exponential backoff for transform state persistence retrying (#102512) --- docs/changelog/102512.yaml | 6 ++ .../transforms/ClientTransformIndexer.java | 2 +- .../transforms/TransformFailureHandler.java | 7 +- .../transforms/TransformIndexer.java | 16 +++-- .../scheduling/TransformScheduledTask.java | 35 +--------- .../scheduling/TransformSchedulingUtils.java | 46 +++++++++++++ ...IndexerFailureOnStatePersistenceTests.java | 1 - .../TransformScheduledTaskTests.java | 49 -------------- .../TransformSchedulingUtilsTests.java | 66 +++++++++++++++++++ 9 files changed, 137 insertions(+), 91 deletions(-) create mode 100644 docs/changelog/102512.yaml create mode 100644 x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulingUtils.java create mode 100644 x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulingUtilsTests.java diff --git a/docs/changelog/102512.yaml b/docs/changelog/102512.yaml new file mode 100644 index 0000000000000..d4bc765ecaf5f --- /dev/null +++ b/docs/changelog/102512.yaml @@ -0,0 +1,6 @@ +pr: 102512 +summary: Implement exponential backoff for transform state persistence retrying +area: Transform +type: enhancement +issues: + - 102528 diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index affa44e4a38f0..f50d12f58ec6b 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -357,7 +357,7 @@ protected void persistState(TransformState state, ActionListener listener) + statsExc.getMessage() ); - if (failureHandler.handleStatePersistenceFailure(statsExc, getConfig().getSettings()) == false) { + if (failureHandler.handleStatePersistenceFailure(statsExc, getConfig().getSettings())) { // get the current seqNo and primary term, however ignore the stored state transformsConfigManager.getTransformStoredDoc( transformConfig.getId(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java index 7354e588c7dbd..c7e0eda5ca5e6 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformFailureHandler.java @@ -93,22 +93,23 @@ void handleIndexerFailure(Exception exception, SettingsConfig settingsConfig) { * * @param e the exception caught * @param settingsConfig The settings + * @return true if there is at least one more retry to be made, false otherwise */ boolean handleStatePersistenceFailure(Exception e, SettingsConfig settingsConfig) { // we use the same setting for retries, however a separate counter, because the failure // counter for search/index gets reset after a successful bulk index request int numFailureRetries = getNumFailureRetries(settingsConfig); - final int failureCount = context.incrementAndGetStatePersistenceFailureCount(e); + int failureCount = context.incrementAndGetStatePersistenceFailureCount(e); if (numFailureRetries != -1 && failureCount > numFailureRetries) { fail( e, "task encountered more than " + numFailureRetries + " failures updating internal state; latest failure: " + e.getMessage() ); - return true; + return false; } - return false; + return true; } /** diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index f43e1cb91eff2..1c34bc5436a61 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -45,6 +46,7 @@ import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.Function.ChangeCollector; import org.elasticsearch.xpack.transform.transforms.RetentionPolicyToDeleteByQueryRequestConverter.RetentionPolicyException; +import org.elasticsearch.xpack.transform.transforms.scheduling.TransformSchedulingUtils; import java.time.Instant; import java.util.Collection; @@ -83,8 +85,9 @@ private enum RunState { private static final long RETENTION_OF_CHECKPOINTS_MS = 864000000L; // 10 days private static final long CHECKPOINT_CLEANUP_INTERVAL = 100L; // every 100 checkpoints - // constant for triggering state persistence, hardcoded for now - public static final long DEFAULT_TRIGGER_SAVE_STATE_INTERVAL_MS = 60_000; // 60s + // Constant for triggering state persistence, used when there are no state persistence errors. + // In face of errors, exponential backoff scheme is used. + public static final TimeValue DEFAULT_TRIGGER_SAVE_STATE_INTERVAL = TimeValue.timeValueSeconds(60); protected final TransformConfigManager transformsConfigManager; private final CheckpointProvider checkpointProvider; @@ -189,8 +192,13 @@ protected boolean triggerSaveState() { if (saveStateListeners.get() != null) { return true; } - - return TimeUnit.NANOSECONDS.toMillis(getTimeNanos()) > lastSaveStateMilliseconds + DEFAULT_TRIGGER_SAVE_STATE_INTERVAL_MS; + long currentTimeMilliseconds = TimeUnit.NANOSECONDS.toMillis(getTimeNanos()); + long nextSaveStateMilliseconds = TransformSchedulingUtils.calculateNextScheduledTime( + lastSaveStateMilliseconds, + DEFAULT_TRIGGER_SAVE_STATE_INTERVAL, + context.getStatePersistenceFailureCount() + ); + return currentTimeMilliseconds > nextSaveStateMilliseconds; } public TransformConfig getConfig() { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduledTask.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduledTask.java index 88405b3e76b79..f2d7b3a9aac70 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduledTask.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformScheduledTask.java @@ -10,9 +10,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.transform.Transform; -import java.time.Duration; import java.util.Objects; +import static org.elasticsearch.xpack.transform.transforms.scheduling.TransformSchedulingUtils.calculateNextScheduledTime; + /** * {@link TransformScheduledTask} is a structure describing the scheduled task in the queue. *

@@ -20,15 +21,6 @@ */ final class TransformScheduledTask { - /** - * Minimum delay that can be applied after a failure. - */ - private static final long MIN_DELAY_MILLIS = Duration.ofSeconds(5).toMillis(); - /** - * Maximum delay that can be applied after a failure. - */ - private static final long MAX_DELAY_MILLIS = Duration.ofHours(1).toMillis(); - private final String transformId; private final TimeValue frequency; private final Long lastTriggeredTimeMillis; @@ -69,29 +61,6 @@ final class TransformScheduledTask { ); } - // Visible for testing - - /** - * Calculates the appropriate next scheduled time taking number of failures into account. - * This method implements exponential backoff approach. - * - * @param lastTriggeredTimeMillis the last time (in millis) the task was triggered - * @param frequency the frequency of the transform - * @param failureCount the number of failures that happened since the task was triggered - * @return next scheduled time for a task - */ - static long calculateNextScheduledTime(Long lastTriggeredTimeMillis, TimeValue frequency, int failureCount) { - final long baseTime = lastTriggeredTimeMillis != null ? lastTriggeredTimeMillis : System.currentTimeMillis(); - - if (failureCount == 0) { - return baseTime + (frequency != null ? frequency : Transform.DEFAULT_TRANSFORM_FREQUENCY).millis(); - } - - // Math.min(failureCount, 32) is applied in order to avoid overflow. - long delayMillis = Math.min(Math.max((1L << Math.min(failureCount, 32)) * 1000, MIN_DELAY_MILLIS), MAX_DELAY_MILLIS); - return baseTime + delayMillis; - } - String getTransformId() { return transformId; } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulingUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulingUtils.java new file mode 100644 index 0000000000000..180fbb4d48522 --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/scheduling/TransformSchedulingUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.transforms.scheduling; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xpack.transform.Transform; + +import java.time.Duration; + +public final class TransformSchedulingUtils { + + /** + * Minimum delay that can be applied after a failure. + */ + private static final long MIN_DELAY_MILLIS = Duration.ofSeconds(5).toMillis(); + /** + * Maximum delay that can be applied after a failure. + */ + private static final long MAX_DELAY_MILLIS = Duration.ofHours(1).toMillis(); + + /** + * Calculates the appropriate next scheduled time taking number of failures into account. + * This method implements exponential backoff approach. + * + * @param lastTriggeredTimeMillis the last time (in millis) the task was triggered + * @param frequency the frequency of the transform + * @param failureCount the number of failures that happened since the task was triggered + * @return next scheduled time for a task + */ + public static long calculateNextScheduledTime(Long lastTriggeredTimeMillis, TimeValue frequency, int failureCount) { + final long baseTime = lastTriggeredTimeMillis != null ? lastTriggeredTimeMillis : System.currentTimeMillis(); + + if (failureCount == 0) { + return baseTime + (frequency != null ? frequency : Transform.DEFAULT_TRANSFORM_FREQUENCY).millis(); + } + + // Math.min(failureCount, 32) is applied in order to avoid overflow. + long delayMillis = Math.min(Math.max((1L << Math.min(failureCount, 32)) * 1000, MIN_DELAY_MILLIS), MAX_DELAY_MILLIS); + return baseTime + delayMillis; + } +} diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java index dba954994f9a3..55ae653c39629 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureOnStatePersistenceTests.java @@ -281,7 +281,6 @@ public void fail(Throwable exception, String failureMessage, ActionListener(PutAnalyticsCollectionAction.INSTANCE, TransportPutAnalyticsCollectionAction.class), - new ActionHandler<>(GetAnalyticsCollectionAction.INSTANCE, TransportGetAnalyticsCollectionAction.class), - new ActionHandler<>(DeleteAnalyticsCollectionAction.INSTANCE, TransportDeleteAnalyticsCollectionAction.class), - new ActionHandler<>(PostAnalyticsEventAction.INSTANCE, TransportPostAnalyticsEventAction.class), - - // Search Applications - new ActionHandler<>(DeleteSearchApplicationAction.INSTANCE, TransportDeleteSearchApplicationAction.class), - new ActionHandler<>(GetSearchApplicationAction.INSTANCE, TransportGetSearchApplicationAction.class), - new ActionHandler<>(ListSearchApplicationAction.INSTANCE, TransportListSearchApplicationAction.class), - new ActionHandler<>(PutSearchApplicationAction.INSTANCE, TransportPutSearchApplicationAction.class), - new ActionHandler<>(QuerySearchApplicationAction.INSTANCE, TransportQuerySearchApplicationAction.class), - new ActionHandler<>(RenderSearchApplicationQueryAction.INSTANCE, TransportRenderSearchApplicationQueryAction.class), - - // Query rules - new ActionHandler<>(DeleteQueryRulesetAction.INSTANCE, TransportDeleteQueryRulesetAction.class), - new ActionHandler<>(GetQueryRulesetAction.INSTANCE, TransportGetQueryRulesetAction.class), - new ActionHandler<>(ListQueryRulesetsAction.INSTANCE, TransportListQueryRulesetsAction.class), - new ActionHandler<>(PutQueryRulesetAction.INSTANCE, TransportPutQueryRulesetAction.class), - usageAction, - infoAction + List> actionHandlers = new ArrayList<>( + List.of( + // Behavioral Analytics + new ActionHandler<>(PutAnalyticsCollectionAction.INSTANCE, TransportPutAnalyticsCollectionAction.class), + new ActionHandler<>(GetAnalyticsCollectionAction.INSTANCE, TransportGetAnalyticsCollectionAction.class), + new ActionHandler<>(DeleteAnalyticsCollectionAction.INSTANCE, TransportDeleteAnalyticsCollectionAction.class), + new ActionHandler<>(PostAnalyticsEventAction.INSTANCE, TransportPostAnalyticsEventAction.class), + + // Search Applications + new ActionHandler<>(DeleteSearchApplicationAction.INSTANCE, TransportDeleteSearchApplicationAction.class), + new ActionHandler<>(GetSearchApplicationAction.INSTANCE, TransportGetSearchApplicationAction.class), + new ActionHandler<>(ListSearchApplicationAction.INSTANCE, TransportListSearchApplicationAction.class), + new ActionHandler<>(PutSearchApplicationAction.INSTANCE, TransportPutSearchApplicationAction.class), + new ActionHandler<>(QuerySearchApplicationAction.INSTANCE, TransportQuerySearchApplicationAction.class), + new ActionHandler<>(RenderSearchApplicationQueryAction.INSTANCE, TransportRenderSearchApplicationQueryAction.class), + + // Query rules + new ActionHandler<>(DeleteQueryRulesetAction.INSTANCE, TransportDeleteQueryRulesetAction.class), + new ActionHandler<>(GetQueryRulesetAction.INSTANCE, TransportGetQueryRulesetAction.class), + new ActionHandler<>(ListQueryRulesetsAction.INSTANCE, TransportListQueryRulesetsAction.class), + new ActionHandler<>(PutQueryRulesetAction.INSTANCE, TransportPutQueryRulesetAction.class), + + usageAction, + infoAction + ) ); + + // Connectors + if (ConnectorAPIFeature.isEnabled()) { + actionHandlers.add(new ActionHandler<>(PutConnectorAction.INSTANCE, TransportPutConnectorAction.class)); + } + + return Collections.unmodifiableList(actionHandlers); } @Override @@ -160,27 +177,36 @@ public List getRestHandlers( return Collections.emptyList(); } - return List.of( - // Behavioral Analytics - new RestPutAnalyticsCollectionAction(getLicenseState()), - new RestGetAnalyticsCollectionAction(getLicenseState()), - new RestDeleteAnalyticsCollectionAction(getLicenseState()), - new RestPostAnalyticsEventAction(getLicenseState()), - - // Search Applications - new RestDeleteSearchApplicationAction(getLicenseState()), - new RestGetSearchApplicationAction(getLicenseState()), - new RestListSearchApplicationAction(getLicenseState()), - new RestPutSearchApplicationAction(getLicenseState()), - new RestQuerySearchApplicationAction(getLicenseState()), - new RestRenderSearchApplicationQueryAction(getLicenseState()), - - // Query rules - new RestDeleteQueryRulesetAction(getLicenseState()), - new RestGetQueryRulesetAction(getLicenseState()), - new RestListQueryRulesetsAction(getLicenseState()), - new RestPutQueryRulesetAction(getLicenseState()) + List restHandlers = new ArrayList<>( + List.of( + // Behavioral Analytics + new RestPutAnalyticsCollectionAction(getLicenseState()), + new RestGetAnalyticsCollectionAction(getLicenseState()), + new RestDeleteAnalyticsCollectionAction(getLicenseState()), + new RestPostAnalyticsEventAction(getLicenseState()), + + // Search Applications + new RestDeleteSearchApplicationAction(getLicenseState()), + new RestGetSearchApplicationAction(getLicenseState()), + new RestListSearchApplicationAction(getLicenseState()), + new RestPutSearchApplicationAction(getLicenseState()), + new RestQuerySearchApplicationAction(getLicenseState()), + new RestRenderSearchApplicationQueryAction(getLicenseState()), + + // Query rules + new RestDeleteQueryRulesetAction(getLicenseState()), + new RestGetQueryRulesetAction(getLicenseState()), + new RestListQueryRulesetsAction(getLicenseState()), + new RestPutQueryRulesetAction(getLicenseState()) + ) ); + + // Connectors + if (ConnectorAPIFeature.isEnabled()) { + restHandlers.add(new RestPutConnectorAction()); + } + + return Collections.unmodifiableList(restHandlers); } @Override diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java new file mode 100644 index 0000000000000..db85b4a076906 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/Connector.java @@ -0,0 +1,476 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Represents a Connector in the Elasticsearch ecosystem. Connectors are used for integrating + * and synchronizing external data sources with Elasticsearch. Each Connector instance encapsulates + * various settings and state information, including: + *

    + *
  • A unique identifier for distinguishing different connectors.
    • + *
  • API key for authenticating with Elasticsearch, ensuring secure access.
    • + *
  • A configuration mapping which holds specific settings and parameters for the connector's operation.
    • + *
  • A {@link ConnectorCustomSchedule} object that defines custom scheduling.
    • + *
  • A description providing an overview or purpose of the connector.
    • + *
  • An error string capturing the latest error encountered during the connector's operation, if any.
    • + *
  • A {@link ConnectorFeatures} object encapsulating the set of features enabled for this connector.
    • + *
  • A list of {@link ConnectorFiltering} objects for applying filtering rules to the data processed by the connector.
    • + *
  • The name of the Elasticsearch index where the synchronized data is stored or managed.
    • + *
  • A boolean flag 'isNative' indicating whether the connector is a native Elasticsearch connector.
    • + *
  • The language associated with the connector.
    • + *
  • A {@link ConnectorSyncInfo} object containing synchronization state and history information.
    • + *
  • The name of the connector.
    • + *
  • A {@link ConnectorIngestPipeline} object specifying the data ingestion pipeline configuration.
    • + *
  • A {@link ConnectorScheduling} object with the scheduling configuration to trigger data sync.
    • + *
  • The type of connector.
    • + *
  • A {@link ConnectorStatus} indicating the current status of the connector.
    • + *
  • A sync cursor, used for incremental syncs.
    • + *
  • A boolean flag 'syncNow', which, when set, triggers an immediate synchronization operation.
    • + *
+ */ +public class Connector implements Writeable, ToXContentObject { + + private final String connectorId; + @Nullable + private final String apiKeyId; + @Nullable + private final Map configuration; // TODO: add explicit types + @Nullable + private final ConnectorCustomSchedule customScheduling; + @Nullable + private final String description; + @Nullable + private final String error; + @Nullable + private final ConnectorFeatures features; + @Nullable + private final List filtering; + @Nullable + private final String indexName; + + private final boolean isNative; + @Nullable + private final String language; + @Nullable + private final ConnectorSyncInfo syncInfo; + @Nullable + private final String name; + @Nullable + private final ConnectorIngestPipeline pipeline; + @Nullable + private final ConnectorScheduling scheduling; + @Nullable + private final String serviceType; + private final ConnectorStatus status; + @Nullable + private final Object syncCursor; + private final boolean syncNow; + + /** + * Constructor for Connector. + * + * @param connectorId Unique identifier for the connector. + * @param apiKeyId API key ID used for authentication/authorization against ES. + * @param configuration Configuration settings for the connector. + * @param customScheduling Custom scheduling settings for the connector. + * @param description Description of the connector. + * @param error Information about the last error encountered by the connector, if any. + * @param features Features enabled for the connector. + * @param filtering Filtering settings applied by the connector. + * @param indexName Name of the index associated with the connector. + * @param isNative Flag indicating whether the connector is a native type. + * @param language The language supported by the connector. + * @param syncInfo Information about the synchronization state of the connector. + * @param name Name of the connector. + * @param pipeline Ingest pipeline configuration. + * @param scheduling Scheduling settings for regular data synchronization. + * @param serviceType Type of service the connector integrates with. + * @param status Current status of the connector. + * @param syncCursor Position or state indicating the current point of synchronization. + * @param syncNow Flag indicating whether an immediate synchronization is requested. + */ + private Connector( + String connectorId, + String apiKeyId, + Map configuration, + ConnectorCustomSchedule customScheduling, + String description, + String error, + ConnectorFeatures features, + List filtering, + String indexName, + boolean isNative, + String language, + ConnectorSyncInfo syncInfo, + String name, + ConnectorIngestPipeline pipeline, + ConnectorScheduling scheduling, + String serviceType, + ConnectorStatus status, + Object syncCursor, + boolean syncNow + ) { + this.connectorId = Objects.requireNonNull(connectorId, "connectorId cannot be null"); + this.apiKeyId = apiKeyId; + this.configuration = configuration; + this.customScheduling = customScheduling; + this.description = description; + this.error = error; + this.features = features; + this.filtering = filtering; + this.indexName = indexName; + this.isNative = isNative; + this.language = language; + this.syncInfo = syncInfo; + this.name = name; + this.pipeline = pipeline; + this.scheduling = scheduling; + this.serviceType = serviceType; + this.status = Objects.requireNonNull(status, "connector status cannot be null"); + this.syncCursor = syncCursor; + this.syncNow = syncNow; + } + + public Connector(StreamInput in) throws IOException { + this.connectorId = in.readString(); + this.apiKeyId = in.readOptionalString(); + this.configuration = in.readMap(StreamInput::readString, StreamInput::readGenericValue); + this.customScheduling = in.readOptionalWriteable(ConnectorCustomSchedule::new); + this.description = in.readOptionalString(); + this.error = in.readOptionalString(); + this.features = in.readOptionalWriteable(ConnectorFeatures::new); + this.filtering = in.readOptionalCollectionAsList(ConnectorFiltering::new); + this.indexName = in.readOptionalString(); + this.isNative = in.readBoolean(); + this.language = in.readOptionalString(); + this.syncInfo = in.readOptionalWriteable(ConnectorSyncInfo::new); + this.name = in.readOptionalString(); + this.pipeline = in.readOptionalWriteable(ConnectorIngestPipeline::new); + this.scheduling = in.readOptionalWriteable(ConnectorScheduling::new); + this.serviceType = in.readOptionalString(); + this.status = in.readEnum(ConnectorStatus.class); + this.syncCursor = in.readGenericValue(); + this.syncNow = in.readBoolean(); + } + + private static final ParseField ID_FIELD = new ParseField("connector_id"); + private static final ParseField API_KEY_ID_FIELD = new ParseField("api_key_id"); + private static final ParseField CONFIGURATION_FIELD = new ParseField("configuration"); + private static final ParseField CUSTOM_SCHEDULING_FIELD = new ParseField("custom_scheduling"); + private static final ParseField DESCRIPTION_FIELD = new ParseField("description"); + private static final ParseField ERROR_FIELD = new ParseField("error"); + private static final ParseField FEATURES_FIELD = new ParseField("features"); + private static final ParseField FILTERING_FIELD = new ParseField("filtering"); + private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name"); + private static final ParseField IS_NATIVE_FIELD = new ParseField("is_native"); + private static final ParseField LANGUAGE_FIELD = new ParseField("language"); + + private static final ParseField NAME_FIELD = new ParseField("name"); + private static final ParseField PIPELINE_FIELD = new ParseField("pipeline"); + private static final ParseField SCHEDULING_FIELD = new ParseField("scheduling"); + private static final ParseField SERVICE_TYPE_FIELD = new ParseField("service_type"); + private static final ParseField STATUS_FIELD = new ParseField("status"); + private static final ParseField SYNC_CURSOR_FIELD = new ParseField("sync_cursor"); + private static final ParseField SYNC_NOW_FIELD = new ParseField("sync_now"); + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(ID_FIELD.getPreferredName(), connectorId); + if (apiKeyId != null) { + builder.field(API_KEY_ID_FIELD.getPreferredName(), apiKeyId); + } + if (configuration != null) { + builder.field(CONFIGURATION_FIELD.getPreferredName(), configuration); + } + if (customScheduling != null) { + builder.field(CUSTOM_SCHEDULING_FIELD.getPreferredName(), customScheduling); + } + if (description != null) { + builder.field(DESCRIPTION_FIELD.getPreferredName(), description); + } + if (error != null) { + builder.field(ERROR_FIELD.getPreferredName(), error); + } + if (features != null) { + builder.field(FEATURES_FIELD.getPreferredName(), features); + } + if (filtering != null) { + builder.xContentList(FILTERING_FIELD.getPreferredName(), filtering); + } + if (indexName != null) { + builder.field(INDEX_NAME_FIELD.getPreferredName(), indexName); + } + builder.field(IS_NATIVE_FIELD.getPreferredName(), isNative); + if (language != null) { + builder.field(LANGUAGE_FIELD.getPreferredName(), language); + } + if (syncInfo != null) { + syncInfo.toXContent(builder, params); + } + if (name != null) { + builder.field(NAME_FIELD.getPreferredName(), name); + } + if (pipeline != null) { + builder.field(PIPELINE_FIELD.getPreferredName(), pipeline); + } + if (scheduling != null) { + builder.field(SCHEDULING_FIELD.getPreferredName(), scheduling); + } + if (serviceType != null) { + builder.field(SERVICE_TYPE_FIELD.getPreferredName(), serviceType); + } + if (syncCursor != null) { + builder.field(SYNC_CURSOR_FIELD.getPreferredName(), syncCursor); + } + builder.field(STATUS_FIELD.getPreferredName(), status.toString()); + builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow); + + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(connectorId); + out.writeOptionalString(apiKeyId); + out.writeMap(configuration, StreamOutput::writeString, StreamOutput::writeGenericValue); + out.writeOptionalWriteable(customScheduling); + out.writeOptionalString(description); + out.writeOptionalString(error); + out.writeOptionalWriteable(features); + out.writeOptionalCollection(filtering); + out.writeOptionalString(indexName); + out.writeBoolean(isNative); + out.writeOptionalString(language); + out.writeOptionalWriteable(syncInfo); + out.writeOptionalString(name); + out.writeOptionalWriteable(pipeline); + out.writeOptionalWriteable(scheduling); + out.writeOptionalString(serviceType); + out.writeEnum(status); + out.writeGenericValue(syncCursor); + out.writeBoolean(syncNow); + } + + public String getConnectorId() { + return connectorId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Connector connector = (Connector) o; + return isNative == connector.isNative + && syncNow == connector.syncNow + && Objects.equals(connectorId, connector.connectorId) + && Objects.equals(apiKeyId, connector.apiKeyId) + && Objects.equals(configuration, connector.configuration) + && Objects.equals(customScheduling, connector.customScheduling) + && Objects.equals(description, connector.description) + && Objects.equals(error, connector.error) + && Objects.equals(features, connector.features) + && Objects.equals(filtering, connector.filtering) + && Objects.equals(indexName, connector.indexName) + && Objects.equals(language, connector.language) + && Objects.equals(syncInfo, connector.syncInfo) + && Objects.equals(name, connector.name) + && Objects.equals(pipeline, connector.pipeline) + && Objects.equals(scheduling, connector.scheduling) + && Objects.equals(serviceType, connector.serviceType) + && status == connector.status + && Objects.equals(syncCursor, connector.syncCursor); + } + + @Override + public int hashCode() { + return Objects.hash( + connectorId, + apiKeyId, + configuration, + customScheduling, + description, + error, + features, + filtering, + indexName, + isNative, + language, + syncInfo, + name, + pipeline, + scheduling, + serviceType, + status, + syncCursor, + syncNow + ); + } + + public static class Builder { + + private String connectorId; + private String apiKeyId; + private Map configuration = Collections.emptyMap(); + private ConnectorCustomSchedule customScheduling; + private String description; + private String error; + private ConnectorFeatures features; + private List filtering = List.of(ConnectorFiltering.getDefaultConnectorFilteringConfig()); + private String indexName; + private boolean isNative = false; + private String language; + private ConnectorSyncInfo syncInfo = new ConnectorSyncInfo.Builder().build(); + private String name; + private ConnectorIngestPipeline pipeline; + private ConnectorScheduling scheduling = ConnectorScheduling.getDefaultConnectorScheduling(); + private String serviceType; + private ConnectorStatus status = ConnectorStatus.CREATED; + private Object syncCursor; + private boolean syncNow = false; + + public Builder setConnectorId(String connectorId) { + this.connectorId = connectorId; + return this; + } + + public Builder setApiKeyId(String apiKeyId) { + this.apiKeyId = apiKeyId; + return this; + } + + public Builder setConfiguration(Map configuration) { + this.configuration = configuration; + return this; + } + + public Builder setCustomScheduling(ConnectorCustomSchedule customScheduling) { + this.customScheduling = customScheduling; + return this; + } + + public Builder setDescription(String description) { + this.description = description; + return this; + } + + public Builder setError(String error) { + this.error = error; + return this; + } + + public Builder setFeatures(ConnectorFeatures features) { + this.features = features; + return this; + } + + public Builder setFiltering(List filtering) { + this.filtering = filtering; + return this; + } + + public Builder setIndexName(String indexName) { + this.indexName = indexName; + return this; + } + + public Builder setIsNative(boolean isNative) { + this.isNative = isNative; + if (isNative) { + this.status = ConnectorStatus.NEEDS_CONFIGURATION; + } + return this; + } + + public Builder setLanguage(String language) { + this.language = language; + return this; + } + + public Builder setSyncInfo(ConnectorSyncInfo syncInfo) { + this.syncInfo = syncInfo; + return this; + } + + public Builder setName(String name) { + this.name = Objects.requireNonNullElse(name, ""); + return this; + } + + public Builder setPipeline(ConnectorIngestPipeline pipeline) { + this.pipeline = pipeline; + return this; + } + + public Builder setScheduling(ConnectorScheduling scheduling) { + this.scheduling = scheduling; + return this; + } + + public Builder setServiceType(String serviceType) { + this.serviceType = serviceType; + return this; + } + + public Builder setStatus(ConnectorStatus status) { + this.status = status; + return this; + } + + public Builder setSyncCursor(Object syncCursor) { + this.syncCursor = syncCursor; + return this; + } + + public Builder setSyncNow(boolean syncNow) { + this.syncNow = syncNow; + return this; + } + + public Connector build() { + return new Connector( + connectorId, + apiKeyId, + configuration, + customScheduling, + description, + error, + features, + filtering, + indexName, + isNative, + language, + syncInfo, + name, + pipeline, + scheduling, + serviceType, + status, + syncCursor, + syncNow + ); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorAPIFeature.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorAPIFeature.java new file mode 100644 index 0000000000000..40dcf02a2bf19 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorAPIFeature.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.util.FeatureFlag; + +/** + * Connector API feature flag. When the feature is complete, this flag will be removed. + */ +public class ConnectorAPIFeature { + + private static final FeatureFlag CONNECTOR_API_FEATURE_FLAG = new FeatureFlag("connector_api"); + + public static boolean isEnabled() { + return CONNECTOR_API_FEATURE_FLAG.isEnabled(); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java new file mode 100644 index 0000000000000..081a0245f9279 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorCustomSchedule.java @@ -0,0 +1,335 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.scheduler.Cron; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class ConnectorCustomSchedule implements Writeable, ToXContentObject { + + private final ConfigurationOverrides configurationOverrides; + private final boolean enabled; + private final Cron interval; + @Nullable + private final Instant lastSynced; + private final String name; + + /** + * Constructor for ConnectorCustomSchedule. + * + * @param configurationOverrides Configuration overrides {@link ConfigurationOverrides} specifies custom settings overrides. + * @param enabled Flag indicating whether the custom schedule is active or not. + * @param interval The interval at which the custom schedule runs, specified in a cron-like format. + * @param lastSynced The timestamp of the last successful synchronization performed under this custom schedule, if any. + * @param name The name of the custom schedule, used for identification and reference purposes. + */ + private ConnectorCustomSchedule( + ConfigurationOverrides configurationOverrides, + boolean enabled, + Cron interval, + Instant lastSynced, + String name + ) { + this.configurationOverrides = Objects.requireNonNull(configurationOverrides, CONFIG_OVERRIDES_FIELD.getPreferredName()); + this.enabled = enabled; + this.interval = Objects.requireNonNull(interval, INTERVAL_FIELD.getPreferredName()); + this.lastSynced = lastSynced; + this.name = Objects.requireNonNull(name, NAME_FIELD.getPreferredName()); + } + + public ConnectorCustomSchedule(StreamInput in) throws IOException { + this.configurationOverrides = new ConfigurationOverrides(in); + this.enabled = in.readBoolean(); + this.interval = new Cron(in.readString()); + this.lastSynced = in.readOptionalInstant(); + this.name = in.readString(); + } + + private static final ParseField CONFIG_OVERRIDES_FIELD = new ParseField("configuration_overrides"); + private static final ParseField ENABLED_FIELD = new ParseField("enabled"); + private static final ParseField INTERVAL_FIELD = new ParseField("interval"); + private static final ParseField LAST_SYNCED_FIELD = new ParseField("last_synced"); + + private static final ParseField NAME_FIELD = new ParseField("name"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_custom_schedule", + true, + args -> new Builder().setConfigurationOverrides((ConfigurationOverrides) args[0]) + .setEnabled((Boolean) args[1]) + .setInterval(new Cron((String) args[2])) + .setLastSynced((Instant) args[3]) + .setName((String) args[4]) + .build() + ); + + static { + PARSER.declareField( + constructorArg(), + (parser, context) -> ConfigurationOverrides.fromXContent(parser), + CONFIG_OVERRIDES_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareBoolean(constructorArg(), ENABLED_FIELD); + PARSER.declareString(constructorArg(), INTERVAL_FIELD); + PARSER.declareString(optionalConstructorArg(), LAST_SYNCED_FIELD); + PARSER.declareString(constructorArg(), NAME_FIELD); + } + + public static ConnectorCustomSchedule fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(CONFIG_OVERRIDES_FIELD.getPreferredName(), configurationOverrides); + builder.field(ENABLED_FIELD.getPreferredName(), enabled); + builder.field(INTERVAL_FIELD.getPreferredName(), interval); + if (lastSynced != null) { + builder.field(LAST_SYNCED_FIELD.getPreferredName(), lastSynced); + } + builder.field(NAME_FIELD.getPreferredName(), name); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeWriteable(configurationOverrides); + out.writeBoolean(enabled); + out.writeString(interval.toString()); + out.writeOptionalInstant(lastSynced); + out.writeString(name); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorCustomSchedule that = (ConnectorCustomSchedule) o; + return enabled == that.enabled + && Objects.equals(configurationOverrides, that.configurationOverrides) + && Objects.equals(interval, that.interval) + && Objects.equals(lastSynced, that.lastSynced) + && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(configurationOverrides, enabled, interval, lastSynced, name); + } + + public static class Builder { + + private ConfigurationOverrides configurationOverrides; + private boolean enabled; + private Cron interval; + private Instant lastSynced; + private String name; + + public Builder setConfigurationOverrides(ConfigurationOverrides configurationOverrides) { + this.configurationOverrides = configurationOverrides; + return this; + } + + public Builder setEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + public Builder setInterval(Cron interval) { + this.interval = interval; + return this; + } + + public Builder setLastSynced(Instant lastSynced) { + this.lastSynced = lastSynced; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public ConnectorCustomSchedule build() { + return new ConnectorCustomSchedule(configurationOverrides, enabled, interval, lastSynced, name); + } + } + + public static class ConfigurationOverrides implements Writeable, ToXContentObject { + @Nullable + private final Integer maxCrawlDepth; + @Nullable + private final Boolean sitemapDiscoveryDisabled; + @Nullable + private final List domainAllowList; + @Nullable + private final List sitemapUrls; + @Nullable + private final List seedUrls; + + private ConfigurationOverrides( + Integer maxCrawlDepth, + Boolean sitemapDiscoveryDisabled, + List domainAllowList, + List sitemapUrls, + List seedUrls + ) { + this.maxCrawlDepth = maxCrawlDepth; + this.sitemapDiscoveryDisabled = sitemapDiscoveryDisabled; + this.domainAllowList = domainAllowList; + this.sitemapUrls = sitemapUrls; + this.seedUrls = seedUrls; + } + + public ConfigurationOverrides(StreamInput in) throws IOException { + this.maxCrawlDepth = in.readOptionalInt(); + this.sitemapDiscoveryDisabled = in.readOptionalBoolean(); + this.domainAllowList = in.readOptionalStringCollectionAsList(); + this.sitemapUrls = in.readOptionalStringCollectionAsList(); + this.seedUrls = in.readOptionalStringCollectionAsList(); + } + + private static final ParseField MAX_CRAWL_DEPTH_FIELD = new ParseField("max_crawl_depth"); + private static final ParseField SITEMAP_DISCOVERY_DISABLED_FIELD = new ParseField("sitemap_discovery_disabled"); + private static final ParseField DOMAIN_ALLOWLIST_FIELD = new ParseField("domain_allowlist"); + private static final ParseField SITEMAP_URLS_FIELD = new ParseField("sitemap_urls"); + private static final ParseField SEED_URLS_FIELD = new ParseField("seed_urls"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "configuration_override", + true, + args -> new Builder().setMaxCrawlDepth((Integer) args[0]) + .setSitemapDiscoveryDisabled((Boolean) args[1]) + .setDomainAllowList((List) args[2]) + .setSitemapUrls((List) args[3]) + .setSeedUrls((List) args[4]) + .build() + ); + + static { + PARSER.declareInt(optionalConstructorArg(), MAX_CRAWL_DEPTH_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), SITEMAP_DISCOVERY_DISABLED_FIELD); + PARSER.declareStringArray(optionalConstructorArg(), DOMAIN_ALLOWLIST_FIELD); + PARSER.declareStringArray(optionalConstructorArg(), SITEMAP_URLS_FIELD); + PARSER.declareStringArray(optionalConstructorArg(), SEED_URLS_FIELD); + } + + public static ConfigurationOverrides fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (maxCrawlDepth != null) { + builder.field(MAX_CRAWL_DEPTH_FIELD.getPreferredName(), maxCrawlDepth); + } + if (sitemapDiscoveryDisabled != null) { + builder.field(SITEMAP_DISCOVERY_DISABLED_FIELD.getPreferredName(), sitemapDiscoveryDisabled); + } + if (domainAllowList != null) { + builder.stringListField(DOMAIN_ALLOWLIST_FIELD.getPreferredName(), domainAllowList); + } + if (sitemapUrls != null) { + builder.stringListField(SITEMAP_URLS_FIELD.getPreferredName(), sitemapUrls); + } + if (seedUrls != null) { + builder.stringListField(SEED_URLS_FIELD.getPreferredName(), seedUrls); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalInt(maxCrawlDepth); + out.writeOptionalBoolean(sitemapDiscoveryDisabled); + out.writeOptionalStringCollection(domainAllowList); + out.writeOptionalStringCollection(sitemapUrls); + out.writeOptionalStringCollection(seedUrls); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConfigurationOverrides that = (ConfigurationOverrides) o; + return Objects.equals(maxCrawlDepth, that.maxCrawlDepth) + && Objects.equals(sitemapDiscoveryDisabled, that.sitemapDiscoveryDisabled) + && Objects.equals(domainAllowList, that.domainAllowList) + && Objects.equals(sitemapUrls, that.sitemapUrls) + && Objects.equals(seedUrls, that.seedUrls); + } + + @Override + public int hashCode() { + return Objects.hash(maxCrawlDepth, sitemapDiscoveryDisabled, domainAllowList, sitemapUrls, seedUrls); + } + + public static class Builder { + + private Integer maxCrawlDepth; + private Boolean sitemapDiscoveryDisabled; + private List domainAllowList; + private List sitemapUrls; + private List seedUrls; + + public Builder setMaxCrawlDepth(Integer maxCrawlDepth) { + this.maxCrawlDepth = maxCrawlDepth; + return this; + } + + public Builder setSitemapDiscoveryDisabled(Boolean sitemapDiscoveryDisabled) { + this.sitemapDiscoveryDisabled = sitemapDiscoveryDisabled; + return this; + } + + public Builder setDomainAllowList(List domainAllowList) { + this.domainAllowList = domainAllowList; + return this; + } + + public Builder setSitemapUrls(List sitemapUrls) { + this.sitemapUrls = sitemapUrls; + return this; + } + + public Builder setSeedUrls(List seedUrls) { + this.seedUrls = seedUrls; + return this; + } + + public ConfigurationOverrides build() { + return new ConfigurationOverrides(maxCrawlDepth, sitemapDiscoveryDisabled, domainAllowList, sitemapUrls, seedUrls); + } + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java new file mode 100644 index 0000000000000..b90b230381b8e --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFeatures.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class ConnectorFeatures implements Writeable, ToXContentObject { + + @Nullable + private final Boolean documentLevelSecurityEnabled; + @Nullable + private final Boolean filteringAdvancedConfigEnabled; + @Nullable + private final Boolean filteringRulesEnabled; + @Nullable + private final Boolean incrementalSyncEnabled; + @Nullable + private final Boolean syncRulesAdvancedEnabled; + @Nullable + private final Boolean syncRulesBasicEnabled; + + /** + * Constructs a new instance of ConnectorFeatures. + * + * @param documentLevelSecurityEnabled A flag indicating whether document-level security is enabled. + * @param filteringAdvancedConfig A flag indicating whether advanced filtering configuration is enabled. + * @param filteringRules A flag indicating whether filtering rules are enabled. + * @param incrementalSyncEnabled A flag indicating whether incremental synchronization is enabled. + * @param syncRulesAdvancedEnabled A flag indicating whether advanced synchronization rules are enabled. + * @param syncRulesBasicEnabled A flag indicating whether basic synchronization rules are enabled. + */ + private ConnectorFeatures( + Boolean documentLevelSecurityEnabled, + Boolean filteringAdvancedConfig, + Boolean filteringRules, + Boolean incrementalSyncEnabled, + Boolean syncRulesAdvancedEnabled, + Boolean syncRulesBasicEnabled + ) { + this.documentLevelSecurityEnabled = documentLevelSecurityEnabled; + this.filteringAdvancedConfigEnabled = filteringAdvancedConfig; + this.filteringRulesEnabled = filteringRules; + this.incrementalSyncEnabled = incrementalSyncEnabled; + this.syncRulesAdvancedEnabled = syncRulesAdvancedEnabled; + this.syncRulesBasicEnabled = syncRulesBasicEnabled; + } + + public ConnectorFeatures(StreamInput in) throws IOException { + this.documentLevelSecurityEnabled = in.readOptionalBoolean(); + this.filteringAdvancedConfigEnabled = in.readOptionalBoolean(); + this.filteringRulesEnabled = in.readOptionalBoolean(); + this.incrementalSyncEnabled = in.readOptionalBoolean(); + this.syncRulesAdvancedEnabled = in.readOptionalBoolean(); + this.syncRulesBasicEnabled = in.readOptionalBoolean(); + } + + private static final ParseField DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD = new ParseField("document_level_security"); + private static final ParseField FILTERING_ADVANCED_CONFIG_ENABLED_FIELD = new ParseField("filtering_advanced_config"); + private static final ParseField FILTERING_RULES_ENABLED_FIELD = new ParseField("filtering_rules"); + private static final ParseField INCREMENTAL_SYNC_ENABLED_FIELD = new ParseField("incremental_sync"); + private static final ParseField SYNC_RULES_ADVANCED_ENABLED_FIELD = new ParseField("advanced_sync_rules"); + private static final ParseField SYNC_RULES_BASIC_ENABLED_FIELD = new ParseField("basic_sync_rules"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_features", + true, + args -> new Builder().setDocumentLevelSecurityEnabled((Boolean) args[0]) + .setFilteringAdvancedConfig((Boolean) args[1]) + .setFilteringRules((Boolean) args[2]) + .setIncrementalSyncEnabled((Boolean) args[3]) + .setSyncRulesAdvancedEnabled((Boolean) args[4]) + .setSyncRulesBasicEnabled((Boolean) args[5]) + .build() + ); + + static { + PARSER.declareBoolean(optionalConstructorArg(), DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), FILTERING_ADVANCED_CONFIG_ENABLED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), FILTERING_RULES_ENABLED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), INCREMENTAL_SYNC_ENABLED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), SYNC_RULES_ADVANCED_ENABLED_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), SYNC_RULES_BASIC_ENABLED_FIELD); + } + + public static ConnectorFeatures fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + if (documentLevelSecurityEnabled != null) { + builder.startObject(DOCUMENT_LEVEL_SECURITY_ENABLED_FIELD.getPreferredName()); + { + builder.field("enabled", documentLevelSecurityEnabled); + } + builder.endObject(); + } + if (filteringAdvancedConfigEnabled != null) { + builder.field(FILTERING_ADVANCED_CONFIG_ENABLED_FIELD.getPreferredName(), filteringAdvancedConfigEnabled); + } + if (filteringRulesEnabled != null) { + builder.field(FILTERING_RULES_ENABLED_FIELD.getPreferredName(), filteringRulesEnabled); + } + if (incrementalSyncEnabled != null) { + builder.startObject(INCREMENTAL_SYNC_ENABLED_FIELD.getPreferredName()); + { + builder.field("enabled", incrementalSyncEnabled); + } + builder.endObject(); + } + builder.startObject("sync_rules"); + { + if (syncRulesAdvancedEnabled != null) { + builder.startObject("advanced"); + { + builder.field("enabled", syncRulesAdvancedEnabled); + } + builder.endObject(); + } + if (syncRulesBasicEnabled != null) { + builder.startObject("basic"); + { + builder.field("enabled", syncRulesBasicEnabled); + } + builder.endObject(); + } + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalBoolean(documentLevelSecurityEnabled); + out.writeOptionalBoolean(filteringAdvancedConfigEnabled); + out.writeOptionalBoolean(filteringRulesEnabled); + out.writeOptionalBoolean(incrementalSyncEnabled); + out.writeOptionalBoolean(syncRulesAdvancedEnabled); + out.writeOptionalBoolean(syncRulesBasicEnabled); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorFeatures that = (ConnectorFeatures) o; + return Objects.equals(documentLevelSecurityEnabled, that.documentLevelSecurityEnabled) + && Objects.equals(filteringAdvancedConfigEnabled, that.filteringAdvancedConfigEnabled) + && Objects.equals(filteringRulesEnabled, that.filteringRulesEnabled) + && Objects.equals(incrementalSyncEnabled, that.incrementalSyncEnabled) + && Objects.equals(syncRulesAdvancedEnabled, that.syncRulesAdvancedEnabled) + && Objects.equals(syncRulesBasicEnabled, that.syncRulesBasicEnabled); + } + + @Override + public int hashCode() { + return Objects.hash( + documentLevelSecurityEnabled, + filteringAdvancedConfigEnabled, + filteringRulesEnabled, + incrementalSyncEnabled, + syncRulesAdvancedEnabled, + syncRulesBasicEnabled + ); + } + + public static class Builder { + + private Boolean documentLevelSecurityEnabled; + private Boolean filteringAdvancedConfig; + private Boolean filteringRules; + private Boolean incrementalSyncEnabled; + private Boolean syncRulesAdvancedEnabled; + private Boolean syncRulesBasicEnabled; + + public Builder setDocumentLevelSecurityEnabled(Boolean documentLevelSecurityEnabled) { + this.documentLevelSecurityEnabled = documentLevelSecurityEnabled; + return this; + } + + public Builder setFilteringAdvancedConfig(Boolean filteringAdvancedConfig) { + this.filteringAdvancedConfig = filteringAdvancedConfig; + return this; + } + + public Builder setFilteringRules(Boolean filteringRules) { + this.filteringRules = filteringRules; + return this; + } + + public Builder setIncrementalSyncEnabled(Boolean incrementalSyncEnabled) { + this.incrementalSyncEnabled = incrementalSyncEnabled; + return this; + } + + public Builder setSyncRulesAdvancedEnabled(Boolean syncRulesAdvancedEnabled) { + this.syncRulesAdvancedEnabled = syncRulesAdvancedEnabled; + return this; + } + + public Builder setSyncRulesBasicEnabled(Boolean syncRulesBasicEnabled) { + this.syncRulesBasicEnabled = syncRulesBasicEnabled; + return this; + } + + public ConnectorFeatures build() { + return new ConnectorFeatures( + documentLevelSecurityEnabled, + filteringAdvancedConfig, + filteringRules, + incrementalSyncEnabled, + syncRulesAdvancedEnabled, + syncRulesBasicEnabled + ); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java new file mode 100644 index 0000000000000..b20970e80381c --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorFiltering.java @@ -0,0 +1,590 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +public class ConnectorFiltering implements Writeable, ToXContentObject { + + private final FilteringRules active; + private final String domain; + private final FilteringRules draft; + + /** + * Constructs a new ConnectorFiltering instance. + * + * @param active The active filtering rules. + * @param domain The domain associated with the filtering. + * @param draft The draft filtering rules. + */ + public ConnectorFiltering(FilteringRules active, String domain, FilteringRules draft) { + this.active = active; + this.domain = domain; + this.draft = draft; + } + + public ConnectorFiltering(StreamInput in) throws IOException { + this.active = new FilteringRules(in); + this.domain = in.readString(); + this.draft = new FilteringRules(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("active", active); + builder.field("domain", domain); + builder.field("draft", draft); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + active.writeTo(out); + out.writeString(domain); + draft.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorFiltering that = (ConnectorFiltering) o; + return Objects.equals(active, that.active) && Objects.equals(domain, that.domain) && Objects.equals(draft, that.draft); + } + + @Override + public int hashCode() { + return Objects.hash(active, domain, draft); + } + + public static class Builder { + + private FilteringRules active; + private String domain; + private FilteringRules draft; + + public Builder setActive(FilteringRules active) { + this.active = active; + return this; + } + + public Builder setDomain(String domain) { + this.domain = domain; + return this; + } + + public Builder setDraft(FilteringRules draft) { + this.draft = draft; + return this; + } + + public ConnectorFiltering build() { + return new ConnectorFiltering(active, domain, draft); + } + } + + public static class FilteringRules implements Writeable, ToXContentObject { + + private final Instant advancedSnippetCreatedAt; + private final Instant advancedSnippetUpdatedAt; + private final Map advancedSnippetValue; + private final List rules; + private final List validationErrors; + private final FilteringValidationState validationState; + + /** + * Constructs a new FilteringRules instance. + * + * @param advancedSnippetCreatedAt The creation timestamp of the advanced snippet. + * @param advancedSnippetUpdatedAt The update timestamp of the advanced snippet. + * @param advancedSnippetValue The map of the advanced snippet. + * @param rules The list of {@link FilteringRule} objects + * @param validationErrors The list of {@link FilteringValidation} errors for the filtering rules. + * @param validationState The {@link FilteringValidationState} of the filtering rules. + */ + public FilteringRules( + Instant advancedSnippetCreatedAt, + Instant advancedSnippetUpdatedAt, + Map advancedSnippetValue, + List rules, + List validationErrors, + FilteringValidationState validationState + ) { + this.advancedSnippetCreatedAt = advancedSnippetCreatedAt; + this.advancedSnippetUpdatedAt = advancedSnippetUpdatedAt; + this.advancedSnippetValue = advancedSnippetValue; + this.rules = rules; + this.validationErrors = validationErrors; + this.validationState = validationState; + } + + public FilteringRules(StreamInput in) throws IOException { + this.advancedSnippetCreatedAt = in.readInstant(); + this.advancedSnippetUpdatedAt = in.readInstant(); + this.advancedSnippetValue = in.readMap(StreamInput::readString, StreamInput::readGenericValue); + this.rules = in.readCollectionAsList(FilteringRule::new); + this.validationErrors = in.readCollectionAsList(FilteringValidation::new); + this.validationState = in.readEnum(FilteringValidationState.class); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.startObject("advanced_snippet"); + { + builder.field("created_at", advancedSnippetCreatedAt); + builder.field("updated_at", advancedSnippetUpdatedAt); + builder.field("value", advancedSnippetValue); + } + builder.endObject(); + + builder.startArray("rules"); + for (FilteringRule rule : rules) { + rule.toXContent(builder, params); + } + builder.endArray(); + + builder.startObject("validation"); + { + builder.startArray("errors"); + for (FilteringValidation error : validationErrors) { + error.toXContent(builder, params); + } + builder.endArray(); + builder.field("state", validationState.toString()); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInstant(advancedSnippetCreatedAt); + out.writeInstant(advancedSnippetUpdatedAt); + out.writeMap(advancedSnippetValue, StreamOutput::writeString, StreamOutput::writeGenericValue); + out.writeCollection(rules); + out.writeCollection(validationErrors); + out.writeEnum(validationState); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FilteringRules that = (FilteringRules) o; + return Objects.equals(advancedSnippetCreatedAt, that.advancedSnippetCreatedAt) + && Objects.equals(advancedSnippetUpdatedAt, that.advancedSnippetUpdatedAt) + && Objects.equals(advancedSnippetValue, that.advancedSnippetValue) + && Objects.equals(rules, that.rules) + && Objects.equals(validationErrors, that.validationErrors) + && validationState == that.validationState; + } + + @Override + public int hashCode() { + return Objects.hash( + advancedSnippetCreatedAt, + advancedSnippetUpdatedAt, + advancedSnippetValue, + rules, + validationErrors, + validationState + ); + } + + public static class Builder { + + private Instant advancedSnippetCreatedAt; + private Instant advancedSnippetUpdatedAt; + private Map advancedSnippetValue; + private List rules; + private List validationErrors; + private FilteringValidationState validationState; + + public Builder setAdvancedSnippetCreatedAt(Instant advancedSnippetCreatedAt) { + this.advancedSnippetCreatedAt = advancedSnippetCreatedAt; + return this; + } + + public Builder setAdvancedSnippetUpdatedAt(Instant advancedSnippetUpdatedAt) { + this.advancedSnippetUpdatedAt = advancedSnippetUpdatedAt; + return this; + } + + public Builder setAdvancedSnippetValue(Map advancedSnippetValue) { + this.advancedSnippetValue = advancedSnippetValue; + return this; + } + + public Builder setRules(List rules) { + this.rules = rules; + return this; + } + + public Builder setValidationErrors(List validationErrors) { + this.validationErrors = validationErrors; + return this; + } + + public Builder setValidationState(FilteringValidationState validationState) { + this.validationState = validationState; + return this; + } + + public FilteringRules build() { + return new FilteringRules( + advancedSnippetCreatedAt, + advancedSnippetUpdatedAt, + advancedSnippetValue, + rules, + validationErrors, + validationState + ); + } + } + } + + public static class FilteringRule implements Writeable, ToXContentObject { + + private final Instant createdAt; + private final String field; + private final String id; + private final Integer order; + private final FilteringPolicy policy; + private final FilteringRuleCondition rule; + private final Instant updatedAt; + private final String value; + + /** + * Constructs a new FilteringRule instance. + * + * @param createdAt The creation timestamp of the filtering rule. + * @param field The field associated with the filtering rule. + * @param id The identifier of the filtering rule. + * @param order The order of the filtering rule. + * @param policy The {@link FilteringPolicy} of the filtering rule. + * @param rule The specific {@link FilteringRuleCondition} + * @param updatedAt The update timestamp of the filtering rule. + * @param value The value associated with the filtering rule. + */ + public FilteringRule( + Instant createdAt, + String field, + String id, + Integer order, + FilteringPolicy policy, + FilteringRuleCondition rule, + Instant updatedAt, + String value + ) { + this.createdAt = createdAt; + this.field = field; + this.id = id; + this.order = order; + this.policy = policy; + this.rule = rule; + this.updatedAt = updatedAt; + this.value = value; + } + + public FilteringRule(StreamInput in) throws IOException { + this.createdAt = in.readInstant(); + this.field = in.readString(); + this.id = in.readString(); + this.order = in.readInt(); + this.policy = in.readEnum(FilteringPolicy.class); + this.rule = in.readEnum(FilteringRuleCondition.class); + this.updatedAt = in.readInstant(); + this.value = in.readString(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("createdAt", createdAt); + builder.field("field", field); + builder.field("id", id); + builder.field("order", order); + builder.field("policy", policy.toString()); + builder.field("rule", rule.toString()); + builder.field("updatedAt", updatedAt); + builder.field("value", value); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInstant(createdAt); + out.writeString(field); + out.writeString(id); + out.writeInt(order); + out.writeEnum(policy); + out.writeEnum(rule); + out.writeInstant(updatedAt); + out.writeString(value); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FilteringRule that = (FilteringRule) o; + return Objects.equals(createdAt, that.createdAt) + && Objects.equals(field, that.field) + && Objects.equals(id, that.id) + && Objects.equals(order, that.order) + && policy == that.policy + && rule == that.rule + && Objects.equals(updatedAt, that.updatedAt) + && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(createdAt, field, id, order, policy, rule, updatedAt, value); + } + + public static class Builder { + + private Instant createdAt; + private String field; + private String id; + private Integer order; + private FilteringPolicy policy; + private FilteringRuleCondition rule; + private Instant updatedAt; + private String value; + + public Builder setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + return this; + } + + public Builder setField(String field) { + this.field = field; + return this; + } + + public Builder setId(String id) { + this.id = id; + return this; + } + + public Builder setOrder(Integer order) { + this.order = order; + return this; + } + + public Builder setPolicy(FilteringPolicy policy) { + this.policy = policy; + return this; + } + + public Builder setRule(FilteringRuleCondition rule) { + this.rule = rule; + return this; + } + + public Builder setUpdatedAt(Instant updatedAt) { + this.updatedAt = updatedAt; + return this; + } + + public Builder setValue(String value) { + this.value = value; + return this; + } + + public FilteringRule build() { + return new FilteringRule(createdAt, field, id, order, policy, rule, updatedAt, value); + } + } + } + + public static class FilteringValidation implements Writeable, ToXContentObject { + private final List ids; + private final List messages; + + /** + * Constructs a new FilteringValidation instance. + * + * @param ids The list of identifiers associated with the validation. + * @param messages The list of messages describing the validation results. + */ + public FilteringValidation(List ids, List messages) { + this.ids = ids; + this.messages = messages; + } + + public FilteringValidation(StreamInput in) throws IOException { + this.ids = in.readStringCollectionAsList(); + this.messages = in.readStringCollectionAsList(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.stringListField("ids", ids); + builder.stringListField("messages", messages); + } + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(ids); + out.writeStringCollection(messages); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FilteringValidation that = (FilteringValidation) o; + return Objects.equals(ids, that.ids) && Objects.equals(messages, that.messages); + } + + @Override + public int hashCode() { + return Objects.hash(ids, messages); + } + + public static class Builder { + + private List ids; + private List messages; + + public Builder setIds(List ids) { + this.ids = ids; + return this; + } + + public Builder setMessages(List messages) { + this.messages = messages; + return this; + } + + public FilteringValidation build() { + return new FilteringValidation(ids, messages); + } + } + } + + public enum FilteringValidationState { + EDITED, + INVALID, + VALID; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public enum FilteringPolicy { + EXCLUDE, + INCLUDE; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public enum FilteringRuleCondition { + CONTAINS("contains"), + ENDS_WITH("ends_with"), + EQUALS("equals"), + GT(">"), + LT("<"), + REGEX("regex"), + STARTS_WITH("starts_with"); + + private final String value; + + FilteringRuleCondition(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } + } + + public static ConnectorFiltering getDefaultConnectorFilteringConfig() { + + Instant currentTimestamp = Instant.now(); + + return new ConnectorFiltering.Builder().setActive( + new FilteringRules.Builder().setAdvancedSnippetCreatedAt(currentTimestamp) + .setAdvancedSnippetUpdatedAt(currentTimestamp) + .setAdvancedSnippetValue(Collections.emptyMap()) + .setRules( + List.of( + new FilteringRule.Builder().setCreatedAt(currentTimestamp) + .setField("_") + .setId("DEFAULT") + .setOrder(0) + .setPolicy(FilteringPolicy.INCLUDE) + .setRule(FilteringRuleCondition.REGEX) + .setUpdatedAt(currentTimestamp) + .setValue(".*") + .build() + ) + ) + .setValidationErrors(Collections.emptyList()) + .setValidationState(FilteringValidationState.VALID) + .build() + ) + .setDomain("DEFAULT") + .setDraft( + new FilteringRules.Builder().setAdvancedSnippetCreatedAt(currentTimestamp) + .setAdvancedSnippetUpdatedAt(currentTimestamp) + .setAdvancedSnippetValue(Collections.emptyMap()) + .setRules( + List.of( + new FilteringRule.Builder().setCreatedAt(currentTimestamp) + .setField("_") + .setId("DEFAULT") + .setOrder(0) + .setPolicy(FilteringPolicy.INCLUDE) + .setRule(FilteringRuleCondition.REGEX) + .setUpdatedAt(currentTimestamp) + .setValue(".*") + .build() + ) + ) + .setValidationErrors(Collections.emptyList()) + .setValidationState(FilteringValidationState.VALID) + .build() + ) + .build(); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java new file mode 100644 index 0000000000000..0ae971d161b17 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.client.internal.OriginSettingClient; +import org.elasticsearch.xcontent.ToXContent; + +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; + +/** + * A service that manages persistent {@link Connector} configurations. + */ +public class ConnectorIndexService { + + private final Client clientWithOrigin; + + public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; + + /** + * @param client A client for executing actions on the connector index + */ + public ConnectorIndexService(Client client) { + this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); + } + + /** + * Creates or updates the {@link Connector} in the underlying index. + * + * @param connector The connector object. + * @param listener The action listener to invoke on response/failure. + */ + public void putConnector(Connector connector, ActionListener listener) { + try { + final IndexRequest indexRequest = new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connector.getConnectorId()) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(connector.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); + clientWithOrigin.index(indexRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipeline.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipeline.java new file mode 100644 index 0000000000000..8b94624ee981c --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipeline.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class ConnectorIngestPipeline implements Writeable, ToXContentObject { + + private final Boolean extractBinaryContent; + private final String name; + private final Boolean reduceWhitespace; + private final Boolean runMlInference; + + /** + * Constructs a new instance of ConnectorIngestPipeline. + * + * @param extractBinaryContent A Boolean flag indicating whether to extract binary content during ingestion. + * @param name The name of the ingest pipeline. + * @param reduceWhitespace A Boolean flag indicating whether to reduce extraneous whitespace in the ingested content. + * @param runMlInference A Boolean flag indicating whether to run machine learning inference on the ingested content. + */ + private ConnectorIngestPipeline(Boolean extractBinaryContent, String name, Boolean reduceWhitespace, Boolean runMlInference) { + this.extractBinaryContent = Objects.requireNonNull(extractBinaryContent, EXTRACT_BINARY_CONTENT_FIELD.getPreferredName()); + this.name = Objects.requireNonNull(name, NAME_FIELD.getPreferredName()); + this.reduceWhitespace = Objects.requireNonNull(reduceWhitespace, REDUCE_WHITESPACE_FIELD.getPreferredName()); + this.runMlInference = Objects.requireNonNull(runMlInference, RUN_ML_INFERENCE_FIELD.getPreferredName()); + } + + public ConnectorIngestPipeline(StreamInput in) throws IOException { + this.extractBinaryContent = in.readBoolean(); + this.name = in.readString(); + this.reduceWhitespace = in.readBoolean(); + this.runMlInference = in.readBoolean(); + } + + private static final ParseField EXTRACT_BINARY_CONTENT_FIELD = new ParseField("extract_binary_content"); + private static final ParseField NAME_FIELD = new ParseField("name"); + private static final ParseField REDUCE_WHITESPACE_FIELD = new ParseField("reduce_whitespace"); + private static final ParseField RUN_ML_INFERENCE_FIELD = new ParseField("run_ml_inference"); + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(EXTRACT_BINARY_CONTENT_FIELD.getPreferredName(), extractBinaryContent); + builder.field(NAME_FIELD.getPreferredName(), name); + builder.field(REDUCE_WHITESPACE_FIELD.getPreferredName(), reduceWhitespace); + builder.field(RUN_ML_INFERENCE_FIELD.getPreferredName(), runMlInference); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(extractBinaryContent); + out.writeString(name); + out.writeBoolean(reduceWhitespace); + out.writeBoolean(runMlInference); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorIngestPipeline that = (ConnectorIngestPipeline) o; + return Objects.equals(extractBinaryContent, that.extractBinaryContent) + && Objects.equals(name, that.name) + && Objects.equals(reduceWhitespace, that.reduceWhitespace) + && Objects.equals(runMlInference, that.runMlInference); + } + + @Override + public int hashCode() { + return Objects.hash(extractBinaryContent, name, reduceWhitespace, runMlInference); + } + + public static class Builder { + + private Boolean extractBinaryContent; + private String name; + private Boolean reduceWhitespace; + private Boolean runMlInference; + + public Builder setExtractBinaryContent(Boolean extractBinaryContent) { + this.extractBinaryContent = extractBinaryContent; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setReduceWhitespace(Boolean reduceWhitespace) { + this.reduceWhitespace = reduceWhitespace; + return this; + } + + public Builder setRunMlInference(Boolean runMlInference) { + this.runMlInference = runMlInference; + return this; + } + + public ConnectorIngestPipeline build() { + return new ConnectorIngestPipeline(extractBinaryContent, name, reduceWhitespace, runMlInference); + } + } + +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorListItem.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorListItem.java new file mode 100644 index 0000000000000..22dd8d2ae3f22 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorListItem.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * This class is used for returning information for lists of connectors, to avoid including all + * {@link Connector} information which can be retrieved using subsequent GetConnector requests. + */ +public class ConnectorListItem implements Writeable, ToXContentObject { + + private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); + private static final ParseField NAME_FIELD = new ParseField("name"); + + private final String connectorId; + + @Nullable + private final String name; + + public ConnectorListItem(String connectorId, @Nullable String name) { + this.connectorId = connectorId; + this.name = name; + } + + public ConnectorListItem(StreamInput in) throws IOException { + this.connectorId = in.readString(); + this.name = in.readOptionalString(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId); + if (name != null) { + builder.field(NAME_FIELD.getPreferredName(), name); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(connectorId); + out.writeOptionalString(name); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java new file mode 100644 index 0000000000000..233bea5d4a842 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorScheduling.java @@ -0,0 +1,249 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.scheduler.Cron; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; + +public class ConnectorScheduling implements Writeable, ToXContentObject { + + private final ScheduleConfig accessControl; + private final ScheduleConfig full; + private final ScheduleConfig incremental; + + private static final ParseField ACCESS_CONTROL_FIELD = new ParseField("access_control"); + private static final ParseField FULL_FIELD = new ParseField("full"); + private static final ParseField INCREMENTAL_FIELD = new ParseField("incremental"); + + /** + * @param accessControl connector access control sync schedule represented as {@link ScheduleConfig} + * @param full connector full sync schedule represented as {@link ScheduleConfig} + * @param incremental connector incremental sync schedule represented as {@link ScheduleConfig} + */ + private ConnectorScheduling(ScheduleConfig accessControl, ScheduleConfig full, ScheduleConfig incremental) { + this.accessControl = Objects.requireNonNull(accessControl, ACCESS_CONTROL_FIELD.getPreferredName()); + this.full = Objects.requireNonNull(full, FULL_FIELD.getPreferredName()); + this.incremental = Objects.requireNonNull(incremental, INCREMENTAL_FIELD.getPreferredName()); + } + + public ConnectorScheduling(StreamInput in) throws IOException { + this.accessControl = new ScheduleConfig(in); + this.full = new ScheduleConfig(in); + this.incremental = new ScheduleConfig(in); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_scheduling", + true, + args -> new Builder().setAccessControl((ScheduleConfig) args[0]) + .setFull((ScheduleConfig) args[1]) + .setIncremental((ScheduleConfig) args[2]) + .build() + ); + + static { + PARSER.declareField( + constructorArg(), + (p, c) -> ScheduleConfig.fromXContent(p), + ACCESS_CONTROL_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField(constructorArg(), (p, c) -> ScheduleConfig.fromXContent(p), FULL_FIELD, ObjectParser.ValueType.OBJECT); + PARSER.declareField(constructorArg(), (p, c) -> ScheduleConfig.fromXContent(p), INCREMENTAL_FIELD, ObjectParser.ValueType.OBJECT); + } + + public static ConnectorScheduling fromXContentBytes(BytesReference source, XContentType xContentType) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return ConnectorScheduling.fromXContent(parser); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static ConnectorScheduling fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(ACCESS_CONTROL_FIELD.getPreferredName(), accessControl); + builder.field(FULL_FIELD.getPreferredName(), full); + builder.field(INCREMENTAL_FIELD.getPreferredName(), incremental); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + accessControl.writeTo(out); + full.writeTo(out); + incremental.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorScheduling that = (ConnectorScheduling) o; + return Objects.equals(accessControl, that.accessControl) + && Objects.equals(full, that.full) + && Objects.equals(incremental, that.incremental); + } + + @Override + public int hashCode() { + return Objects.hash(accessControl, full, incremental); + } + + public static class Builder { + + private ScheduleConfig accessControl; + private ScheduleConfig full; + private ScheduleConfig incremental; + + public Builder setAccessControl(ScheduleConfig accessControl) { + this.accessControl = accessControl; + return this; + } + + public Builder setFull(ScheduleConfig full) { + this.full = full; + return this; + } + + public Builder setIncremental(ScheduleConfig incremental) { + this.incremental = incremental; + return this; + } + + public ConnectorScheduling build() { + return new ConnectorScheduling(accessControl, full, incremental); + } + } + + public static class ScheduleConfig implements Writeable, ToXContentObject { + private final boolean enabled; + private final Cron interval; + + private static final ParseField ENABLED_FIELD = new ParseField("enabled"); + private static final ParseField INTERVAL_FIELD = new ParseField("interval"); + + /** + * @param enabled flag to disable/enable scheduling + * @param interval CRON expression representing the sync schedule + */ + private ScheduleConfig(boolean enabled, Cron interval) { + this.enabled = enabled; + this.interval = Objects.requireNonNull(interval, INTERVAL_FIELD.getPreferredName()); + } + + public ScheduleConfig(StreamInput in) throws IOException { + this.enabled = in.readBoolean(); + this.interval = new Cron(in.readString()); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "schedule_config", + true, + args -> new Builder().setEnabled((boolean) args[0]).setInterval(new Cron((String) args[1])).build() + ); + + static { + PARSER.declareBoolean(constructorArg(), ENABLED_FIELD); + PARSER.declareString(constructorArg(), INTERVAL_FIELD); + } + + public static ScheduleConfig fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + public static ConstructingObjectParser getParser() { + return PARSER; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(ENABLED_FIELD.getPreferredName(), enabled); + builder.field(INTERVAL_FIELD.getPreferredName(), interval); + } + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(enabled); + out.writeString(interval.toString()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScheduleConfig that = (ScheduleConfig) o; + return enabled == that.enabled && Objects.equals(interval, that.interval); + } + + @Override + public int hashCode() { + return Objects.hash(enabled, interval); + } + + public static class Builder { + + private boolean enabled; + private Cron interval; + + public Builder setEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + public Builder setInterval(Cron interval) { + this.interval = interval; + return this; + } + + public ScheduleConfig build() { + return new ScheduleConfig(enabled, interval); + } + } + } + + public static ConnectorScheduling getDefaultConnectorScheduling() { + return new ConnectorScheduling.Builder().setAccessControl( + new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(false).setInterval(new Cron("0 0 0 * * ?")).build() + ) + .setFull(new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(false).setInterval(new Cron("0 0 0 * * ?")).build()) + .setIncremental(new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(false).setInterval(new Cron("0 0 0 * * ?")).build()) + .build(); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStatus.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStatus.java new file mode 100644 index 0000000000000..b994ccc89c1d9 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorStatus.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import java.util.Locale; + +/** + * Enum representing the various states of a Connector: + *
    + *
  • CREATED: The connector has been created but is not yet configured.
    • + *
  • NEEDS_CONFIGURATION: The connector requires further configuration to become operational.
    • + *
  • CONFIGURED: The connector has been configured but has not yet established a connection.
    • + *
  • CONNECTED: The connector is successfully connected and operational.
    • + *
  • ERROR: The connector encountered an error and may not be operational.
    • + *
+ */ +public enum ConnectorStatus { + CREATED, + NEEDS_CONFIGURATION, + CONFIGURED, + CONNECTED, + ERROR; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfo.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfo.java new file mode 100644 index 0000000000000..2cc158fdd5f64 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfo.java @@ -0,0 +1,286 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +public class ConnectorSyncInfo implements Writeable, ToXContentFragment { + @Nullable + private final String lastAccessControlSyncError; + @Nullable + private final Instant lastAccessControlSyncScheduledAt; + @Nullable + private final ConnectorSyncStatus lastAccessControlSyncStatus; + @Nullable + private final Long lastDeletedDocumentCount; + @Nullable + private final Instant lastIncrementalSyncScheduledAt; + @Nullable + private final Long lastIndexedDocumentCount; + @Nullable + private final Instant lastSeen; + @Nullable + private final String lastSyncError; + @Nullable + private final Instant lastSyncScheduledAt; + @Nullable + private final ConnectorSyncStatus lastSyncStatus; + @Nullable + private final Instant lastSynced; + + /** + * @param lastAccessControlSyncError The last error message related to access control sync, if any. + * @param lastAccessControlSyncScheduledAt The timestamp when the last access control sync was scheduled. + * @param lastAccessControlSyncStatus The status of the last access control sync. + * @param lastDeletedDocumentCount The count of documents last deleted during sync. + * @param lastIncrementalSyncScheduledAt The timestamp when the last incremental sync was scheduled. + * @param lastIndexedDocumentCount The count of documents last indexed during sync. + * @param lastSeen The timestamp when the connector was last active or seen. + * @param lastSyncError The last error message encountered during sync, if any. + * @param lastSyncScheduledAt The timestamp when the last sync was scheduled. + * @param lastSyncStatus The status of the last sync. + * @param lastSynced The timestamp when the connector was last successfully synchronized. + */ + private ConnectorSyncInfo( + String lastAccessControlSyncError, + Instant lastAccessControlSyncScheduledAt, + ConnectorSyncStatus lastAccessControlSyncStatus, + Long lastDeletedDocumentCount, + Instant lastIncrementalSyncScheduledAt, + Long lastIndexedDocumentCount, + Instant lastSeen, + String lastSyncError, + Instant lastSyncScheduledAt, + ConnectorSyncStatus lastSyncStatus, + Instant lastSynced + ) { + this.lastAccessControlSyncError = lastAccessControlSyncError; + this.lastAccessControlSyncScheduledAt = lastAccessControlSyncScheduledAt; + this.lastAccessControlSyncStatus = lastAccessControlSyncStatus; + this.lastDeletedDocumentCount = lastDeletedDocumentCount; + this.lastIncrementalSyncScheduledAt = lastIncrementalSyncScheduledAt; + this.lastIndexedDocumentCount = lastIndexedDocumentCount; + this.lastSeen = lastSeen; + this.lastSyncError = lastSyncError; + this.lastSyncScheduledAt = lastSyncScheduledAt; + this.lastSyncStatus = lastSyncStatus; + this.lastSynced = lastSynced; + } + + public ConnectorSyncInfo(StreamInput in) throws IOException { + this.lastAccessControlSyncError = in.readOptionalString(); + this.lastAccessControlSyncScheduledAt = in.readOptionalInstant(); + this.lastAccessControlSyncStatus = in.readOptionalEnum(ConnectorSyncStatus.class); + this.lastDeletedDocumentCount = in.readOptionalLong(); + this.lastIncrementalSyncScheduledAt = in.readOptionalInstant(); + this.lastIndexedDocumentCount = in.readOptionalLong(); + this.lastSeen = in.readOptionalInstant(); + this.lastSyncError = in.readOptionalString(); + this.lastSyncScheduledAt = in.readOptionalInstant(); + this.lastSyncStatus = in.readOptionalEnum(ConnectorSyncStatus.class); + this.lastSynced = in.readOptionalInstant(); + } + + private static final ParseField LAST_ACCESS_CONTROL_SYNC_ERROR = new ParseField("last_access_control_sync_error"); + private static final ParseField LAST_ACCESS_CONTROL_SYNC_STATUS_FIELD = new ParseField("last_access_control_sync_status"); + private static final ParseField LAST_ACCESS_CONTROL_SYNC_SCHEDULED_AT_FIELD = new ParseField("last_access_control_sync_scheduled_at"); + private static final ParseField LAST_DELETED_DOCUMENT_COUNT_FIELD = new ParseField("last_deleted_document_count"); + private static final ParseField LAST_INCREMENTAL_SYNC_SCHEDULED_AT_FIELD = new ParseField("last_incremental_sync_scheduled_at"); + private static final ParseField LAST_INDEXED_DOCUMENT_COUNT_FIELD = new ParseField("last_indexed_document_count"); + private static final ParseField LAST_SEEN_FIELD = new ParseField("last_seen"); + private static final ParseField LAST_SYNC_ERROR_FIELD = new ParseField("last_sync_error"); + private static final ParseField LAST_SYNC_SCHEDULED_AT_FIELD = new ParseField("last_sync_scheduled_at"); + private static final ParseField LAST_SYNC_STATUS_FIELD = new ParseField("last_sync_status"); + private static final ParseField LAST_SYNCED_FIELD = new ParseField("last_synced"); + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + if (lastAccessControlSyncError != null) { + builder.field(LAST_ACCESS_CONTROL_SYNC_ERROR.getPreferredName(), lastAccessControlSyncError); + } + if (lastAccessControlSyncStatus != null) { + builder.field(LAST_ACCESS_CONTROL_SYNC_STATUS_FIELD.getPreferredName(), lastAccessControlSyncStatus); + } + if (lastAccessControlSyncScheduledAt != null) { + builder.field(LAST_ACCESS_CONTROL_SYNC_SCHEDULED_AT_FIELD.getPreferredName(), lastAccessControlSyncScheduledAt); + } + if (lastDeletedDocumentCount != null) { + builder.field(LAST_DELETED_DOCUMENT_COUNT_FIELD.getPreferredName(), lastDeletedDocumentCount); + } + if (lastIncrementalSyncScheduledAt != null) { + builder.field(LAST_INCREMENTAL_SYNC_SCHEDULED_AT_FIELD.getPreferredName(), lastIncrementalSyncScheduledAt); + } + if (lastIndexedDocumentCount != null) { + builder.field(LAST_INDEXED_DOCUMENT_COUNT_FIELD.getPreferredName(), lastIndexedDocumentCount); + } + if (lastSeen != null) { + builder.field(LAST_SEEN_FIELD.getPreferredName(), lastSeen); + } + if (lastSyncError != null) { + builder.field(LAST_SYNC_ERROR_FIELD.getPreferredName(), lastSyncError); + } + if (lastSyncScheduledAt != null) { + builder.field(LAST_SYNC_SCHEDULED_AT_FIELD.getPreferredName(), lastSyncScheduledAt); + } + if (lastSyncStatus != null) { + builder.field(LAST_SYNC_STATUS_FIELD.getPreferredName(), lastSyncStatus); + } + if (lastSynced != null) { + builder.field(LAST_SYNCED_FIELD.getPreferredName(), lastSynced); + } + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(lastAccessControlSyncError); + out.writeOptionalInstant(lastAccessControlSyncScheduledAt); + out.writeOptionalEnum(lastAccessControlSyncStatus); + out.writeOptionalLong(lastDeletedDocumentCount); + out.writeOptionalInstant(lastIncrementalSyncScheduledAt); + out.writeOptionalLong(lastIndexedDocumentCount); + out.writeOptionalInstant(lastSeen); + out.writeOptionalString(lastSyncError); + out.writeOptionalInstant(lastSyncScheduledAt); + out.writeOptionalEnum(lastSyncStatus); + out.writeOptionalInstant(lastSynced); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ConnectorSyncInfo that = (ConnectorSyncInfo) o; + return Objects.equals(lastAccessControlSyncError, that.lastAccessControlSyncError) + && Objects.equals(lastAccessControlSyncScheduledAt, that.lastAccessControlSyncScheduledAt) + && lastAccessControlSyncStatus == that.lastAccessControlSyncStatus + && Objects.equals(lastDeletedDocumentCount, that.lastDeletedDocumentCount) + && Objects.equals(lastIncrementalSyncScheduledAt, that.lastIncrementalSyncScheduledAt) + && Objects.equals(lastIndexedDocumentCount, that.lastIndexedDocumentCount) + && Objects.equals(lastSeen, that.lastSeen) + && Objects.equals(lastSyncError, that.lastSyncError) + && Objects.equals(lastSyncScheduledAt, that.lastSyncScheduledAt) + && lastSyncStatus == that.lastSyncStatus + && Objects.equals(lastSynced, that.lastSynced); + } + + @Override + public int hashCode() { + return Objects.hash( + lastAccessControlSyncError, + lastAccessControlSyncScheduledAt, + lastAccessControlSyncStatus, + lastDeletedDocumentCount, + lastIncrementalSyncScheduledAt, + lastIndexedDocumentCount, + lastSeen, + lastSyncError, + lastSyncScheduledAt, + lastSyncStatus, + lastSynced + ); + } + + public static class Builder { + + private String lastAccessControlSyncError; + private Instant lastAccessControlSyncScheduledAt; + private ConnectorSyncStatus lastAccessControlSyncStatus; + private Long lastDeletedDocumentCount; + private Instant lastIncrementalSyncScheduledAt; + private Long lastIndexedDocumentCount; + private Instant lastSeen; + private String lastSyncError; + private Instant lastSyncScheduledAt; + private ConnectorSyncStatus lastSyncStatus; + private Instant lastSynced; + + public Builder setLastAccessControlSyncError(String lastAccessControlSyncError) { + this.lastAccessControlSyncError = lastAccessControlSyncError; + return this; + } + + public Builder setLastAccessControlSyncScheduledAt(Instant lastAccessControlSyncScheduledAt) { + this.lastAccessControlSyncScheduledAt = lastAccessControlSyncScheduledAt; + return this; + } + + public Builder setLastAccessControlSyncStatus(ConnectorSyncStatus lastAccessControlSyncStatus) { + this.lastAccessControlSyncStatus = lastAccessControlSyncStatus; + return this; + } + + public Builder setLastDeletedDocumentCount(Long lastDeletedDocumentCount) { + this.lastDeletedDocumentCount = lastDeletedDocumentCount; + return this; + } + + public Builder setLastIncrementalSyncScheduledAt(Instant lastIncrementalSyncScheduledAt) { + this.lastIncrementalSyncScheduledAt = lastIncrementalSyncScheduledAt; + return this; + } + + public Builder setLastIndexedDocumentCount(Long lastIndexedDocumentCount) { + this.lastIndexedDocumentCount = lastIndexedDocumentCount; + return this; + } + + public Builder setLastSeen(Instant lastSeen) { + this.lastSeen = lastSeen; + return this; + } + + public Builder setLastSyncError(String lastSyncError) { + this.lastSyncError = lastSyncError; + return this; + } + + public Builder setLastSyncScheduledAt(Instant lastSyncScheduledAt) { + this.lastSyncScheduledAt = lastSyncScheduledAt; + return this; + } + + public Builder setLastSyncStatus(ConnectorSyncStatus lastSyncStatus) { + this.lastSyncStatus = lastSyncStatus; + return this; + } + + public Builder setLastSynced(Instant lastSynced) { + this.lastSynced = lastSynced; + return this; + } + + public ConnectorSyncInfo build() { + return new ConnectorSyncInfo( + lastAccessControlSyncError, + lastAccessControlSyncScheduledAt, + lastAccessControlSyncStatus, + lastDeletedDocumentCount, + lastIncrementalSyncScheduledAt, + lastIndexedDocumentCount, + lastSeen, + lastSyncError, + lastSyncScheduledAt, + lastSyncStatus, + lastSynced + ); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncStatus.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncStatus.java new file mode 100644 index 0000000000000..97ea570b4f866 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorSyncStatus.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import java.util.Locale; + +/** + * Enum representing the synchronization status of a Connector: + *
    + *
  • CANCELING: The synchronization process is in the process of being canceled.
    • + *
  • CANCELED: The synchronization process has been canceled before completion.
    • + *
  • COMPLETED: The synchronization process has completed successfully.
    • + *
  • ERROR: The synchronization process encountered an error and may not have completed successfully.
    • + *
  • IN_PROGRESS: The synchronization process is currently in progress.
    • + *
  • PENDING: The synchronization process is scheduled and waiting to start.
    • + *
  • SUSPENDED: The synchronization process has been suspended.
    • + *
+ */ +public enum ConnectorSyncStatus { + CANCELING, + CANCELED, + COMPLETED, + ERROR, + IN_PROGRESS, + PENDING, + SUSPENDED; + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java new file mode 100644 index 0000000000000..6abb5ef548be5 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java @@ -0,0 +1,273 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class PutConnectorAction extends ActionType { + + public static final PutConnectorAction INSTANCE = new PutConnectorAction(); + public static final String NAME = "cluster:admin/xpack/connector/put"; + + public PutConnectorAction() { + super(NAME, PutConnectorAction.Response::new); + } + + public static class Request extends ActionRequest implements ToXContentObject { + + private final String connectorId; + + @Nullable + private final String description; + @Nullable + private final String indexName; + @Nullable + private final Boolean isNative; + @Nullable + private final String language; + @Nullable + private final String name; + @Nullable + private final String serviceType; + + public Request( + String connectorId, + String description, + String indexName, + Boolean isNative, + String language, + String name, + String serviceType + ) { + this.connectorId = connectorId; + this.description = description; + this.indexName = indexName; + this.isNative = isNative; + this.language = language; + this.name = name; + this.serviceType = serviceType; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorId = in.readString(); + this.description = in.readOptionalString(); + this.indexName = in.readOptionalString(); + this.isNative = in.readOptionalBoolean(); + this.language = in.readOptionalString(); + this.name = in.readOptionalString(); + this.serviceType = in.readOptionalString(); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "connector_put_request", + false, + ((args, connectorId) -> new Request( + connectorId, + (String) args[0], + (String) args[1], + (Boolean) args[2], + (String) args[3], + (String) args[4], + (String) args[5] + )) + ); + + static { + PARSER.declareString(optionalConstructorArg(), new ParseField("description")); + PARSER.declareString(optionalConstructorArg(), new ParseField("index_name")); + PARSER.declareBoolean(optionalConstructorArg(), new ParseField("is_native")); + PARSER.declareString(optionalConstructorArg(), new ParseField("language")); + PARSER.declareString(optionalConstructorArg(), new ParseField("name")); + PARSER.declareString(optionalConstructorArg(), new ParseField("service_type")); + } + + public static Request fromXContentBytes(String connectorId, BytesReference source, XContentType xContentType) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return Request.fromXContent(parser, connectorId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse: " + source.utf8ToString(), e); + } + } + + public static Request fromXContent(XContentParser parser, String connectorId) throws IOException { + return PARSER.parse(parser, connectorId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + if (description != null) { + builder.field("description", description); + } + if (indexName != null) { + builder.field("index_name", indexName); + } + if (isNative != null) { + builder.field("is_native", isNative); + } + if (language != null) { + builder.field("language", language); + } + if (name != null) { + builder.field("name", name); + } + if (serviceType != null) { + builder.field("service_type", serviceType); + } + } + builder.endObject(); + return builder; + } + + @Override + public ActionRequestValidationException validate() { + + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(getConnectorId())) { + validationException = addValidationError("connector_id cannot be null or empty", validationException); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorId); + out.writeOptionalString(description); + out.writeOptionalString(indexName); + out.writeOptionalBoolean(isNative); + out.writeOptionalString(language); + out.writeOptionalString(name); + out.writeOptionalString(serviceType); + } + + public String getConnectorId() { + return connectorId; + } + + public String getDescription() { + return description; + } + + public String getIndexName() { + return indexName; + } + + public Boolean getIsNative() { + return isNative; + } + + public String getLanguage() { + return language; + } + + public String getName() { + return name; + } + + public String getServiceType() { + return serviceType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorId, request.connectorId) + && Objects.equals(description, request.description) + && Objects.equals(indexName, request.indexName) + && Objects.equals(isNative, request.isNative) + && Objects.equals(language, request.language) + && Objects.equals(name, request.name) + && Objects.equals(serviceType, request.serviceType); + } + + @Override + public int hashCode() { + return Objects.hash(connectorId, description, indexName, isNative, language, name, serviceType); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + final DocWriteResponse.Result result; + + public Response(StreamInput in) throws IOException { + super(in); + result = DocWriteResponse.Result.readFrom(in); + } + + public Response(DocWriteResponse.Result result) { + this.result = result; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + this.result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("result", this.result.getLowercase()); + builder.endObject(); + return builder; + } + + public RestStatus status() { + return switch (result) { + case CREATED -> RestStatus.CREATED; + case NOT_FOUND -> RestStatus.NOT_FOUND; + default -> RestStatus.OK; + }; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return result == response.result; + } + + @Override + public int hashCode() { + return Objects.hash(result); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestPutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestPutConnectorAction.java new file mode 100644 index 0000000000000..e87719943fc29 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestPutConnectorAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.PUT; + +public class RestPutConnectorAction extends BaseRestHandler { + + @Override + public String getName() { + return "connector_put_action"; + } + + @Override + public List routes() { + return List.of(new Route(PUT, "/" + EnterpriseSearch.CONNECTOR_API_ENDPOINT + "/{connector_id}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + PutConnectorAction.Request request = PutConnectorAction.Request.fromXContentBytes( + restRequest.param("connector_id"), + restRequest.content(), + restRequest.getXContentType() + ); + return channel -> client.execute( + PutConnectorAction.INSTANCE, + request, + new RestToXContentListener<>(channel, PutConnectorAction.Response::status, r -> null) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java new file mode 100644 index 0000000000000..013a8f4a8334d --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportPutConnectorAction.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorIndexService; + +import java.util.Objects; + +public class TransportPutConnectorAction extends HandledTransportAction { + + protected final ConnectorIndexService connectorIndexService; + + @Inject + public TransportPutConnectorAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client + ) { + super( + PutConnectorAction.NAME, + transportService, + actionFilters, + PutConnectorAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorIndexService = new ConnectorIndexService(client); + } + + @Override + protected void doExecute(Task task, PutConnectorAction.Request request, ActionListener listener) { + + Boolean isNative = Objects.requireNonNullElse(request.getIsNative(), false); + + Connector connector = new Connector.Builder().setConnectorId(request.getConnectorId()) + .setDescription(request.getDescription()) + .setIndexName(request.getIndexName()) + .setIsNative(isNative) + .setLanguage(request.getLanguage()) + .setName(request.getName()) + .setServiceType(request.getServiceType()) + .build(); + + connectorIndexService.putConnector(connector, listener.map(r -> new PutConnectorAction.Response(r.getResult()))); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorCustomScheduleTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorCustomScheduleTests.java new file mode 100644 index 0000000000000..2509405874869 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorCustomScheduleTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorCustomScheduleTests extends ESTestCase { + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorCustomSchedule testInstance = ConnectorTestUtils.getRandomConnectorCustomSchedule(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(ConnectorCustomSchedule testInstance) throws IOException { + ConnectorCustomSchedule deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorCustomSchedule copyInstance(ConnectorCustomSchedule instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorCustomSchedule::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java new file mode 100644 index 0000000000000..f620586457099 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFeaturesTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorFeaturesTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorFeatures testInstance = ConnectorTestUtils.getRandomConnectorFeatures(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(ConnectorFeatures testInstance) throws IOException { + ConnectorFeatures deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorFeatures copyInstance(ConnectorFeatures instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorFeatures::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFilteringTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFilteringTests.java new file mode 100644 index 0000000000000..c3e6cc2d8b7ad --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorFilteringTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorFilteringTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorFiltering testInstance = ConnectorTestUtils.getRandomConnectorFiltering(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(ConnectorFiltering testInstance) throws IOException { + ConnectorFiltering deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorFiltering copyInstance(ConnectorFiltering instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorFiltering::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java new file mode 100644 index 0000000000000..6e1c42dab130f --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.Before; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.CoreMatchers.anyOf; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { + + private static final int REQUEST_TIMEOUT_SECONDS = 10; + + private ConnectorIndexService connectorIndexService; + + @Before + public void setup() { + this.connectorIndexService = new ConnectorIndexService(client()); + } + + public void testPutConnector() throws Exception { + + Connector connector = ConnectorTestUtils.getRandomConnector(); + + DocWriteResponse resp = awaitPutConnector(connector); + assertThat(resp.status(), anyOf(equalTo(RestStatus.CREATED), equalTo(RestStatus.OK))); + + // TODO: more checks once GET endpoint is implemented :) + } + + private DocWriteResponse awaitPutConnector(Connector connector) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorIndexService.putConnector(connector, new ActionListener<>() { + @Override + public void onResponse(DocWriteResponse indexResponse) { + resp.set(indexResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + }); + assertTrue("Timeout waiting for put request", latch.await(REQUEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from put request", resp.get()); + return resp.get(); + } + +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java new file mode 100644 index 0000000000000..9f9269d60dc70 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIngestPipelineTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorIngestPipelineTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorIngestPipeline testInstance = ConnectorTestUtils.getRandomConnectorIngestPipeline(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(ConnectorIngestPipeline testInstance) throws IOException { + ConnectorIngestPipeline deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorIngestPipeline copyInstance(ConnectorIngestPipeline instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorIngestPipeline::new); + } + +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSchedulingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSchedulingTests.java new file mode 100644 index 0000000000000..cb986ce9992e0 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSchedulingTests.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorSchedulingTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorScheduling testInstance = ConnectorTestUtils.getRandomConnectorScheduling(); + assertTransportSerialization(testInstance); + } + } + + public void testToXContent() throws IOException { + String content = XContentHelper.stripWhitespace(""" + { + "access_control": { + "enabled": false, + "interval": "0 0 0 * * ?" + }, + "full": { + "enabled": false, + "interval": "0 0 0 * * ?" + }, + "incremental": { + "enabled": false, + "interval": "0 0 0 * * ?" + } + }"""); + + ConnectorScheduling scheduling = ConnectorScheduling.fromXContentBytes(new BytesArray(content), XContentType.JSON); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(scheduling, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + ConnectorScheduling parsed; + try (XContentParser parser = createParser(XContentType.JSON.xContent(), originalBytes)) { + parsed = ConnectorScheduling.fromXContent(parser); + } + assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); + } + + private void assertTransportSerialization(ConnectorScheduling testInstance) throws IOException { + ConnectorScheduling deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorScheduling copyInstance(ConnectorScheduling instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorScheduling::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfoTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfoTests.java new file mode 100644 index 0000000000000..0e6a4792d2145 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorSyncInfoTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorSyncInfoTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + ConnectorSyncInfo testInstance = ConnectorTestUtils.getRandomConnectorSyncInfo(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(ConnectorSyncInfo testInstance) throws IOException { + ConnectorSyncInfo deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private ConnectorSyncInfo copyInstance(ConnectorSyncInfo instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, ConnectorSyncInfo::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java new file mode 100644 index 0000000000000..59590b3d7e85e --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTestUtils.java @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.xpack.application.connector.action.PutConnectorAction; +import org.elasticsearch.xpack.core.scheduler.Cron; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; +import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.elasticsearch.test.ESTestCase.randomFrom; +import static org.elasticsearch.test.ESTestCase.randomInt; +import static org.elasticsearch.test.ESTestCase.randomList; +import static org.elasticsearch.test.ESTestCase.randomLong; + +public final class ConnectorTestUtils { + public static PutConnectorAction.Request getRandomPutConnectorActionRequest() { + return new PutConnectorAction.Request( + randomAlphaOfLengthBetween(5, 15), + randomFrom(randomAlphaOfLengthBetween(5, 15)), + randomFrom(randomAlphaOfLengthBetween(5, 15)), + randomFrom(randomBoolean()), + randomFrom(randomAlphaOfLengthBetween(5, 15)), + randomFrom(randomAlphaOfLengthBetween(5, 15)), + randomFrom(randomAlphaOfLengthBetween(5, 15)) + ); + } + + public static ConnectorScheduling getRandomConnectorScheduling() { + return new ConnectorScheduling.Builder().setAccessControl( + new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(randomBoolean()).setInterval(getRandomCronExpression()).build() + ) + .setFull( + new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(randomBoolean()).setInterval(getRandomCronExpression()).build() + ) + .setIncremental( + new ConnectorScheduling.ScheduleConfig.Builder().setEnabled(randomBoolean()).setInterval(getRandomCronExpression()).build() + ) + .build(); + } + + public static ConnectorIngestPipeline getRandomConnectorIngestPipeline() { + return new ConnectorIngestPipeline.Builder().setName(randomAlphaOfLengthBetween(5, 15)) + .setExtractBinaryContent(randomBoolean()) + .setReduceWhitespace(randomBoolean()) + .setRunMlInference(randomBoolean()) + .build(); + } + + public static ConnectorSyncInfo getRandomConnectorSyncInfo() { + return new ConnectorSyncInfo.Builder().setLastAccessControlSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setLastAccessControlSyncScheduledAt(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .setLastAccessControlSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) + .setLastDeletedDocumentCount(randomFrom(new Long[] { null, randomLong() })) + .setLastIncrementalSyncScheduledAt(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .setLastIndexedDocumentCount(randomFrom(new Long[] { null, randomLong() })) + .setLastSeen(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .setLastSyncError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setLastSyncScheduledAt(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .setLastSyncStatus(randomFrom(new ConnectorSyncStatus[] { null, getRandomSyncStatus() })) + .setLastSynced(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .build(); + } + + public static ConnectorFeatures getRandomConnectorFeatures() { + return new ConnectorFeatures.Builder().setDocumentLevelSecurityEnabled(randomFrom(new Boolean[] { null, randomBoolean() })) + .setFilteringRules(randomFrom(new Boolean[] { null, randomBoolean() })) + .setFilteringAdvancedConfig(randomFrom(new Boolean[] { null, randomBoolean() })) + .setIncrementalSyncEnabled(randomFrom(new Boolean[] { null, randomBoolean() })) + .setSyncRulesAdvancedEnabled(randomFrom(new Boolean[] { null, randomBoolean() })) + .setSyncRulesBasicEnabled(randomFrom(new Boolean[] { null, randomBoolean() })) + .build(); + } + + public static ConnectorCustomSchedule getRandomConnectorCustomSchedule() { + return new ConnectorCustomSchedule.Builder().setInterval(getRandomCronExpression()) + .setEnabled(randomBoolean()) + .setLastSynced(randomFrom(new Instant[] { null, Instant.ofEpochMilli(randomLong()) })) + .setName(randomAlphaOfLength(10)) + .setConfigurationOverrides( + new ConnectorCustomSchedule.ConfigurationOverrides.Builder().setMaxCrawlDepth(randomInt()) + .setSitemapDiscoveryDisabled(randomBoolean()) + .setDomainAllowList(randomList(1, 5, () -> randomAlphaOfLength(5))) + .setSeedUrls(randomList(1, 5, () -> randomAlphaOfLength(5))) + .setSitemapUrls(randomList(1, 5, () -> randomAlphaOfLength(5))) + .build() + ) + .build(); + } + + public static ConnectorFiltering getRandomConnectorFiltering() { + + Instant currentTimestamp = Instant.now(); + + return new ConnectorFiltering.Builder().setActive( + new ConnectorFiltering.FilteringRules.Builder().setAdvancedSnippetCreatedAt(currentTimestamp) + .setAdvancedSnippetUpdatedAt(currentTimestamp) + .setAdvancedSnippetValue(Collections.emptyMap()) + .setRules( + List.of( + new ConnectorFiltering.FilteringRule.Builder().setCreatedAt(currentTimestamp) + .setField(randomAlphaOfLength(10)) + .setId(randomAlphaOfLength(10)) + .setOrder(randomInt()) + .setPolicy(getRandomFilteringPolicy()) + .setRule(getRandomFilteringRule()) + .setUpdatedAt(currentTimestamp) + .setValue(randomAlphaOfLength(10)) + .build() + ) + ) + .setValidationErrors(Collections.emptyList()) + .setValidationState(getRandomFilteringValidationState()) + .build() + ) + .setDomain(randomAlphaOfLength(10)) + .setDraft( + new ConnectorFiltering.FilteringRules.Builder().setAdvancedSnippetCreatedAt(currentTimestamp) + .setAdvancedSnippetUpdatedAt(currentTimestamp) + .setAdvancedSnippetValue(Collections.emptyMap()) + .setRules( + List.of( + new ConnectorFiltering.FilteringRule.Builder().setCreatedAt(currentTimestamp) + .setField(randomAlphaOfLength(10)) + .setId(randomAlphaOfLength(10)) + .setOrder(randomInt()) + .setPolicy(getRandomFilteringPolicy()) + .setRule(getRandomFilteringRule()) + .setUpdatedAt(currentTimestamp) + .setValue(randomAlphaOfLength(10)) + .build() + ) + ) + .setValidationErrors(Collections.emptyList()) + .setValidationState(getRandomFilteringValidationState()) + .build() + ) + .build(); + } + + public static Connector getRandomConnector() { + return new Connector.Builder().setConnectorId(randomAlphaOfLength(10)) + .setApiKeyId(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setConfiguration(Collections.emptyMap()) + .setCustomScheduling(randomBoolean() ? getRandomConnectorCustomSchedule() : null) + .setDescription(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setError(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setFeatures(randomBoolean() ? getRandomConnectorFeatures() : null) + .setFiltering(randomBoolean() ? List.of(getRandomConnectorFiltering()) : null) + .setIndexName(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setIsNative(randomBoolean()) + .setLanguage(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setSyncInfo(getRandomConnectorSyncInfo()) + .setName(randomFrom(new String[] { null, randomAlphaOfLength(10) })) + .setPipeline(randomBoolean() ? getRandomConnectorIngestPipeline() : null) + .setScheduling(randomBoolean() ? getRandomConnectorScheduling() : null) + .setStatus(getRandomConnectorStatus()) + .setSyncCursor(randomFrom(new Object[] { null, randomAlphaOfLength(1) })) + .setSyncNow(randomBoolean()) + .build(); + } + + /** + * Second (0 - 59) Minute (0 - 59) Hour (0 - 23) Day of month (1 - 31) Month (1 - 12) + */ + private static Cron getRandomCronExpression() { + return new Cron( + String.format( + Locale.ROOT, + "%d %d %d %d %d ?", + randomInt(59), + randomInt(59), + randomInt(23), + randomInt(30) + 1, + randomInt(11) + 1 + ) + ); + } + + private static ConnectorSyncStatus getRandomSyncStatus() { + ConnectorSyncStatus[] values = ConnectorSyncStatus.values(); + return values[randomInt(values.length - 1)]; + } + + private static ConnectorStatus getRandomConnectorStatus() { + ConnectorStatus[] values = ConnectorStatus.values(); + return values[randomInt(values.length - 1)]; + } + + private static ConnectorFiltering.FilteringPolicy getRandomFilteringPolicy() { + ConnectorFiltering.FilteringPolicy[] values = ConnectorFiltering.FilteringPolicy.values(); + return values[randomInt(values.length - 1)]; + } + + private static ConnectorFiltering.FilteringRuleCondition getRandomFilteringRule() { + ConnectorFiltering.FilteringRuleCondition[] values = ConnectorFiltering.FilteringRuleCondition.values(); + return values[randomInt(values.length - 1)]; + } + + private static ConnectorFiltering.FilteringValidationState getRandomFilteringValidationState() { + ConnectorFiltering.FilteringValidationState[] values = ConnectorFiltering.FilteringValidationState.values(); + return values[randomInt(values.length - 1)]; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java new file mode 100644 index 0000000000000..7d200ff9bdd94 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.CoreMatchers.equalTo; + +public class ConnectorTests extends ESTestCase { + + private NamedWriteableRegistry namedWriteableRegistry; + + @Before + public void registerNamedObjects() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedWriteables = searchModule.getNamedWriteables(); + namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); + } + + public final void testRandomSerialization() throws IOException { + for (int runs = 0; runs < 10; runs++) { + Connector testInstance = ConnectorTestUtils.getRandomConnector(); + assertTransportSerialization(testInstance); + } + } + + private void assertTransportSerialization(Connector testInstance) throws IOException { + Connector deserializedInstance = copyInstance(testInstance); + assertNotSame(testInstance, deserializedInstance); + assertThat(testInstance, equalTo(deserializedInstance)); + } + + private Connector copyInstance(Connector instance) throws IOException { + return copyWriteable(instance, namedWriteableRegistry, Connector::new); + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..f618b4562fdc9 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class PutConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { + + private String connectorId; + + @Override + protected Writeable.Reader instanceReader() { + return PutConnectorAction.Request::new; + } + + @Override + protected PutConnectorAction.Request createTestInstance() { + PutConnectorAction.Request testInstance = ConnectorTestUtils.getRandomPutConnectorActionRequest(); + this.connectorId = testInstance.getConnectorId(); + return testInstance; + } + + @Override + protected PutConnectorAction.Request mutateInstance(PutConnectorAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected PutConnectorAction.Request doParseInstance(XContentParser parser) throws IOException { + return PutConnectorAction.Request.fromXContent(parser, this.connectorId); + } + + @Override + protected PutConnectorAction.Request mutateInstanceForVersion(PutConnectorAction.Request instance, TransportVersion version) { + return instance; + } +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionResponseBWCSerializingTests.java new file mode 100644 index 0000000000000..94be7e9b6b9ca --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionResponseBWCSerializingTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; + +import java.io.IOException; + +public class PutConnectorActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { + @Override + protected Writeable.Reader instanceReader() { + return PutConnectorAction.Response::new; + } + + @Override + protected PutConnectorAction.Response createTestInstance() { + return new PutConnectorAction.Response(randomFrom(DocWriteResponse.Result.values())); + } + + @Override + protected PutConnectorAction.Response mutateInstance(PutConnectorAction.Response instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + @Override + protected PutConnectorAction.Response mutateInstanceForVersion(PutConnectorAction.Response instance, TransportVersion version) { + return instance; + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 7a65a03277faf..5c70b64aa7b3a 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -123,6 +123,7 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", + "cluster:admin/xpack/connector/put", "cluster:admin/xpack/deprecation/info", "cluster:admin/xpack/deprecation/nodes/info", "cluster:admin/xpack/enrich/delete", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 475102e1a2152..798396c249e75 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -27,6 +27,7 @@ import static org.elasticsearch.synonyms.SynonymsManagementAPIService.SYNONYMS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.APM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENT_SEARCH_ORIGIN; @@ -151,6 +152,7 @@ public static void switchUserBasedOnActionOriginAndExecute( case LOGSTASH_MANAGEMENT_ORIGIN: case FLEET_ORIGIN: case ENT_SEARCH_ORIGIN: + case CONNECTORS_ORIGIN: case INFERENCE_ORIGIN: case TASKS_ORIGIN: // TODO use a more limited user for tasks securityContext.executeAsInternalUser(InternalUsers.XPACK_USER, version, consumer); From 9206a4795520d08c44b015b1eab0919d32a38ec7 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 23 Nov 2023 18:54:07 +0200 Subject: [PATCH 06/35] ESQL: Make fieldcaps calls lighter (#102510) Fixes https://github.com/elastic/elasticsearch/issues/101763 Fixes https://github.com/elastic/elasticsearch/issues/102393. Tests updates pending. This change will make queries that don't need any fields from the `_field_caps` call to actually ask only for `_index` field, instead of asking all of them. This will make the `_field_caps` call itself much lighter and its response size almost inexistent (it contains only the list of indices and `_index`). --- docs/changelog/102510.yaml | 7 ++ .../src/main/resources/keep.csv-spec | 3 +- .../src/main/resources/stats.csv-spec | 6 +- .../xpack/esql/analysis/Analyzer.java | 5 +- .../xpack/esql/session/EsqlSession.java | 28 +++-- .../xpack/esql/analysis/AnalyzerTests.java | 102 +++++++++++++++++- .../optimizer/LogicalPlanOptimizerTests.java | 10 ++ .../session/IndexResolverFieldNamesTests.java | 100 +++++++++++++---- .../esql/type/EsqlDataTypeRegistryTests.java | 3 +- .../resources/empty_field_caps_response.json | 16 +++ .../xpack/ql/index/IndexResolver.java | 40 ++++--- .../analysis/index/IndexResolverTests.java | 8 +- 12 files changed, 276 insertions(+), 52 deletions(-) create mode 100644 docs/changelog/102510.yaml create mode 100644 x-pack/plugin/esql/src/test/resources/empty_field_caps_response.json diff --git a/docs/changelog/102510.yaml b/docs/changelog/102510.yaml new file mode 100644 index 0000000000000..2b654b5c85929 --- /dev/null +++ b/docs/changelog/102510.yaml @@ -0,0 +1,7 @@ +pr: 102510 +summary: "ESQL: Make fieldcaps calls lighter" +area: ES|QL +type: enhancement +issues: + - 101763 + - 102393 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec index 3637081c3c4b6..facf06eb6a960 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec @@ -533,8 +533,7 @@ emp_no:integer | salary_change:double |salary_change.int:integer|salary_chan projectAllButConstant -from employees | eval c = 1 | keep c | limit 2 -; +from employees | eval c = 1 | keep c | limit 2; c:i 1 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 083ff90397623..6050dba6acf3b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -532,10 +532,14 @@ c:l 10 ; +countStar +from employees | stats count=count(*) | sort count desc | limit 0; +count:l +; countAllGrouped -from employees | stats c = count(*) by languages | rename languages as l | sort l DESC ; +from employees | stats c = count(*) by languages | rename languages as l | sort l DESC; c:l | l:i 10 |null diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 557da9639a086..24a9076f69768 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -81,6 +81,9 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED; public class Analyzer extends ParameterizedRuleExecutor { + static final List NO_FIELDS = List.of( + new ReferenceAttribute(Source.EMPTY, "", DataTypes.NULL, null, Nullability.TRUE, null, false) + ); private static final Iterable> rules; static { @@ -142,7 +145,7 @@ protected LogicalPlan rule(EsqlUnresolvedRelation plan, AnalyzerContext context) EsIndex esIndex = context.indexResolution().get(); var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(plan.metadataFields()); - return new EsRelation(plan.source(), esIndex, attributes); + return new EsRelation(plan.source(), esIndex, attributes.isEmpty() ? NO_FIELDS : attributes); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index d1a073f64fe81..7adcb0a1f9623 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.TypedParamValue; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Keep; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -37,6 +38,7 @@ import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.AttributeSet; +import org.elasticsearch.xpack.ql.expression.EmptyAttribute; import org.elasticsearch.xpack.ql.expression.MetadataAttribute; import org.elasticsearch.xpack.ql.expression.UnresolvedStar; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -185,12 +187,8 @@ private void preAnalyzeIndices(LogicalPlan parsed, ActionListener void preAnalyzeIndices(LogicalPlan parsed, ActionListener void preAnalyzeIndices(LogicalPlan parsed, ActionListener fieldNames(LogicalPlan parsed) { + static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields) { if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) { // no explicit columns selection, for example "from employees" return IndexResolver.ALL_FIELDS; @@ -242,6 +243,12 @@ static Set fieldNames(LogicalPlan parsed) { for (Attribute extracted : re.extractedFields()) { references.removeIf(attr -> matchByName(attr, extracted.qualifiedName(), false)); } + } else if (p instanceof Enrich) { + AttributeSet enrichRefs = p.references(); + // Enrich adds an EmptyAttribute if no match field is specified + // The exact name of the field will be added later as part of enrichPolicyMatchFields Set + enrichRefs.removeIf(attr -> attr instanceof EmptyAttribute); + references.addAll(enrichRefs); } else { references.addAll(p.references()); if (p instanceof Keep) { @@ -266,10 +273,13 @@ static Set fieldNames(LogicalPlan parsed) { // otherwise, in some edge cases, we will fail to ask for "*" (all fields) instead references.removeIf(a -> a instanceof MetadataAttribute || MetadataAttribute.isSupported(a.qualifiedName())); Set fieldNames = references.names(); - if (fieldNames.isEmpty()) { - return IndexResolver.ALL_FIELDS; + if (fieldNames.isEmpty() && enrichPolicyMatchFields.isEmpty()) { + // there cannot be an empty list of fields, we'll ask the simplest and lightest one instead: _index + return IndexResolver.INDEX_METADATA_FIELD; } else { fieldNames.addAll(subfields(fieldNames)); + fieldNames.addAll(enrichPolicyMatchFields); + fieldNames.addAll(subfields(enrichPolicyMatchFields)); return fieldNames; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 6e75eea75f655..87c5310889023 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -7,13 +7,23 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; import org.elasticsearch.xpack.esql.plan.logical.EsqlUnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.esql.session.EsqlSession; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import org.elasticsearch.xpack.ql.expression.Alias; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expressions; @@ -24,22 +34,29 @@ import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.index.IndexResolver; import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Aggregate; import org.elasticsearch.xpack.ql.plan.logical.EsRelation; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.TypesTests; +import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -68,7 +85,7 @@ public void testIndexResolution() { var plan = analyzer.analyze(UNRESOLVED_RELATION); var limit = as(plan, Limit.class); - assertEquals(new EsRelation(EMPTY, idx, false), limit.child()); + assertEquals(new EsRelation(EMPTY, idx, NO_FIELDS), limit.child()); } public void testFailOnUnresolvedIndex() { @@ -86,7 +103,7 @@ public void testIndexWithClusterResolution() { var plan = analyzer.analyze(UNRESOLVED_RELATION); var limit = as(plan, Limit.class); - assertEquals(new EsRelation(EMPTY, idx, false), limit.child()); + assertEquals(new EsRelation(EMPTY, idx, NO_FIELDS), limit.child()); } public void testAttributeResolution() { @@ -1120,6 +1137,61 @@ public void testAggsWithoutAggAndFollowingCommand() throws Exception { assertEquals(agg.groupings(), agg.aggregates()); } + public void testEmptyEsRelationOnLimitZeroWithCount() throws IOException { + var query = """ + from test* + | stats count=count(*) + | sort count desc + | limit 0"""; + var plan = analyzeWithEmptyFieldCapsResponse(query); + var limit = as(plan, Limit.class); + limit = as(limit.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(0)); + var orderBy = as(limit.child(), OrderBy.class); + var agg = as(orderBy.child(), Aggregate.class); + assertEmptyEsRelation(agg.child()); + } + + public void testEmptyEsRelationOnConstantEvalAndKeep() throws IOException { + var query = """ + from test* + | eval c = 1 + | keep c + | limit 2"""; + var plan = analyzeWithEmptyFieldCapsResponse(query); + var limit = as(plan, Limit.class); + limit = as(limit.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(2)); + var project = as(limit.child(), EsqlProject.class); + var eval = as(project.child(), Eval.class); + assertEmptyEsRelation(eval.child()); + } + + public void testEmptyEsRelationOnConstantEvalAndStats() throws IOException { + var query = """ + from test* + | limit 10 + | eval x = 1 + | stats c = count(x)"""; + var plan = analyzeWithEmptyFieldCapsResponse(query); + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var eval = as(agg.child(), Eval.class); + limit = as(eval.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(10)); + assertEmptyEsRelation(limit.child()); + } + + public void testEmptyEsRelationOnCountStar() throws IOException { + var query = """ + from test* + | stats c = count(*)"""; + var plan = analyzeWithEmptyFieldCapsResponse(query); + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertEmptyEsRelation(agg.child()); + } + public void testUnsupportedFieldsInStats() { var errorMsg = "Cannot use field [point] with unsupported type [geo_point]"; @@ -1361,4 +1433,30 @@ protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); } + private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throws IOException { + IndexResolution resolution = IndexResolver.mergedMappings( + EsqlDataTypeRegistry.INSTANCE, + "test*", + readFieldCapsResponse("empty_field_caps_response.json"), + EsqlSession::specificValidity, + IndexResolver.PRESERVE_PROPERTIES, + IndexResolver.INDEX_METADATA_FIELD + ); + var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query)); + return analyze(query, analyzer); + } + + private static FieldCapabilitiesResponse readFieldCapsResponse(String resourceName) throws IOException { + InputStream stream = AnalyzerTests.class.getResourceAsStream("/" + resourceName); + BytesReference ref = Streams.readFully(stream); + XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, ref, XContentType.JSON); + return FieldCapabilitiesResponse.fromXContent(parser); + } + + private void assertEmptyEsRelation(LogicalPlan plan) { + assertThat(plan, instanceOf(EsRelation.class)); + EsRelation esRelation = (EsRelation) plan; + assertThat(esRelation.output(), equalTo(NO_FIELDS)); + assertTrue(esRelation.index().mapping().isEmpty()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index b82bb46ec103e..352dccc046588 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -2511,6 +2511,16 @@ public void testMultipleRenameStatsDropGroupMultirow() { var row = as(agg.child(), EsRelation.class); } + public void testLimitZeroUsesLocalRelation() { + LogicalPlan plan = optimizedPlan(""" + from test + | stats count=count(*) + | sort count desc + | limit 0"""); + + assertThat(plan, instanceOf(LocalRelation.class)); + } + private T aliased(Expression exp, Class clazz) { var alias = as(exp, Alias.class); return as(alias.child(), clazz); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 5736353223f6e..0aaf4a1a18e32 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -10,9 +10,11 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.parser.EsqlParser; +import java.util.Collections; import java.util.Set; import static org.elasticsearch.xpack.ql.index.IndexResolver.ALL_FIELDS; +import static org.elasticsearch.xpack.ql.index.IndexResolver.INDEX_METADATA_FIELD; import static org.hamcrest.Matchers.equalTo; public class IndexResolverFieldNamesTests extends ESTestCase { @@ -27,6 +29,10 @@ public void testBasicFromCommandWithMetadata() { assertFieldNames("from test [metadata _index, _id, _version]", ALL_FIELDS); } + public void testBasicEvalAndDrop() { + assertFieldNames("from test | eval x = 1 | drop x", ALL_FIELDS); + } + public void testSimple1() { assertFieldNames( "from employees | sort emp_no | keep emp_no, still_hired | limit 3", @@ -278,18 +284,24 @@ public void testMultivalueInput1() { | keep emp_no, a, b, c""", Set.of("emp_no", "emp_no.*", "job_positions", "job_positions.*")); } + public void testLimitZero() { + assertFieldNames(""" + FROM employees + | LIMIT 0""", ALL_FIELDS); + } + public void testDocsDropHeight() { assertFieldNames(""" FROM employees | DROP height - | LIMIT 0""", Set.of("*")); + | LIMIT 0""", ALL_FIELDS); } public void testDocsDropHeightWithWildcard() { assertFieldNames(""" FROM employees | DROP height* - | LIMIT 0""", Set.of("*")); + | LIMIT 0""", ALL_FIELDS); } public void testDocsEval() { @@ -312,7 +324,7 @@ public void testDocsKeepDoubleWildcard() { assertFieldNames(""" FROM employees | KEEP h*, * - | LIMIT 0""", Set.of("*")); + | LIMIT 0""", ALL_FIELDS); } public void testDocsRename() { @@ -339,7 +351,7 @@ public void testDocsStats() { } public void testSortWithLimitOne_DropHeight() { - assertFieldNames("from employees | sort languages | limit 1 | drop height*", Set.of("*")); + assertFieldNames("from employees | sort languages | limit 1 | drop height*", ALL_FIELDS); } public void testDropAllColumns() { @@ -378,7 +390,7 @@ public void testUselessEnrich() { from employees | eval x = "abc" | enrich languages_policy on x - | limit 1""", Set.of("*")); + | limit 1""", ALL_FIELDS); } public void testSimpleSortLimit() { @@ -608,11 +620,11 @@ public void testMultivalueInput() { } public void testSelectAll() { - assertFieldNames("FROM apps [metadata _id]", Set.of("*")); + assertFieldNames("FROM apps [metadata _id]", ALL_FIELDS); } public void testFilterById() { - assertFieldNames("FROM apps [metadata _id]| WHERE _id == \"4\"", Set.of("*")); + assertFieldNames("FROM apps [metadata _id]| WHERE _id == \"4\"", ALL_FIELDS); } public void testKeepId() { @@ -640,7 +652,7 @@ public void testConcatId() { } public void testStatsOnId() { - assertFieldNames("FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id)", Set.of("*")); + assertFieldNames("FROM apps [metadata _id] | stats c = count(_id), d = count_distinct(_id)", INDEX_METADATA_FIELD); } public void testStatsOnIdByGroup() { @@ -758,7 +770,7 @@ public void testRenameDrop() { | rename hire_date as x, emp_no as y | drop first_name, last_name, gender, birth_date, salary, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* - | limit 5""", Set.of("*")); + | limit 5""", ALL_FIELDS); } public void testMaxOfLong() { @@ -960,7 +972,7 @@ public void testRenameReuseAlias() { assertFieldNames(""" from test | rename emp_no as e, first_name as e - """, Set.of("*")); + """, ALL_FIELDS); } public void testIfDuplicateNamesGroupingHasPriority() { @@ -1027,7 +1039,7 @@ public void testEvalOverride() { } public void testBasicWildcardKeep() { - assertFieldNames("from test | keep *", Set.of("*")); + assertFieldNames("from test | keep *", ALL_FIELDS); } public void testBasicWildcardKeep2() { @@ -1041,7 +1053,7 @@ public void testWildcardKeep() { assertFieldNames(""" from test | keep first_name, *, last_name - """, Set.of("*")); + """, ALL_FIELDS); } public void testProjectThenDropName() { @@ -1073,21 +1085,21 @@ public void testProjectDropPattern() { from test | keep * | drop *_name - """, Set.of("*")); + """, ALL_FIELDS); } public void testProjectDropNoStarPattern() { assertFieldNames(""" from test | drop *_name - """, Set.of("*")); + """, ALL_FIELDS); } public void testProjectOrderPatternWithRest() { assertFieldNames(""" from test | keep *name, *, emp_no - """, Set.of("*")); + """, ALL_FIELDS); } public void testProjectDropPatternAndKeepOthers() { @@ -1105,18 +1117,70 @@ public void testAliasesThatGetDropped() { | where first_name like "%A" | eval first_name = concat(first_name, "xyz") | drop first_name - """, Set.of("*")); + """, ALL_FIELDS); } public void testWhereClauseNoProjection() { assertFieldNames(""" from test | where first_name is not null - """, Set.of("*")); + """, ALL_FIELDS); + } + + public void testCountAllGrouped() { + assertFieldNames(""" + from test + | stats c = count(*) by languages + | rename languages as l + | sort l DESC + """, Set.of("languages", "languages.*")); + } + + public void testCountAllAndOtherStatGrouped() { + assertFieldNames(""" + from test + | stats c = count(*), min = min(emp_no) by languages + | sort languages + """, Set.of("emp_no", "emp_no.*", "languages", "languages.*")); + } + + public void testCountAllWithEval() { + assertFieldNames(""" + from test + | rename languages as l + | stats min = min(salary) by l + | eval x = min + 1 + | stats ca = count(*), cx = count(x) by l + | sort l + """, Set.of("languages", "languages.*", "salary", "salary.*")); + } + + public void testCountStar() { + assertFieldNames(""" + from test + | stats count=count(*) + | sort count desc + | limit 0 + """, INDEX_METADATA_FIELD); + } + + public void testEnrichOnDefaultFieldWithKeep() { + Set fieldNames = EsqlSession.fieldNames(parser.createStatement(""" + from employees + | enrich languages_policy + | keep emp_no"""), Set.of("language_name")); + assertThat(fieldNames, equalTo(Set.of("emp_no", "emp_no.*", "language_name", "language_name.*"))); + } + + public void testEnrichOnDefaultField() { + Set fieldNames = EsqlSession.fieldNames(parser.createStatement(""" + from employees + | enrich languages_policy"""), Set.of("language_name")); + assertThat(fieldNames, equalTo(ALL_FIELDS)); } private void assertFieldNames(String query, Set expected) { - Set fieldNames = EsqlSession.fieldNames(parser.createStatement(query)); + Set fieldNames = EsqlSession.fieldNames(parser.createStatement(query), Collections.emptySet()); assertThat(fieldNames, equalTo(expected)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index 1b8ab355cc2eb..e4fa78fac0dee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -57,7 +57,8 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, "idx-*", caps, EsqlSession::specificValidity, - IndexResolver.PRESERVE_PROPERTIES + IndexResolver.PRESERVE_PROPERTIES, + null ); EsField f = resolution.get().mapping().get(fieldCap.getName()); diff --git a/x-pack/plugin/esql/src/test/resources/empty_field_caps_response.json b/x-pack/plugin/esql/src/test/resources/empty_field_caps_response.json new file mode 100644 index 0000000000000..fe8b293e3c0b9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/empty_field_caps_response.json @@ -0,0 +1,16 @@ +{ + "indices": [ + "test1", + "test2" + ], + "fields": { + "_index": { + "_index": { + "type": "_index", + "metadata_field": true, + "searchable": true, + "aggregatable": true + } + } + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 291722f42ca94..fd15a69977f02 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -135,6 +135,7 @@ public String toString() { ); public static final Set ALL_FIELDS = Set.of("*"); + public static final Set INDEX_METADATA_FIELD = Set.of("_index"); public static final String UNMAPPED = "unmapped"; private final Client client; @@ -359,7 +360,7 @@ public void resolveAsMergedMapping( client.fieldCaps( fieldRequest, listener.delegateFailureAndWrap( - (l, response) -> l.onResponse(mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, null)) + (l, response) -> l.onResponse(mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, null, null)) ) ); } @@ -371,14 +372,16 @@ public void resolveAsMergedMapping( Map runtimeMappings, ActionListener listener, BiFunction, InvalidMappedField> specificValidityVerifier, - BiConsumer fieldUpdater - + BiConsumer fieldUpdater, + Set allowedMetadataFields ) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, fieldNames, includeFrozen, runtimeMappings); client.fieldCaps( fieldRequest, listener.delegateFailureAndWrap( - (l, response) -> l.onResponse(mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, fieldUpdater)) + (l, response) -> l.onResponse( + mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, fieldUpdater, allowedMetadataFields) + ) ) ); } @@ -389,7 +392,7 @@ public static IndexResolution mergedMappings( FieldCapabilitiesResponse fieldCapsResponse, BiFunction, InvalidMappedField> specificValidityVerifier ) { - return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, specificValidityVerifier, null); + return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, specificValidityVerifier, null, null); } public static IndexResolution mergedMappings( @@ -397,7 +400,8 @@ public static IndexResolution mergedMappings( String indexPattern, FieldCapabilitiesResponse fieldCapsResponse, BiFunction, InvalidMappedField> specificValidityVerifier, - BiConsumer fieldUpdater + BiConsumer fieldUpdater, + Set allowedMetadataFields ) { if (fieldCapsResponse.getIndices().length == 0) { @@ -470,7 +474,8 @@ public static IndexResolution mergedMappings( null, i -> indexPattern, validityVerifier, - fieldUpdater + fieldUpdater, + allowedMetadataFields ); if (indices.size() > 1) { @@ -494,7 +499,7 @@ public static IndexResolution mergedMappings( String indexPattern, FieldCapabilitiesResponse fieldCapsResponse ) { - return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, (fieldName, types) -> null, null); + return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, (fieldName, types) -> null, null, null); } private static EsField createField( @@ -662,7 +667,7 @@ public static List separateMappings( FieldCapabilitiesResponse fieldCaps, Map> aliases ) { - return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null, null); + return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null, null, null); } private static class Fields { @@ -681,7 +686,8 @@ private static List buildIndices( Map> aliases, Function indexNameProcessor, BiFunction, InvalidMappedField> validityVerifier, - BiConsumer fieldUpdater + BiConsumer fieldUpdater, + Set allowedMetadataFields ) { if ((fieldCapsResponse.getIndices() == null || fieldCapsResponse.getIndices().length == 0) @@ -706,8 +712,9 @@ private static List buildIndices( final Map> fieldCaps = fieldCapsResponse.get(); for (Entry> entry : fieldCaps.entrySet()) { String fieldName = entry.getKey(); - // skip metadata field! - if (fieldCapsResponse.isMetadataField(fieldName) == false) { + // skip specific metadata fields + if ((allowedMetadataFields != null && allowedMetadataFields.contains(fieldName)) + || fieldCapsResponse.isMetadataField(fieldName) == false) { sortedFields.put(fieldName, entry.getValue()); } } @@ -718,6 +725,10 @@ private static List buildIndices( final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types); // apply verification for fields belonging to index aliases Map invalidFieldsForAliases = getInvalidFieldsForAliases(fieldName, types, aliases); + // For ESQL there are scenarios where there is no field asked from field_caps and the field_caps response only contains + // the list of indices. To be able to still have an "indices" list properly built (even if empty), the metadata fields are + // accepted but not actually added to each index hierarchy. + boolean isMetadataField = allowedMetadataFields != null && allowedMetadataFields.contains(fieldName); // check each type for (Entry typeEntry : types.entrySet()) { @@ -750,7 +761,8 @@ private static List buildIndices( Fields indexFields = indices.computeIfAbsent(indexName, k -> new Fields()); EsField field = indexFields.flattedMapping.get(fieldName); // create field hierarchy or update it in case of an invalid field - if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) { + if (isMetadataField == false + && (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false))) { createField(typeRegistry, fieldName, indexFields, fieldCaps, invalidField, typeCap); // In evolving mappings, it is possible for a field to be promoted to an object in new indices @@ -777,7 +789,7 @@ private static List buildIndices( for (String index : uniqueAliases) { Fields indexFields = indices.computeIfAbsent(index, k -> new Fields()); EsField field = indexFields.flattedMapping.get(fieldName); - if (field == null && invalidFieldsForAliases.get(index) == null) { + if (isMetadataField == false && field == null && invalidFieldsForAliases.get(index) == null) { createField(typeRegistry, fieldName, indexFields, fieldCaps, invalidField, typeCap); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 16ddd8e058894..72de6c99191cc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -413,8 +413,8 @@ public void testMergeObjectIncompatibleTypes() throws Exception { "*", response, (fieldName, types) -> null, - IndexResolver.PRESERVE_PROPERTIES - + IndexResolver.PRESERVE_PROPERTIES, + null ); assertTrue(resolution.isValid()); @@ -442,8 +442,8 @@ public void testMergeObjectUnsupportedTypes() throws Exception { "*", response, (fieldName, types) -> null, - IndexResolver.PRESERVE_PROPERTIES - + IndexResolver.PRESERVE_PROPERTIES, + null ); assertTrue(resolution.isValid()); From 832e48e87faa547a29aafc008f4d4fc2820a3074 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 23 Nov 2023 18:05:51 +0100 Subject: [PATCH 07/35] Do not run tests using minitestcontainer on windows --- .../test/fixtures/minio/MinioTestContainer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index 1d9632bae3a1c..e29fb76e450b7 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -41,7 +41,10 @@ public MinioTestContainer(boolean enabled) { @Override public void start() { - Assume.assumeFalse("https://github.com/elastic/elasticsearch/issues/102532", System.getProperty("os.name").startsWith("windows")); + Assume.assumeFalse( + "https://github.com/elastic/elasticsearch/issues/102532", + System.getProperty("os.name").toLowerCase().startsWith("windows") + ); if (enabled) { super.start(); } From f2a9c150cbb29b3fc9a61a752c8f0826d628920e Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Thu, 23 Nov 2023 20:52:30 +0100 Subject: [PATCH 08/35] Apply docker environment checks for TestContainers (#102553) This is related to #102532. We need to have checks we did in the build logic now in our test container setup. --- .../fixtures/minio/MinioTestContainer.java | 9 +- .../DockerEnvironmentAwareTestContainer.java | 115 ++++++++++++++++++ .../TestContainersThreadFilter.java | 4 + 3 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index e29fb76e450b7..2572f7986dd93 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -8,12 +8,11 @@ package org.elasticsearch.test.fixtures.minio; -import org.junit.Assume; +import org.elasticsearch.test.fixtures.testcontainers.DockerEnvironmentAwareTestContainer; import org.junit.rules.TestRule; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.images.builder.ImageFromDockerfile; -public class MinioTestContainer extends GenericContainer implements TestRule { +public class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule { private static final int servicePort = 9000; private final boolean enabled; @@ -41,10 +40,6 @@ public MinioTestContainer(boolean enabled) { @Override public void start() { - Assume.assumeFalse( - "https://github.com/elastic/elasticsearch/issues/102532", - System.getProperty("os.name").toLowerCase().startsWith("windows") - ); if (enabled) { super.start(); } diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java new file mode 100644 index 0000000000000..588fbf3d6a615 --- /dev/null +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.fixtures.testcontainers; + +import org.elasticsearch.test.fixtures.minio.MinioTestContainer; +import org.junit.Assume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.images.builder.ImageFromDockerfile; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class DockerEnvironmentAwareTestContainer extends GenericContainer { + private static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions"; + + private static final boolean CI = Boolean.parseBoolean(System.getProperty("CI", "false")); + private static final boolean EXCLUDED_OS = CI == false || isExcludedOs(); + + protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class); + + public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfile) { + super(imageFromDockerfile); + } + + @Override + public void start() { + Assume.assumeTrue("Docker is not available", shouldDockerBeAvailable()); + super.start(); + } + + private boolean shouldDockerBeAvailable() { + return CI == false || (EXCLUDED_OS == false); + } + + static String deriveId(Map values) { + return values.get("ID") + "-" + values.get("VERSION_ID"); + } + + private static boolean isExcludedOs() { + if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { + return true; + } + final Path osRelease = Paths.get("/etc/os-release"); + if (Files.exists(osRelease)) { + Map values; + + try { + final List osReleaseLines = Files.readAllLines(osRelease); + values = parseOsRelease(osReleaseLines); + } catch (IOException e) { + throw new RuntimeException("Failed to read /etc/os-release", e); + } + + final String id = deriveId(values); + final boolean excluded = getLinuxExclusionList().contains(id); + + if (excluded) { + LOGGER.warn("Linux OS id [{}] is present in the Docker exclude list. Tasks requiring Docker will be disabled.", id); + } + + return excluded; + } + + return false; + } + + private static List getLinuxExclusionList() { + File exclusionsFile = new File(DOCKER_ON_LINUX_EXCLUSIONS_FILE); + if (exclusionsFile.exists()) { + try { + return Files.readAllLines(exclusionsFile.toPath()) + .stream() + .map(String::trim) + .filter(line -> (line.isEmpty() || line.startsWith("#")) == false) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new RuntimeException("Failed to read " + exclusionsFile.getAbsolutePath(), e); + } + } else { + return Collections.emptyList(); + } + } + + // visible for testing + static Map parseOsRelease(final List osReleaseLines) { + final Map values = new HashMap<>(); + + osReleaseLines.stream().map(String::trim).filter(line -> (line.isEmpty() || line.startsWith("#")) == false).forEach(line -> { + final String[] parts = line.split("=", 2); + final String key = parts[0]; + // remove optional leading and trailing quotes and whitespace + final String value = parts[1].replaceAll("^['\"]?\\s*", "").replaceAll("\\s*['\"]?$", ""); + + values.put(key, value.toLowerCase()); + }); + + return values; + } +} diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java index 6a85d896b87d5..1b0dacbacfd1a 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/TestContainersThreadFilter.java @@ -10,6 +10,10 @@ import com.carrotsearch.randomizedtesting.ThreadFilter; +/** + * test container spawns extra threads, which causes our thread leak + * detection to fail. Filter these threads out since we can't clean them up. + */ public class TestContainersThreadFilter implements ThreadFilter { @Override public boolean reject(Thread t) { From 98e8b10acb57d91e0e1ff8f67d85d8ad1d134b73 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 23 Nov 2023 12:24:05 -0800 Subject: [PATCH 09/35] Replace Block.Ref with Block reference count (#102513) Block.Ref was introduced to help memory management for Evaluator. However, we can replace it with the Block referencing count now. --- .../compute/gen/EvaluatorImplementer.java | 18 +- .../compute/gen/MvEvaluatorImplementer.java | 15 +- .../org/elasticsearch/compute/gen/Types.java | 1 - .../blockhash/PackedValuesBlockHash.java | 2 +- .../org/elasticsearch/compute/data/Block.java | 44 ----- .../operator/ColumnExtractOperator.java | 3 +- .../compute/operator/EvalOperator.java | 14 +- .../compute/operator/FilterOperator.java | 5 +- .../compute/operator/MultivalueDedupe.java | 70 ++++---- .../operator/StringExtractOperator.java | 3 +- .../operator/ColumnExtractOperatorTests.java | 7 +- .../compute/operator/EvalOperatorTests.java | 10 +- .../compute/operator/FilterOperatorTests.java | 4 +- .../operator/MultivalueDedupeTests.java | 4 +- .../operator/StringExtractOperatorTests.java | 12 +- .../comparison/EqualsBoolsEvaluator.java | 14 +- .../comparison/EqualsDoublesEvaluator.java | 14 +- .../comparison/EqualsIntsEvaluator.java | 14 +- .../comparison/EqualsKeywordsEvaluator.java | 14 +- .../comparison/EqualsLongsEvaluator.java | 14 +- .../GreaterThanDoublesEvaluator.java | 14 +- .../comparison/GreaterThanIntsEvaluator.java | 14 +- .../GreaterThanKeywordsEvaluator.java | 14 +- .../comparison/GreaterThanLongsEvaluator.java | 14 +- .../GreaterThanOrEqualDoublesEvaluator.java | 14 +- .../GreaterThanOrEqualIntsEvaluator.java | 14 +- .../GreaterThanOrEqualKeywordsEvaluator.java | 14 +- .../GreaterThanOrEqualLongsEvaluator.java | 14 +- .../comparison/LessThanDoublesEvaluator.java | 14 +- .../comparison/LessThanIntsEvaluator.java | 14 +- .../comparison/LessThanKeywordsEvaluator.java | 14 +- .../comparison/LessThanLongsEvaluator.java | 14 +- .../LessThanOrEqualDoublesEvaluator.java | 14 +- .../LessThanOrEqualIntsEvaluator.java | 14 +- .../LessThanOrEqualKeywordsEvaluator.java | 14 +- .../LessThanOrEqualLongsEvaluator.java | 14 +- .../comparison/NotEqualsBoolsEvaluator.java | 14 +- .../comparison/NotEqualsDoublesEvaluator.java | 14 +- .../comparison/NotEqualsIntsEvaluator.java | 14 +- .../NotEqualsKeywordsEvaluator.java | 14 +- .../comparison/NotEqualsLongsEvaluator.java | 14 +- .../operator/logical/NotEvaluator.java | 9 +- .../operator/regex/RegexMatchEvaluator.java | 9 +- .../conditional/GreatestBooleanEvaluator.java | 14 +- .../GreatestBytesRefEvaluator.java | 14 +- .../conditional/GreatestDoubleEvaluator.java | 14 +- .../conditional/GreatestIntEvaluator.java | 14 +- .../conditional/GreatestLongEvaluator.java | 14 +- .../conditional/LeastBooleanEvaluator.java | 14 +- .../conditional/LeastBytesRefEvaluator.java | 14 +- .../conditional/LeastDoubleEvaluator.java | 14 +- .../scalar/conditional/LeastIntEvaluator.java | 14 +- .../conditional/LeastLongEvaluator.java | 14 +- .../date/DateExtractConstantEvaluator.java | 9 +- .../scalar/date/DateExtractEvaluator.java | 14 +- .../date/DateFormatConstantEvaluator.java | 9 +- .../scalar/date/DateFormatEvaluator.java | 14 +- .../date/DateParseConstantEvaluator.java | 9 +- .../scalar/date/DateParseEvaluator.java | 14 +- .../scalar/date/DateTruncEvaluator.java | 9 +- .../function/scalar/date/NowEvaluator.java | 4 +- .../scalar/ip/CIDRMatchEvaluator.java | 19 +-- .../scalar/math/AbsDoubleEvaluator.java | 9 +- .../function/scalar/math/AbsIntEvaluator.java | 9 +- .../scalar/math/AbsLongEvaluator.java | 9 +- .../function/scalar/math/AcosEvaluator.java | 9 +- .../function/scalar/math/AsinEvaluator.java | 9 +- .../function/scalar/math/Atan2Evaluator.java | 14 +- .../function/scalar/math/AtanEvaluator.java | 9 +- .../scalar/math/CastIntToDoubleEvaluator.java | 9 +- .../scalar/math/CastIntToLongEvaluator.java | 9 +- .../math/CastIntToUnsignedLongEvaluator.java | 9 +- .../math/CastLongToDoubleEvaluator.java | 9 +- .../math/CastLongToUnsignedLongEvaluator.java | 9 +- .../CastUnsignedLongToDoubleEvaluator.java | 9 +- .../scalar/math/CeilDoubleEvaluator.java | 9 +- .../function/scalar/math/CosEvaluator.java | 9 +- .../function/scalar/math/CoshEvaluator.java | 9 +- .../scalar/math/FloorDoubleEvaluator.java | 9 +- .../scalar/math/IsFiniteEvaluator.java | 9 +- .../scalar/math/IsInfiniteEvaluator.java | 9 +- .../function/scalar/math/IsNaNEvaluator.java | 9 +- .../scalar/math/Log10DoubleEvaluator.java | 9 +- .../scalar/math/Log10IntEvaluator.java | 9 +- .../scalar/math/Log10LongEvaluator.java | 9 +- .../math/Log10UnsignedLongEvaluator.java | 9 +- .../function/scalar/math/PowEvaluator.java | 14 +- .../scalar/math/RoundDoubleEvaluator.java | 14 +- .../math/RoundDoubleNoDecimalsEvaluator.java | 9 +- .../scalar/math/RoundIntEvaluator.java | 14 +- .../scalar/math/RoundLongEvaluator.java | 14 +- .../math/RoundUnsignedLongEvaluator.java | 14 +- .../function/scalar/math/SinEvaluator.java | 9 +- .../function/scalar/math/SinhEvaluator.java | 9 +- .../scalar/math/SqrtDoubleEvaluator.java | 9 +- .../scalar/math/SqrtIntEvaluator.java | 9 +- .../scalar/math/SqrtLongEvaluator.java | 9 +- .../math/SqrtUnsignedLongEvaluator.java | 9 +- .../function/scalar/math/TanEvaluator.java | 9 +- .../function/scalar/math/TanhEvaluator.java | 9 +- .../multivalue/MvAvgDoubleEvaluator.java | 72 ++++---- .../scalar/multivalue/MvAvgIntEvaluator.java | 154 +++++++++--------- .../scalar/multivalue/MvAvgLongEvaluator.java | 154 +++++++++--------- .../MvAvgUnsignedLongEvaluator.java | 154 +++++++++--------- .../multivalue/MvMaxBooleanEvaluator.java | 134 +++++++-------- .../multivalue/MvMaxBytesRefEvaluator.java | 150 ++++++++--------- .../multivalue/MvMaxDoubleEvaluator.java | 134 +++++++-------- .../scalar/multivalue/MvMaxIntEvaluator.java | 134 +++++++-------- .../scalar/multivalue/MvMaxLongEvaluator.java | 134 +++++++-------- .../multivalue/MvMedianDoubleEvaluator.java | 72 ++++---- .../multivalue/MvMedianIntEvaluator.java | 134 +++++++-------- .../multivalue/MvMedianLongEvaluator.java | 134 +++++++-------- .../MvMedianUnsignedLongEvaluator.java | 134 +++++++-------- .../multivalue/MvMinBooleanEvaluator.java | 134 +++++++-------- .../multivalue/MvMinBytesRefEvaluator.java | 150 ++++++++--------- .../multivalue/MvMinDoubleEvaluator.java | 134 +++++++-------- .../scalar/multivalue/MvMinIntEvaluator.java | 134 +++++++-------- .../scalar/multivalue/MvMinLongEvaluator.java | 134 +++++++-------- .../multivalue/MvSumDoubleEvaluator.java | 72 ++++---- .../scalar/multivalue/MvSumIntEvaluator.java | 48 +++--- .../scalar/multivalue/MvSumLongEvaluator.java | 48 +++--- .../MvSumUnsignedLongEvaluator.java | 48 +++--- .../scalar/string/ConcatEvaluator.java | 14 +- .../scalar/string/EndsWithEvaluator.java | 14 +- .../scalar/string/LTrimEvaluator.java | 9 +- .../function/scalar/string/LeftEvaluator.java | 14 +- .../scalar/string/LengthEvaluator.java | 9 +- .../scalar/string/RTrimEvaluator.java | 9 +- .../string/ReplaceConstantEvaluator.java | 14 +- .../scalar/string/ReplaceEvaluator.java | 19 +-- .../scalar/string/RightEvaluator.java | 14 +- .../string/SplitSingleByteEvaluator.java | 9 +- .../scalar/string/SplitVariableEvaluator.java | 14 +- .../scalar/string/StartsWithEvaluator.java | 14 +- .../scalar/string/SubstringEvaluator.java | 19 +-- .../string/SubstringNoLengthEvaluator.java | 14 +- .../function/scalar/string/TrimEvaluator.java | 9 +- .../arithmetic/AddDatetimesEvaluator.java | 9 +- .../arithmetic/AddDoublesEvaluator.java | 14 +- .../operator/arithmetic/AddIntsEvaluator.java | 14 +- .../arithmetic/AddLongsEvaluator.java | 14 +- .../arithmetic/AddUnsignedLongsEvaluator.java | 14 +- .../arithmetic/DivDoublesEvaluator.java | 14 +- .../operator/arithmetic/DivIntsEvaluator.java | 14 +- .../arithmetic/DivLongsEvaluator.java | 14 +- .../arithmetic/DivUnsignedLongsEvaluator.java | 14 +- .../arithmetic/ModDoublesEvaluator.java | 14 +- .../operator/arithmetic/ModIntsEvaluator.java | 14 +- .../arithmetic/ModLongsEvaluator.java | 14 +- .../arithmetic/ModUnsignedLongsEvaluator.java | 14 +- .../arithmetic/MulDoublesEvaluator.java | 14 +- .../operator/arithmetic/MulIntsEvaluator.java | 14 +- .../arithmetic/MulLongsEvaluator.java | 14 +- .../arithmetic/MulUnsignedLongsEvaluator.java | 14 +- .../arithmetic/NegDoublesEvaluator.java | 9 +- .../operator/arithmetic/NegIntsEvaluator.java | 9 +- .../arithmetic/NegLongsEvaluator.java | 9 +- .../arithmetic/SubDatetimesEvaluator.java | 9 +- .../arithmetic/SubDoublesEvaluator.java | 14 +- .../operator/arithmetic/SubIntsEvaluator.java | 14 +- .../arithmetic/SubLongsEvaluator.java | 14 +- .../arithmetic/SubUnsignedLongsEvaluator.java | 14 +- .../xpack/esql/evaluator/EvalMapper.java | 50 +++--- .../evaluator/mapper/EvaluatorMapper.java | 6 +- .../operator/comparison/InMapper.java | 16 +- .../function/scalar/conditional/Case.java | 15 +- .../convert/AbstractConvertFunction.java | 11 +- .../AbstractMultivalueFunction.java | 56 ++++--- .../function/scalar/multivalue/MvConcat.java | 9 +- .../function/scalar/multivalue/MvCount.java | 34 ++-- .../function/scalar/nulls/Coalesce.java | 10 +- .../function/AbstractFunctionTestCase.java | 21 ++- .../esql/expression/function/DeepCopy.java | 6 +- .../scalar/conditional/CaseTests.java | 10 +- .../function/scalar/math/RoundTests.java | 8 +- .../function/scalar/nulls/CoalesceTests.java | 6 +- .../function/scalar/string/ConcatTests.java | 6 +- .../function/scalar/string/LeftTests.java | 6 +- .../function/scalar/string/RightTests.java | 6 +- .../function/scalar/string/SplitTests.java | 4 +- .../scalar/string/SubstringTests.java | 4 +- .../AbstractBinaryOperatorTestCase.java | 4 +- .../operator/arithmetic/NegTests.java | 4 +- 183 files changed, 2104 insertions(+), 2511 deletions(-) diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java index e6ea75af38494..1b44e0d274e32 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java @@ -35,7 +35,7 @@ import static org.elasticsearch.compute.gen.Methods.appendMethod; import static org.elasticsearch.compute.gen.Methods.buildFromFactory; import static org.elasticsearch.compute.gen.Methods.getMethod; -import static org.elasticsearch.compute.gen.Types.BLOCK_REF; +import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; @@ -121,7 +121,7 @@ private MethodSpec ctor() { private MethodSpec eval() { MethodSpec.Builder builder = MethodSpec.methodBuilder("eval").addAnnotation(Override.class); - builder.addModifiers(Modifier.PUBLIC).returns(BLOCK_REF).addParameter(PAGE, "page"); + builder.addModifiers(Modifier.PUBLIC).returns(BLOCK).addParameter(PAGE, "page"); processFunction.args.stream().forEach(a -> a.evalToBlock(builder)); String invokeBlockEval = invokeRealEval(true); @@ -132,7 +132,7 @@ private MethodSpec eval() { } private String invokeRealEval(boolean blockStyle) { - StringBuilder builder = new StringBuilder("return Block.Ref.floating(eval(page.getPositionCount()"); + StringBuilder builder = new StringBuilder("return eval(page.getPositionCount()"); String params = processFunction.args.stream() .map(a -> a.paramName(blockStyle)) .filter(a -> a != null) @@ -145,7 +145,6 @@ private String invokeRealEval(boolean blockStyle) { if (processFunction.resultDataType(blockStyle).simpleName().endsWith("Vector")) { builder.append(".asBlock()"); } - builder.append(")"); return builder.toString(); } @@ -346,7 +345,7 @@ private interface ProcessFunctionArg { String factoryInvocation(MethodSpec.Builder factoryMethodBuilder); /** - * Emits code to evaluate this parameter to a Block.Ref or array of Block.Refs + * Emits code to evaluate this parameter to a Block or array of Blocks * and begins a {@code try} block for those refs. Noop if the parameter is {@link Fixed}. */ void evalToBlock(MethodSpec.Builder builder); @@ -440,8 +439,7 @@ public String factoryInvocation(MethodSpec.Builder factoryMethodBuilder) { @Override public void evalToBlock(MethodSpec.Builder builder) { TypeName blockType = blockType(type); - builder.beginControlFlow("try (Block.Ref $LRef = $L.eval(page))", name, name); - builder.addStatement("$T $LBlock = ($T) $LRef.block()", blockType, name, blockType, name); + builder.beginControlFlow("try ($T $LBlock = ($T) $L.eval(page))", blockType, name, blockType, name); } @Override @@ -561,13 +559,11 @@ public String factoryInvocation(MethodSpec.Builder factoryMethodBuilder) { @Override public void evalToBlock(MethodSpec.Builder builder) { TypeName blockType = blockType(componentType); - builder.addStatement("Block.Ref[] $LRefs = new Block.Ref[$L.length]", name, name); - builder.beginControlFlow("try ($T $LRelease = $T.wrap($LRefs))", RELEASABLE, name, RELEASABLES, name); builder.addStatement("$T[] $LBlocks = new $T[$L.length]", blockType, name, blockType, name); + builder.beginControlFlow("try ($T $LRelease = $T.wrap($LBlocks))", RELEASABLE, name, RELEASABLES, name); builder.beginControlFlow("for (int i = 0; i < $LBlocks.length; i++)", name); { - builder.addStatement("$LRefs[i] = $L[i].eval(page)", name, name); - builder.addStatement("$LBlocks[i] = ($T) $LRefs[i].block()", name, blockType, name); + builder.addStatement("$LBlocks[i] = ($T)$L[i].eval(page)", name, blockType, name); } builder.endControlFlow(); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java index 797665d4f62bb..0e794d6fa533f 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/MvEvaluatorImplementer.java @@ -31,7 +31,7 @@ import static org.elasticsearch.compute.gen.Methods.getMethod; import static org.elasticsearch.compute.gen.Types.ABSTRACT_MULTIVALUE_FUNCTION_EVALUATOR; import static org.elasticsearch.compute.gen.Types.ABSTRACT_NULLABLE_MULTIVALUE_FUNCTION_EVALUATOR; -import static org.elasticsearch.compute.gen.Types.BLOCK_REF; +import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; @@ -184,7 +184,7 @@ private MethodSpec evalShell( Consumer body ) { MethodSpec.Builder builder = MethodSpec.methodBuilder(name); - builder.returns(BLOCK_REF).addParameter(BLOCK_REF, "ref"); + builder.returns(BLOCK).addParameter(BLOCK, "fieldVal"); if (override) { builder.addAnnotation(Override.class).addModifiers(Modifier.PUBLIC); } else { @@ -194,9 +194,7 @@ private MethodSpec evalShell( TypeName blockType = blockType(fieldType); preflight.accept(builder); - - builder.beginControlFlow("try (ref)"); - builder.addStatement("$T v = ($T) ref.block()", blockType, blockType); + builder.addStatement("$T v = ($T) fieldVal", blockType, blockType); builder.addStatement("int positionCount = v.getPositionCount()"); TypeName builderType; if (nullable) { @@ -247,8 +245,7 @@ private MethodSpec evalShell( } builder.endControlFlow(); - builder.addStatement("return Block.Ref.floating(builder.build()$L)", nullable ? "" : ".asBlock()"); - builder.endControlFlow(); + builder.addStatement("return builder.build()$L", nullable ? "" : ".asBlock()"); builder.endControlFlow(); return builder.build(); } @@ -259,8 +256,8 @@ private MethodSpec eval(String name, boolean nullable) { if (ascendingFunction == null) { return; } - builder.beginControlFlow("if (ref.block().mvSortedAscending())"); - builder.addStatement("return $L(ref)", name.replace("eval", "evalAscending")); + builder.beginControlFlow("if (fieldVal.mvSortedAscending())"); + builder.addStatement("return $L(fieldVal)", name.replace("eval", "evalAscending")); builder.endControlFlow(); }, builder -> { builder.addStatement("int first = v.getFirstValueIndex(p)"); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 5c316a7c1bdc6..1a09160dae3cd 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -32,7 +32,6 @@ public class Types { static final ClassName PAGE = ClassName.get(DATA_PACKAGE, "Page"); static final ClassName BLOCK = ClassName.get(DATA_PACKAGE, "Block"); static final TypeName BLOCK_ARRAY = ArrayTypeName.of(BLOCK); - static final ClassName BLOCK_REF = ClassName.get(DATA_PACKAGE, "Block", "Ref"); static final ClassName VECTOR = ClassName.get(DATA_PACKAGE, "Vector"); static final ClassName BIG_ARRAYS = ClassName.get("org.elasticsearch.common.util", "BigArrays"); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java index 06b833974a5db..b58c50b79311a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java @@ -99,7 +99,7 @@ class AddWork extends LongLongBlockHash.AbstractAddBlock { AddWork(Page page, GroupingAggregatorFunction.AddInput addInput, int batchSize) { super(blockFactory, emitBatchSize, addInput); for (Group group : groups) { - group.encoder = MultivalueDedupe.batchEncoder(new Block.Ref(page.getBlock(group.spec.channel()), page), batchSize, true); + group.encoder = MultivalueDedupe.batchEncoder(page.getBlock(group.spec.channel()), batchSize, true); } bytes.grow(nullTrackingBytes); this.positionCount = page.getPositionCount(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index ee9889d7d3be8..481a914dc89e9 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -240,49 +239,6 @@ static Block[] buildAll(Block.Builder... builders) { } } - /** - * A reference to a {@link Block}. This is {@link Releasable} and - * {@link Ref#close closing} it will {@link Block#close() release} - * the underlying {@link Block} if it wasn't borrowed from a {@link Page}. - * - * The usual way to use this is: - * 
{@code
-     *   try (Block.Ref ref = eval.eval(page)) {
-     *     return ref.block().doStuff;
-     *   }
-     * }
- * - * The {@code try} block will return the memory used by the block to the - * breaker if it was "free floating", but if it was attached to a {@link Page} - * then it'll do nothing. - * - * @param block the block referenced - * @param containedIn the page containing it or null, if it is "free floating". - */ - // We probably want to remove this; instead, we could incRef and decRef consistently in the EvalOperator. - record Ref(Block block, @Nullable Page containedIn) implements Releasable { - /** - * Create a "free floating" {@link Ref}. - */ - public static Ref floating(Block block) { - return new Ref(block, null); - } - - /** - * Is this block "free floating" or attached to a page? - */ - public boolean floating() { - return containedIn == null; - } - - @Override - public void close() { - if (floating()) { - block.close(); - } - } - } - static List getNamedWriteables() { return List.of( IntBlock.ENTRY, diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java index 58bf9e097bec3..e83a258957104 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ColumnExtractOperator.java @@ -63,8 +63,7 @@ protected Page process(Page page) { blockBuilders[i] = types[i].newBlockBuilder(rowsCount, driverContext.blockFactory()); } - try (Block.Ref ref = inputEvaluator.eval(page)) { - BytesRefBlock input = (BytesRefBlock) ref.block(); + try (BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page)) { BytesRef spare = new BytesRef(); for (int row = 0; row < rowsCount; row++) { if (input.isNull(row)) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index 1612a2b2ece18..2a6a3c9b6210b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -42,12 +42,7 @@ public EvalOperator(BlockFactory blockFactory, ExpressionEvaluator evaluator) { @Override protected Page process(Page page) { - Block.Ref ref = evaluator.eval(page); - Block block = ref.block(); - if (ref.floating() == false) { - // We take ownership of this block, so we need to shallow copy (incRef) to avoid double releases. - block.incRef(); - } + Block block = evaluator.eval(page); return page.appendBlock(block); } @@ -72,8 +67,9 @@ interface Factory { /** * Evaluate the expression. + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ - Block.Ref eval(Page page); + Block eval(Page page); } public static final ExpressionEvaluator.Factory CONSTANT_NULL_FACTORY = new ExpressionEvaluator.Factory() { @@ -90,8 +86,8 @@ public String toString() { public static final ExpressionEvaluator CONSTANT_NULL = new ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { - return Block.Ref.floating(Block.constantNullBlock(page.getPositionCount())); + public Block eval(Page page) { + return Block.constantNullBlock(page.getPositionCount()); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java index d3e86352d1f29..81d788611125b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java @@ -41,13 +41,12 @@ protected Page process(Page page) { int rowCount = 0; int[] positions = new int[page.getPositionCount()]; - try (Block.Ref ref = evaluator.eval(page)) { - if (ref.block().areAllValuesNull()) { + try (BooleanBlock test = (BooleanBlock) evaluator.eval(page)) { + if (test.areAllValuesNull()) { // All results are null which is like false. No values selected. page.releaseBlocks(); return null; } - BooleanBlock test = (BooleanBlock) ref.block(); // TODO we can detect constant true or false from the type // TODO or we could make a new method in bool-valued evaluators that returns a list of numbers for (int p = 0; p < page.getPositionCount(); p++) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MultivalueDedupe.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MultivalueDedupe.java index c0ede17588016..36aa8621062a5 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MultivalueDedupe.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/MultivalueDedupe.java @@ -81,33 +81,26 @@ public static Block dedupeToBlockUsingCopyAndSort(Block block, BlockFactory bloc */ public static ExpressionEvaluator.Factory evaluator(ElementType elementType, ExpressionEvaluator.Factory field) { return switch (elementType) { - case BOOLEAN -> new EvaluatorFactory(field, (blockFactory, ref) -> { - try (ref) { - return Block.Ref.floating(new MultivalueDedupeBoolean((BooleanBlock) ref.block()).dedupeToBlock(blockFactory)); - } - }); - case BYTES_REF -> new EvaluatorFactory(field, (blockFactory, ref) -> { - try (ref) { - return Block.Ref.floating( - new MultivalueDedupeBytesRef((BytesRefBlock) ref.block()).dedupeToBlockAdaptive(blockFactory) - ); - } - }); - case INT -> new EvaluatorFactory(field, (blockFactory, ref) -> { - try (ref) { - return Block.Ref.floating(new MultivalueDedupeInt((IntBlock) ref.block()).dedupeToBlockAdaptive(blockFactory)); - } - }); - case LONG -> new EvaluatorFactory(field, (blockFactory, ref) -> { - try (ref) { - return Block.Ref.floating(new MultivalueDedupeLong((LongBlock) ref.block()).dedupeToBlockAdaptive(blockFactory)); - } - }); - case DOUBLE -> new EvaluatorFactory(field, (blockFactory, ref) -> { - try (ref) { - return Block.Ref.floating(new MultivalueDedupeDouble((DoubleBlock) ref.block()).dedupeToBlockAdaptive(blockFactory)); - } - }); + case BOOLEAN -> new EvaluatorFactory( + field, + (blockFactory, block) -> new MultivalueDedupeBoolean((BooleanBlock) block).dedupeToBlock(blockFactory) + ); + case BYTES_REF -> new EvaluatorFactory( + field, + (blockFactory, block) -> new MultivalueDedupeBytesRef((BytesRefBlock) block).dedupeToBlockAdaptive(blockFactory) + ); + case INT -> new EvaluatorFactory( + field, + (blockFactory, block) -> new MultivalueDedupeInt((IntBlock) block).dedupeToBlockAdaptive(blockFactory) + ); + case LONG -> new EvaluatorFactory( + field, + (blockFactory, block) -> new MultivalueDedupeLong((LongBlock) block).dedupeToBlockAdaptive(blockFactory) + ); + case DOUBLE -> new EvaluatorFactory( + field, + (blockFactory, block) -> new MultivalueDedupeDouble((DoubleBlock) block).dedupeToBlockAdaptive(blockFactory) + ); case NULL -> field; // The page is all nulls and when you dedupe that it's still all nulls default -> throw new IllegalArgumentException("unsupported type [" + elementType + "]"); }; @@ -123,13 +116,12 @@ public record HashResult(IntBlock ords, boolean sawNull) {} * and then encodes the results into a {@link byte[]} which can be used for * things like hashing many fields together. */ - public static BatchEncoder batchEncoder(Block.Ref ref, int batchSize, boolean allowDirectEncoder) { - if (ref.block().areAllValuesNull()) { - return new BatchEncoder.DirectNulls(ref.block()); + public static BatchEncoder batchEncoder(Block block, int batchSize, boolean allowDirectEncoder) { + if (block.areAllValuesNull()) { + return new BatchEncoder.DirectNulls(block); } - var elementType = ref.block().elementType(); - var block = ref.block(); - if (allowDirectEncoder && ref.block().mvDeduplicated()) { + var elementType = block.elementType(); + if (allowDirectEncoder && block.mvDeduplicated()) { return switch (elementType) { case BOOLEAN -> new BatchEncoder.DirectBooleans((BooleanBlock) block); case BYTES_REF -> new BatchEncoder.DirectBytesRefs((BytesRefBlock) block); @@ -150,7 +142,7 @@ public static BatchEncoder batchEncoder(Block.Ref ref, int batchSize, boolean al } } - private record EvaluatorFactory(ExpressionEvaluator.Factory field, BiFunction dedupe) + private record EvaluatorFactory(ExpressionEvaluator.Factory field, BiFunction dedupe) implements ExpressionEvaluator.Factory { @Override @@ -167,17 +159,19 @@ public String toString() { private static class Evaluator implements ExpressionEvaluator { private final BlockFactory blockFactory; private final ExpressionEvaluator field; - private final BiFunction dedupe; + private final BiFunction dedupe; - protected Evaluator(BlockFactory blockFactory, ExpressionEvaluator field, BiFunction dedupe) { + protected Evaluator(BlockFactory blockFactory, ExpressionEvaluator field, BiFunction dedupe) { this.blockFactory = blockFactory; this.field = field; this.dedupe = dedupe; } @Override - public Block.Ref eval(Page page) { - return dedupe.apply(blockFactory, field.eval(page)); + public Block eval(Page page) { + try (Block block = field.eval(page)) { + return dedupe.apply(blockFactory, block); + } } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java index b3b41a542e465..4ffa530bc5d3a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/StringExtractOperator.java @@ -68,8 +68,7 @@ protected Page process(Page page) { blockBuilders[i] = BytesRefBlock.newBlockBuilder(rowsCount, driverContext.blockFactory()); } - try (Block.Ref ref = inputEvaluator.eval(page)) { - BytesRefBlock input = (BytesRefBlock) ref.block(); + try (BytesRefBlock input = (BytesRefBlock) inputEvaluator.eval(page)) { BytesRef spare = new BytesRef(); for (int row = 0; row < rowsCount; row++) { if (input.isNull(row)) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ColumnExtractOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ColumnExtractOperatorTests.java index 6906e5f3adda8..485610f5842bb 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ColumnExtractOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ColumnExtractOperatorTests.java @@ -54,14 +54,15 @@ protected Operator.OperatorFactory simple(BigArrays bigArrays) { new ElementType[] { ElementType.BYTES_REF }, dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { BytesRefBlock input = page.getBlock(0); for (int i = 0; i < input.getPositionCount(); i++) { if (input.getBytesRef(i, new BytesRef()).utf8ToString().startsWith("no_")) { - return Block.Ref.floating(Block.constantNullBlock(input.getPositionCount(), input.blockFactory())); + return Block.constantNullBlock(input.getPositionCount(), input.blockFactory()); } } - return new Block.Ref(input, page); + input.incRef(); + return input; } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java index e7f5db7579869..c755c5eafe08d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/EvalOperatorTests.java @@ -34,14 +34,14 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { record Addition(DriverContext driverContext, int lhs, int rhs) implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { LongVector lhsVector = page.getBlock(0).asVector(); LongVector rhsVector = page.getBlock(1).asVector(); try (LongVector.FixedBuilder result = LongVector.newVectorFixedBuilder(page.getPositionCount(), driverContext.blockFactory())) { for (int p = 0; p < page.getPositionCount(); p++) { result.appendLong(lhsVector.getLong(p) + rhsVector.getLong(p)); } - return Block.Ref.floating(result.build().asBlock()); + return result.build().asBlock(); } } @@ -56,8 +56,10 @@ public void close() {} record LoadFromPage(int channel) implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - return new Block.Ref(page.getBlock(channel), page); + public Block eval(Page page) { + Block block = page.getBlock(channel); + block.incRef(); + return block; } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java index e16f643e1ca4d..d067435ba9aaa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/FilterOperatorTests.java @@ -33,14 +33,14 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { record SameLastDigit(DriverContext context, int lhs, int rhs) implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { LongVector lhsVector = page.getBlock(0).asVector(); LongVector rhsVector = page.getBlock(1).asVector(); BooleanVector.FixedBuilder result = BooleanVector.newVectorFixedBuilder(page.getPositionCount(), context.blockFactory()); for (int p = 0; p < page.getPositionCount(); p++) { result.appendBoolean(lhsVector.getLong(p) % 10 == rhsVector.getLong(p) % 10); } - return Block.Ref.floating(result.build().asBlock()); + return result.build().asBlock(); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MultivalueDedupeTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MultivalueDedupeTests.java index 5b03dc294362f..c228ff9b77fb6 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MultivalueDedupeTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/MultivalueDedupeTests.java @@ -218,7 +218,7 @@ public void testHashWithPreviousValues() { public void testBatchEncodeAll() { int initCapacity = Math.toIntExact(ByteSizeValue.ofKb(10).getBytes()); BasicBlockTests.RandomBlock b = randomBlock(); - var encoder = (BatchEncoder.MVEncoder) MultivalueDedupe.batchEncoder(Block.Ref.floating(b.block()), initCapacity, false); + var encoder = (BatchEncoder.MVEncoder) MultivalueDedupe.batchEncoder(b.block(), initCapacity, false); int valueOffset = 0; for (int p = 0, positionOffset = Integer.MAX_VALUE; p < b.block().getPositionCount(); p++, positionOffset++) { @@ -235,7 +235,7 @@ public void testBatchEncodeAll() { public void testBatchEncoderStartSmall() { assumeFalse("Booleans don't grow in the same way", elementType == ElementType.BOOLEAN); BasicBlockTests.RandomBlock b = randomBlock(); - var encoder = (BatchEncoder.MVEncoder) MultivalueDedupe.batchEncoder(Block.Ref.floating(b.block()), 0, false); + var encoder = (BatchEncoder.MVEncoder) MultivalueDedupe.batchEncoder(b.block(), 0, false); /* * We run can't fit the first non-null position into our 0 bytes. diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java index c55fbeb29a25e..70ef2118fcef0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/StringExtractOperatorTests.java @@ -47,8 +47,10 @@ protected Operator.OperatorFactory simple(BigArrays bigArrays) { new String[] { "test" }, dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { - return new Block.Ref(page.getBlock(0), page); + public Block eval(Page page) { + Block block = page.getBlock(0); + block.incRef(); + return block; } @Override @@ -91,8 +93,10 @@ public void testMultivalueDissectInput() { StringExtractOperator operator = new StringExtractOperator(new String[] { "test" }, new EvalOperator.ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { - return new Block.Ref(page.getBlock(0), page); + public Block eval(Page page) { + Block block = page.getBlock(0); + block.incRef(); + return block; } @Override diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsBoolsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsBoolsEvaluator.java index 5b971a6dc2f11..b5b05d6d395fa 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsBoolsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsBoolsEvaluator.java @@ -33,20 +33,18 @@ public EqualsBoolsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BooleanBlock lhsBlock = (BooleanBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BooleanBlock rhsBlock = (BooleanBlock) rhsRef.block(); + public Block eval(Page page) { + try (BooleanBlock lhsBlock = (BooleanBlock) lhs.eval(page)) { + try (BooleanBlock rhsBlock = (BooleanBlock) rhs.eval(page)) { BooleanVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BooleanVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsDoublesEvaluator.java index d20b493ca2502..b4a0f127c8fa1 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsDoublesEvaluator.java @@ -35,20 +35,18 @@ public EqualsDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsIntsEvaluator.java index e563e46e854f9..8e491e14c6dc3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsIntsEvaluator.java @@ -35,20 +35,18 @@ public EqualsIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsKeywordsEvaluator.java index 43ce60ccd085c..0fe04c80a66f1 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsKeywordsEvaluator.java @@ -36,20 +36,18 @@ public EqualsKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsLongsEvaluator.java index 80dd80145d91b..9e656111ee074 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EqualsLongsEvaluator.java @@ -35,20 +35,18 @@ public EqualsLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanDoublesEvaluator.java index 940baa0c45cc8..64ab3a28df39c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanDoublesEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanIntsEvaluator.java index e82addad8ecbf..7795e9b5f1b4a 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanIntsEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java index 587e379d8f6b5..21ae9b1464d2a 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanKeywordsEvaluator.java @@ -36,20 +36,18 @@ public GreaterThanKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanLongsEvaluator.java index acf5e20afb8fa..b2b559c715126 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanLongsEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java index d7a2d3daae5eb..b73c6e359afd2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualDoublesEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanOrEqualDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java index b0c49c422fa3a..2a77ee8f068e2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualIntsEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanOrEqualIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java index 9c28bd4b5781d..6909a3b761dd3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualKeywordsEvaluator.java @@ -36,20 +36,18 @@ public GreaterThanOrEqualKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java index 390986a332523..71a68b0bb95e6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqualLongsEvaluator.java @@ -35,20 +35,18 @@ public GreaterThanOrEqualLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanDoublesEvaluator.java index 09ead847ba02d..f4990fe06f6cb 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanDoublesEvaluator.java @@ -35,20 +35,18 @@ public LessThanDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanIntsEvaluator.java index 86c4db7a185dc..db623747a5e61 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanIntsEvaluator.java @@ -35,20 +35,18 @@ public LessThanIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanKeywordsEvaluator.java index 9ac90d754315a..be658c3da46ec 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanKeywordsEvaluator.java @@ -36,20 +36,18 @@ public LessThanKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanLongsEvaluator.java index 991c61705fdec..444c715c753cd 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanLongsEvaluator.java @@ -35,20 +35,18 @@ public LessThanLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java index 9d87cd2d2a83d..bffdf4a80649c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualDoublesEvaluator.java @@ -35,20 +35,18 @@ public LessThanOrEqualDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java index fe50109a776b4..dd47aab76f21c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualIntsEvaluator.java @@ -35,20 +35,18 @@ public LessThanOrEqualIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java index d3626a390ca5a..e7a37b3f0fc41 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualKeywordsEvaluator.java @@ -36,20 +36,18 @@ public LessThanOrEqualKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java index b8d215d06e5d1..fec54d164ac3b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqualLongsEvaluator.java @@ -35,20 +35,18 @@ public LessThanOrEqualLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsBoolsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsBoolsEvaluator.java index fd95c698600fa..a8a8882bf54a4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsBoolsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsBoolsEvaluator.java @@ -33,20 +33,18 @@ public NotEqualsBoolsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BooleanBlock lhsBlock = (BooleanBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BooleanBlock rhsBlock = (BooleanBlock) rhsRef.block(); + public Block eval(Page page) { + try (BooleanBlock lhsBlock = (BooleanBlock) lhs.eval(page)) { + try (BooleanBlock rhsBlock = (BooleanBlock) rhs.eval(page)) { BooleanVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BooleanVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsDoublesEvaluator.java index 582cdd4690c4a..cf5d7a5717600 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsDoublesEvaluator.java @@ -35,20 +35,18 @@ public NotEqualsDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsIntsEvaluator.java index d8d794a0d065c..128118d957222 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsIntsEvaluator.java @@ -35,20 +35,18 @@ public NotEqualsIntsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java index 31db6c75c4b55..c2d12fe5840ab 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsKeywordsEvaluator.java @@ -36,20 +36,18 @@ public NotEqualsKeywordsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - BytesRefBlock lhsBlock = (BytesRefBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - BytesRefBlock rhsBlock = (BytesRefBlock) rhsRef.block(); + public Block eval(Page page) { + try (BytesRefBlock lhsBlock = (BytesRefBlock) lhs.eval(page)) { + try (BytesRefBlock rhsBlock = (BytesRefBlock) rhs.eval(page)) { BytesRefVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } BytesRefVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsLongsEvaluator.java index 44f8de9f7bbbf..57e40c2857449 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEqualsLongsEvaluator.java @@ -35,20 +35,18 @@ public NotEqualsLongsEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java index f7785bdc0605e..de3f57d54d8e4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/logical/NotEvaluator.java @@ -29,14 +29,13 @@ public NotEvaluator(EvalOperator.ExpressionEvaluator v, DriverContext driverCont } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - BooleanBlock vBlock = (BooleanBlock) vRef.block(); + public Block eval(Page page) { + try (BooleanBlock vBlock = (BooleanBlock) v.eval(page)) { BooleanVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/regex/RegexMatchEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/regex/RegexMatchEvaluator.java index f3e09aec53a8e..83860fc328543 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/regex/RegexMatchEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/evaluator/predicate/operator/regex/RegexMatchEvaluator.java @@ -37,14 +37,13 @@ public RegexMatchEvaluator(EvalOperator.ExpressionEvaluator input, CharacterRunA } @Override - public Block.Ref eval(Page page) { - try (Block.Ref inputRef = input.eval(page)) { - BytesRefBlock inputBlock = (BytesRefBlock) inputRef.block(); + public Block eval(Page page) { + try (BytesRefBlock inputBlock = (BytesRefBlock) input.eval(page)) { BytesRefVector inputVector = inputBlock.asVector(); if (inputVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), inputBlock)); + return eval(page.getPositionCount(), inputBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), inputVector).asBlock()); + return eval(page.getPositionCount(), inputVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java index bfb80ec0727ce..e335a2cc50add 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBooleanEvaluator.java @@ -32,22 +32,20 @@ public GreatestBooleanEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - BooleanBlock[] valuesBlocks = new BooleanBlock[values.length]; + public Block eval(Page page) { + BooleanBlock[] valuesBlocks = new BooleanBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (BooleanBlock) valuesRefs[i].block(); + valuesBlocks[i] = (BooleanBlock)values[i].eval(page); } BooleanVector[] valuesVectors = new BooleanVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java index acc9e74910878..0919b6c624572 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestBytesRefEvaluator.java @@ -33,22 +33,20 @@ public GreatestBytesRefEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + public Block eval(Page page) { + BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (BytesRefBlock) valuesRefs[i].block(); + valuesBlocks[i] = (BytesRefBlock)values[i].eval(page); } BytesRefVector[] valuesVectors = new BytesRefVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java index 7287e9a7a20ec..acabb839e0543 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestDoubleEvaluator.java @@ -32,22 +32,20 @@ public GreatestDoubleEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - DoubleBlock[] valuesBlocks = new DoubleBlock[values.length]; + public Block eval(Page page) { + DoubleBlock[] valuesBlocks = new DoubleBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (DoubleBlock) valuesRefs[i].block(); + valuesBlocks[i] = (DoubleBlock)values[i].eval(page); } DoubleVector[] valuesVectors = new DoubleVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java index e2afd0e010091..e2fc35c829b5f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestIntEvaluator.java @@ -32,22 +32,20 @@ public GreatestIntEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - IntBlock[] valuesBlocks = new IntBlock[values.length]; + public Block eval(Page page) { + IntBlock[] valuesBlocks = new IntBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (IntBlock) valuesRefs[i].block(); + valuesBlocks[i] = (IntBlock)values[i].eval(page); } IntVector[] valuesVectors = new IntVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java index 5bb0e6185dca2..8f10c02c53c00 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestLongEvaluator.java @@ -32,22 +32,20 @@ public GreatestLongEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - LongBlock[] valuesBlocks = new LongBlock[values.length]; + public Block eval(Page page) { + LongBlock[] valuesBlocks = new LongBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (LongBlock) valuesRefs[i].block(); + valuesBlocks[i] = (LongBlock)values[i].eval(page); } LongVector[] valuesVectors = new LongVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java index 89f91035ae8b0..ce337ae405cba 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBooleanEvaluator.java @@ -32,22 +32,20 @@ public LeastBooleanEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - BooleanBlock[] valuesBlocks = new BooleanBlock[values.length]; + public Block eval(Page page) { + BooleanBlock[] valuesBlocks = new BooleanBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (BooleanBlock) valuesRefs[i].block(); + valuesBlocks[i] = (BooleanBlock)values[i].eval(page); } BooleanVector[] valuesVectors = new BooleanVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java index 5e1d8e13926a5..621d21e13f691 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastBytesRefEvaluator.java @@ -33,22 +33,20 @@ public LeastBytesRefEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + public Block eval(Page page) { + BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (BytesRefBlock) valuesRefs[i].block(); + valuesBlocks[i] = (BytesRefBlock)values[i].eval(page); } BytesRefVector[] valuesVectors = new BytesRefVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java index 0f871cff8f80c..42255e56c6527 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastDoubleEvaluator.java @@ -32,22 +32,20 @@ public LeastDoubleEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - DoubleBlock[] valuesBlocks = new DoubleBlock[values.length]; + public Block eval(Page page) { + DoubleBlock[] valuesBlocks = new DoubleBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (DoubleBlock) valuesRefs[i].block(); + valuesBlocks[i] = (DoubleBlock)values[i].eval(page); } DoubleVector[] valuesVectors = new DoubleVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java index 39a80b09d97e8..ca95f0096166e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastIntEvaluator.java @@ -31,22 +31,20 @@ public LeastIntEvaluator(EvalOperator.ExpressionEvaluator[] values, DriverContex } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - IntBlock[] valuesBlocks = new IntBlock[values.length]; + public Block eval(Page page) { + IntBlock[] valuesBlocks = new IntBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (IntBlock) valuesRefs[i].block(); + valuesBlocks[i] = (IntBlock)values[i].eval(page); } IntVector[] valuesVectors = new IntVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java index bab58f9b42c24..263972b414dd4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastLongEvaluator.java @@ -32,22 +32,20 @@ public LeastLongEvaluator(EvalOperator.ExpressionEvaluator[] values, } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - LongBlock[] valuesBlocks = new LongBlock[values.length]; + public Block eval(Page page) { + LongBlock[] valuesBlocks = new LongBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (LongBlock) valuesRefs[i].block(); + valuesBlocks[i] = (LongBlock)values[i].eval(page); } LongVector[] valuesVectors = new LongVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java index 5de2f3cf371f2..f4109947c7406 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractConstantEvaluator.java @@ -38,14 +38,13 @@ public DateExtractConstantEvaluator(EvalOperator.ExpressionEvaluator value, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valueRef = value.eval(page)) { - LongBlock valueBlock = (LongBlock) valueRef.block(); + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { LongVector valueVector = valueBlock.asVector(); if (valueVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valueBlock)); + return eval(page.getPositionCount(), valueBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valueVector).asBlock()); + return eval(page.getPositionCount(), valueVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java index 7f024bab34c88..37af410e1d49d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractEvaluator.java @@ -46,20 +46,18 @@ public DateExtractEvaluator(Source source, EvalOperator.ExpressionEvaluator valu } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valueRef = value.eval(page)) { - LongBlock valueBlock = (LongBlock) valueRef.block(); - try (Block.Ref chronoFieldRef = chronoField.eval(page)) { - BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoFieldRef.block(); + public Block eval(Page page) { + try (LongBlock valueBlock = (LongBlock) value.eval(page)) { + try (BytesRefBlock chronoFieldBlock = (BytesRefBlock) chronoField.eval(page)) { LongVector valueVector = valueBlock.asVector(); if (valueVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valueBlock, chronoFieldBlock)); + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); } BytesRefVector chronoFieldVector = chronoFieldBlock.asVector(); if (chronoFieldVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valueBlock, chronoFieldBlock)); + return eval(page.getPositionCount(), valueBlock, chronoFieldBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valueVector, chronoFieldVector)); + return eval(page.getPositionCount(), valueVector, chronoFieldVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java index bee94ff3e64af..1ef4b15860dde 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatConstantEvaluator.java @@ -36,14 +36,13 @@ public DateFormatConstantEvaluator(EvalOperator.ExpressionEvaluator val, DateFor } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java index f84645a53ce82..5f8077f908b39 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatEvaluator.java @@ -40,20 +40,18 @@ public DateFormatEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); - try (Block.Ref formatterRef = formatter.eval(page)) { - BytesRefBlock formatterBlock = (BytesRefBlock) formatterRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { + try (BytesRefBlock formatterBlock = (BytesRefBlock) formatter.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, formatterBlock)); + return eval(page.getPositionCount(), valBlock, formatterBlock); } BytesRefVector formatterVector = formatterBlock.asVector(); if (formatterVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, formatterBlock)); + return eval(page.getPositionCount(), valBlock, formatterBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, formatterVector).asBlock()); + return eval(page.getPositionCount(), valVector, formatterVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java index cf0f196321fb6..84e141dcdf448 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseConstantEvaluator.java @@ -42,14 +42,13 @@ public DateParseConstantEvaluator(Source source, EvalOperator.ExpressionEvaluato } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java index 3deff136d8cff..233d2f45c93fa 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java @@ -45,20 +45,18 @@ public DateParseEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); - try (Block.Ref formatterRef = formatter.eval(page)) { - BytesRefBlock formatterBlock = (BytesRefBlock) formatterRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { + try (BytesRefBlock formatterBlock = (BytesRefBlock) formatter.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, formatterBlock)); + return eval(page.getPositionCount(), valBlock, formatterBlock); } BytesRefVector formatterVector = formatterBlock.asVector(); if (formatterVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, formatterBlock)); + return eval(page.getPositionCount(), valBlock, formatterBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, formatterVector)); + return eval(page.getPositionCount(), valVector, formatterVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java index 28e661e3900b1..ff31d753427d4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncEvaluator.java @@ -34,14 +34,13 @@ public DateTruncEvaluator(EvalOperator.ExpressionEvaluator fieldVal, Rounding.Pr } @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldValRef = fieldVal.eval(page)) { - LongBlock fieldValBlock = (LongBlock) fieldValRef.block(); + public Block eval(Page page) { + try (LongBlock fieldValBlock = (LongBlock) fieldVal.eval(page)) { LongVector fieldValVector = fieldValBlock.asVector(); if (fieldValVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), fieldValBlock)); + return eval(page.getPositionCount(), fieldValBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), fieldValVector).asBlock()); + return eval(page.getPositionCount(), fieldValVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java index c257680f20f00..d4c04b724377e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowEvaluator.java @@ -27,8 +27,8 @@ public NowEvaluator(long now, DriverContext driverContext) { } @Override - public Block.Ref eval(Page page) { - return Block.Ref.floating(eval(page.getPositionCount()).asBlock()); + public Block eval(Page page) { + return eval(page.getPositionCount()).asBlock(); } public LongVector eval(int positionCount) { diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java index 22fa3beb8c7e4..c3a347433ff9f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatchEvaluator.java @@ -38,28 +38,25 @@ public CIDRMatchEvaluator(EvalOperator.ExpressionEvaluator ip, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref ipRef = ip.eval(page)) { - BytesRefBlock ipBlock = (BytesRefBlock) ipRef.block(); - Block.Ref[] cidrsRefs = new Block.Ref[cidrs.length]; - try (Releasable cidrsRelease = Releasables.wrap(cidrsRefs)) { - BytesRefBlock[] cidrsBlocks = new BytesRefBlock[cidrs.length]; + public Block eval(Page page) { + try (BytesRefBlock ipBlock = (BytesRefBlock) ip.eval(page)) { + BytesRefBlock[] cidrsBlocks = new BytesRefBlock[cidrs.length]; + try (Releasable cidrsRelease = Releasables.wrap(cidrsBlocks)) { for (int i = 0; i < cidrsBlocks.length; i++) { - cidrsRefs[i] = cidrs[i].eval(page); - cidrsBlocks[i] = (BytesRefBlock) cidrsRefs[i].block(); + cidrsBlocks[i] = (BytesRefBlock)cidrs[i].eval(page); } BytesRefVector ipVector = ipBlock.asVector(); if (ipVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), ipBlock, cidrsBlocks)); + return eval(page.getPositionCount(), ipBlock, cidrsBlocks); } BytesRefVector[] cidrsVectors = new BytesRefVector[cidrs.length]; for (int i = 0; i < cidrsBlocks.length; i++) { cidrsVectors[i] = cidrsBlocks[i].asVector(); if (cidrsVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), ipBlock, cidrsBlocks)); + return eval(page.getPositionCount(), ipBlock, cidrsBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), ipVector, cidrsVectors).asBlock()); + return eval(page.getPositionCount(), ipVector, cidrsVectors).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java index dbadd5adf963c..b7e061e5e684b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsDoubleEvaluator.java @@ -30,14 +30,13 @@ public AbsDoubleEvaluator(EvalOperator.ExpressionEvaluator fieldVal, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldValRef = fieldVal.eval(page)) { - DoubleBlock fieldValBlock = (DoubleBlock) fieldValRef.block(); + public Block eval(Page page) { + try (DoubleBlock fieldValBlock = (DoubleBlock) fieldVal.eval(page)) { DoubleVector fieldValVector = fieldValBlock.asVector(); if (fieldValVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), fieldValBlock)); + return eval(page.getPositionCount(), fieldValBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), fieldValVector).asBlock()); + return eval(page.getPositionCount(), fieldValVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java index 7cc0692e2481a..9894a8ebcdce3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsIntEvaluator.java @@ -29,14 +29,13 @@ public AbsIntEvaluator(EvalOperator.ExpressionEvaluator fieldVal, DriverContext } @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldValRef = fieldVal.eval(page)) { - IntBlock fieldValBlock = (IntBlock) fieldValRef.block(); + public Block eval(Page page) { + try (IntBlock fieldValBlock = (IntBlock) fieldVal.eval(page)) { IntVector fieldValVector = fieldValBlock.asVector(); if (fieldValVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), fieldValBlock)); + return eval(page.getPositionCount(), fieldValBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), fieldValVector).asBlock()); + return eval(page.getPositionCount(), fieldValVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java index 0ffee4cc699f8..ebbb754e28188 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbsLongEvaluator.java @@ -29,14 +29,13 @@ public AbsLongEvaluator(EvalOperator.ExpressionEvaluator fieldVal, DriverContext } @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldValRef = fieldVal.eval(page)) { - LongBlock fieldValBlock = (LongBlock) fieldValRef.block(); + public Block eval(Page page) { + try (LongBlock fieldValBlock = (LongBlock) fieldVal.eval(page)) { LongVector fieldValVector = fieldValBlock.asVector(); if (fieldValVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), fieldValBlock)); + return eval(page.getPositionCount(), fieldValBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), fieldValVector).asBlock()); + return eval(page.getPositionCount(), fieldValVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java index b818cd2f8adfc..ce43cb0d88d09 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosEvaluator.java @@ -36,14 +36,13 @@ public AcosEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java index 923f27dd275b5..2b8168cd2abc7 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinEvaluator.java @@ -36,14 +36,13 @@ public AsinEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java index 902a6eb0cc6a4..ac4d61502be33 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2Evaluator.java @@ -33,20 +33,18 @@ public Atan2Evaluator(EvalOperator.ExpressionEvaluator y, EvalOperator.Expressio } @Override - public Block.Ref eval(Page page) { - try (Block.Ref yRef = y.eval(page)) { - DoubleBlock yBlock = (DoubleBlock) yRef.block(); - try (Block.Ref xRef = x.eval(page)) { - DoubleBlock xBlock = (DoubleBlock) xRef.block(); + public Block eval(Page page) { + try (DoubleBlock yBlock = (DoubleBlock) y.eval(page)) { + try (DoubleBlock xBlock = (DoubleBlock) x.eval(page)) { DoubleVector yVector = yBlock.asVector(); if (yVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), yBlock, xBlock)); + return eval(page.getPositionCount(), yBlock, xBlock); } DoubleVector xVector = xBlock.asVector(); if (xVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), yBlock, xBlock)); + return eval(page.getPositionCount(), yBlock, xBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), yVector, xVector).asBlock()); + return eval(page.getPositionCount(), yVector, xVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java index 14ef9ab1b0e30..2ce4dac48fbf5 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/AtanEvaluator.java @@ -29,14 +29,13 @@ public AtanEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverC } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java index 330b0d73aaf13..5b09822354480 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToDoubleEvaluator.java @@ -31,14 +31,13 @@ public CastIntToDoubleEvaluator(EvalOperator.ExpressionEvaluator v, DriverContex } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - IntBlock vBlock = (IntBlock) vRef.block(); + public Block eval(Page page) { + try (IntBlock vBlock = (IntBlock) v.eval(page)) { IntVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java index 411203a0cf950..0b9f3a5cd2a51 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToLongEvaluator.java @@ -31,14 +31,13 @@ public CastIntToLongEvaluator(EvalOperator.ExpressionEvaluator v, DriverContext } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - IntBlock vBlock = (IntBlock) vRef.block(); + public Block eval(Page page) { + try (IntBlock vBlock = (IntBlock) v.eval(page)) { IntVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java index a79cb9be2d5eb..ee228b79085b7 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastIntToUnsignedLongEvaluator.java @@ -32,14 +32,13 @@ public CastIntToUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - IntBlock vBlock = (IntBlock) vRef.block(); + public Block eval(Page page) { + try (IntBlock vBlock = (IntBlock) v.eval(page)) { IntVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java index 92e85a7214afe..9a70690bf891d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToDoubleEvaluator.java @@ -32,14 +32,13 @@ public CastLongToDoubleEvaluator(EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - LongBlock vBlock = (LongBlock) vRef.block(); + public Block eval(Page page) { + try (LongBlock vBlock = (LongBlock) v.eval(page)) { LongVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java index 3d59ff67c21d1..a258b2eeb7636 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastLongToUnsignedLongEvaluator.java @@ -30,14 +30,13 @@ public CastLongToUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - LongBlock vBlock = (LongBlock) vRef.block(); + public Block eval(Page page) { + try (LongBlock vBlock = (LongBlock) v.eval(page)) { LongVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java index c3b880b9bbdb5..f57d0f4dae34d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CastUnsignedLongToDoubleEvaluator.java @@ -32,14 +32,13 @@ public CastUnsignedLongToDoubleEvaluator(EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - LongBlock vBlock = (LongBlock) vRef.block(); + public Block eval(Page page) { + try (LongBlock vBlock = (LongBlock) v.eval(page)) { LongVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java index 9c63adc45b978..fb25d318f7336 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CeilDoubleEvaluator.java @@ -29,14 +29,13 @@ public CeilDoubleEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext d } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java index 3b62f3f5e9a0e..7fb5063875834 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CosEvaluator.java @@ -29,14 +29,13 @@ public CosEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverCo } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java index b5c461a2ee9b5..ab862a62c6bfe 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/CoshEvaluator.java @@ -36,14 +36,13 @@ public CoshEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java index bf5c32f816892..99ceca3521883 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/FloorDoubleEvaluator.java @@ -29,14 +29,13 @@ public FloorDoubleEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsFiniteEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsFiniteEvaluator.java index 793cf4da31bb2..6ad3ccb6cb287 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsFiniteEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsFiniteEvaluator.java @@ -31,14 +31,13 @@ public IsFiniteEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext dri } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsInfiniteEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsInfiniteEvaluator.java index 2522c4cd6a9fe..00b260467046c 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsInfiniteEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsInfiniteEvaluator.java @@ -31,14 +31,13 @@ public IsInfiniteEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext d } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsNaNEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsNaNEvaluator.java index 955ff8e5cd355..d7639010d9533 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsNaNEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/IsNaNEvaluator.java @@ -31,14 +31,13 @@ public IsNaNEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driver } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java index 66d4a9dfc2e53..6a42dadae78ea 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10DoubleEvaluator.java @@ -36,14 +36,13 @@ public Log10DoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java index c64ec2ceed680..782e35e9a74ab 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10IntEvaluator.java @@ -37,14 +37,13 @@ public Log10IntEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - IntBlock valBlock = (IntBlock) valRef.block(); + public Block eval(Page page) { + try (IntBlock valBlock = (IntBlock) val.eval(page)) { IntVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java index b630b4b079bf7..cfcf56a637f32 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10LongEvaluator.java @@ -37,14 +37,13 @@ public Log10LongEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java index c5d690f3c0ad4..1b092bcbfd8a6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10UnsignedLongEvaluator.java @@ -37,14 +37,13 @@ public Log10UnsignedLongEvaluator(Source source, EvalOperator.ExpressionEvaluato } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java index 12fe99b0ab2be..775cee816be7b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowEvaluator.java @@ -39,20 +39,18 @@ public PowEvaluator(Source source, EvalOperator.ExpressionEvaluator base, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref baseRef = base.eval(page)) { - DoubleBlock baseBlock = (DoubleBlock) baseRef.block(); - try (Block.Ref exponentRef = exponent.eval(page)) { - DoubleBlock exponentBlock = (DoubleBlock) exponentRef.block(); + public Block eval(Page page) { + try (DoubleBlock baseBlock = (DoubleBlock) base.eval(page)) { + try (DoubleBlock exponentBlock = (DoubleBlock) exponent.eval(page)) { DoubleVector baseVector = baseBlock.asVector(); if (baseVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), baseBlock, exponentBlock)); + return eval(page.getPositionCount(), baseBlock, exponentBlock); } DoubleVector exponentVector = exponentBlock.asVector(); if (exponentVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), baseBlock, exponentBlock)); + return eval(page.getPositionCount(), baseBlock, exponentBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), baseVector, exponentVector)); + return eval(page.getPositionCount(), baseVector, exponentVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java index 0be1080fd8710..a658e73a3b44f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleEvaluator.java @@ -35,20 +35,18 @@ public RoundDoubleEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); - try (Block.Ref decimalsRef = decimals.eval(page)) { - LongBlock decimalsBlock = (LongBlock) decimalsRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { + try (LongBlock decimalsBlock = (LongBlock) decimals.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } LongVector decimalsVector = decimalsBlock.asVector(); if (decimalsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, decimalsVector).asBlock()); + return eval(page.getPositionCount(), valVector, decimalsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java index e9b486acf110a..316655de1d7b7 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundDoubleNoDecimalsEvaluator.java @@ -30,14 +30,13 @@ public RoundDoubleNoDecimalsEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java index a6a410aa94938..71ea5938afe48 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundIntEvaluator.java @@ -35,20 +35,18 @@ public RoundIntEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - IntBlock valBlock = (IntBlock) valRef.block(); - try (Block.Ref decimalsRef = decimals.eval(page)) { - LongBlock decimalsBlock = (LongBlock) decimalsRef.block(); + public Block eval(Page page) { + try (IntBlock valBlock = (IntBlock) val.eval(page)) { + try (LongBlock decimalsBlock = (LongBlock) decimals.eval(page)) { IntVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } LongVector decimalsVector = decimalsBlock.asVector(); if (decimalsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, decimalsVector).asBlock()); + return eval(page.getPositionCount(), valVector, decimalsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java index 5053e92935a82..eae45800fdee0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundLongEvaluator.java @@ -33,20 +33,18 @@ public RoundLongEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); - try (Block.Ref decimalsRef = decimals.eval(page)) { - LongBlock decimalsBlock = (LongBlock) decimalsRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { + try (LongBlock decimalsBlock = (LongBlock) decimals.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } LongVector decimalsVector = decimalsBlock.asVector(); if (decimalsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, decimalsVector).asBlock()); + return eval(page.getPositionCount(), valVector, decimalsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java index 56dcca6ae9420..5f8cb5370b213 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundUnsignedLongEvaluator.java @@ -33,20 +33,18 @@ public RoundUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); - try (Block.Ref decimalsRef = decimals.eval(page)) { - LongBlock decimalsBlock = (LongBlock) decimalsRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { + try (LongBlock decimalsBlock = (LongBlock) decimals.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } LongVector decimalsVector = decimalsBlock.asVector(); if (decimalsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock, decimalsBlock)); + return eval(page.getPositionCount(), valBlock, decimalsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector, decimalsVector).asBlock()); + return eval(page.getPositionCount(), valVector, decimalsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java index 4f146c8380d2b..fd2f0b1e3de64 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinEvaluator.java @@ -29,14 +29,13 @@ public SinEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverCo } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java index 96c1c054b1063..342c1b86a873f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SinhEvaluator.java @@ -36,14 +36,13 @@ public SinhEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java index 9674ff97de891..7be90cb5c87c0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtDoubleEvaluator.java @@ -36,14 +36,13 @@ public SqrtDoubleEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java index e7407c8aa43c8..d7a24ebafec97 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtIntEvaluator.java @@ -37,14 +37,13 @@ public SqrtIntEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - IntBlock valBlock = (IntBlock) valRef.block(); + public Block eval(Page page) { + try (IntBlock valBlock = (IntBlock) val.eval(page)) { IntVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java index 4ed28f4fc6735..57055641877c9 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtLongEvaluator.java @@ -37,14 +37,13 @@ public SqrtLongEvaluator(Source source, EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector)); + return eval(page.getPositionCount(), valVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java index b01d1a8eea222..8eddd0293ae86 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtUnsignedLongEvaluator.java @@ -32,14 +32,13 @@ public SqrtUnsignedLongEvaluator(EvalOperator.ExpressionEvaluator val, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - LongBlock valBlock = (LongBlock) valRef.block(); + public Block eval(Page page) { + try (LongBlock valBlock = (LongBlock) val.eval(page)) { LongVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java index 8b499e9be1c35..2ff4ccba94ae0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanEvaluator.java @@ -29,14 +29,13 @@ public TanEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverCo } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java index 605cb38492b5a..05cfc6446cdb6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/math/TanhEvaluator.java @@ -29,14 +29,13 @@ public TanhEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverC } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - DoubleBlock valBlock = (DoubleBlock) valRef.block(); + public Block eval(Page page) { + try (DoubleBlock valBlock = (DoubleBlock) val.eval(page)) { DoubleVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgDoubleEvaluator.java index e533eb6dd5f33..6a9278efd2f6a 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgDoubleEvaluator.java @@ -34,29 +34,27 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); - builder.appendDouble(result); + public Block evalNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvAvg.process(work, value); + } + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build(); } } @@ -64,25 +62,23 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); - builder.appendDouble(result); + public Block evalNotNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvAvg.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgIntEvaluator.java index 5b55cb44072b5..8f2abc5e759b4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgIntEvaluator.java @@ -35,35 +35,33 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - int value = v.getInt(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - int value = v.getInt(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + int value = v.getInt(first); + double result = MvAvg.single(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + int value = v.getInt(i); + MvAvg.process(work, value); } - return Block.Ref.floating(builder.build()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build(); } } @@ -71,31 +69,29 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - int value = v.getInt(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - int value = v.getInt(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNotNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + int value = v.getInt(first); + double result = MvAvg.single(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + int value = v.getInt(i); + MvAvg.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build().asBlock(); } } @@ -103,26 +99,24 @@ public Block.Ref evalNotNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - int value = v.getInt(first); - double result = MvAvg.single(value); - builder.appendDouble(result); + public Block evalSingleValuedNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + int value = v.getInt(first); + double result = MvAvg.single(value); + builder.appendDouble(result); } + return builder.build(); } } @@ -130,22 +124,20 @@ public Block.Ref evalSingleValuedNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNotNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - int value = v.getInt(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - } - return Block.Ref.floating(builder.build().asBlock()); + public Block evalSingleValuedNotNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + int value = v.getInt(first); + double result = MvAvg.single(value); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgLongEvaluator.java index 21f40e02e92ff..b01424846c4a7 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgLongEvaluator.java @@ -35,35 +35,33 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - long value = v.getLong(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + long value = v.getLong(first); + double result = MvAvg.single(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvAvg.process(work, value); } - return Block.Ref.floating(builder.build()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build(); } } @@ -71,31 +69,29 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - long value = v.getLong(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvAvg.process(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + long value = v.getLong(first); + double result = MvAvg.single(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvAvg.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build().asBlock(); } } @@ -103,26 +99,24 @@ public Block.Ref evalNotNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - long value = v.getLong(first); - double result = MvAvg.single(value); - builder.appendDouble(result); + public Block evalSingleValuedNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + long value = v.getLong(first); + double result = MvAvg.single(value); + builder.appendDouble(result); } + return builder.build(); } } @@ -130,22 +124,20 @@ public Block.Ref evalSingleValuedNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - long value = v.getLong(first); - double result = MvAvg.single(value); - builder.appendDouble(result); - } - return Block.Ref.floating(builder.build().asBlock()); + public Block evalSingleValuedNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + long value = v.getLong(first); + double result = MvAvg.single(value); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgUnsignedLongEvaluator.java index 92ebb14d9a29d..41e18cf1424a3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgUnsignedLongEvaluator.java @@ -36,35 +36,33 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - long value = v.getLong(first); - double result = MvAvg.singleUnsignedLong(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvAvg.processUnsignedLong(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + long value = v.getLong(first); + double result = MvAvg.singleUnsignedLong(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvAvg.processUnsignedLong(work, value); } - return Block.Ref.floating(builder.build()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build(); } } @@ -72,31 +70,29 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - if (valueCount == 1) { - long value = v.getLong(first); - double result = MvAvg.singleUnsignedLong(value); - builder.appendDouble(result); - continue; - } - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvAvg.processUnsignedLong(work, value); - } - double result = MvAvg.finish(work, valueCount); + public Block evalNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + if (valueCount == 1) { + long value = v.getLong(first); + double result = MvAvg.singleUnsignedLong(value); builder.appendDouble(result); + continue; + } + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvAvg.processUnsignedLong(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvAvg.finish(work, valueCount); + builder.appendDouble(result); } + return builder.build().asBlock(); } } @@ -104,26 +100,24 @@ public Block.Ref evalNotNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - long value = v.getLong(first); - double result = MvAvg.singleUnsignedLong(value); - builder.appendDouble(result); + public Block evalSingleValuedNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + long value = v.getLong(first); + double result = MvAvg.singleUnsignedLong(value); + builder.appendDouble(result); } + return builder.build(); } } @@ -131,22 +125,20 @@ public Block.Ref evalSingleValuedNullable(Block.Ref ref) { * Evaluate blocks containing only single valued fields. */ @Override - public Block.Ref evalSingleValuedNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - assert valueCount == 1; - int first = v.getFirstValueIndex(p); - long value = v.getLong(first); - double result = MvAvg.singleUnsignedLong(value); - builder.appendDouble(result); - } - return Block.Ref.floating(builder.build().asBlock()); + public Block evalSingleValuedNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + assert valueCount == 1; + int first = v.getFirstValueIndex(p); + long value = v.getLong(first); + double result = MvAvg.singleUnsignedLong(value); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBooleanEvaluator.java index 7aee385dc4c44..46155b23d7512 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBooleanEvaluator.java @@ -34,32 +34,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - boolean value = v.getBoolean(first); - for (int i = first + 1; i < end; i++) { - boolean next = v.getBoolean(i); - value = MvMax.process(value, next); - } - boolean result = value; - builder.appendBoolean(result); + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + boolean value = v.getBoolean(first); + for (int i = first + 1; i < end; i++) { + boolean next = v.getBoolean(i); + value = MvMax.process(value, next); + } + boolean result = value; + builder.appendBoolean(result); } + return builder.build(); } } @@ -67,72 +65,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - boolean value = v.getBoolean(first); - for (int i = first + 1; i < end; i++) { - boolean next = v.getBoolean(i); - value = MvMax.process(value, next); - } - boolean result = value; - builder.appendBoolean(result); + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + boolean value = v.getBoolean(first); + for (int i = first + 1; i < end; i++) { + boolean next = v.getBoolean(i); + value = MvMax.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + boolean result = value; + builder.appendBoolean(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - boolean result = v.getBoolean(first + idx); - builder.appendBoolean(result); + private Block evalAscendingNullable(Block fieldVal) { + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + boolean result = v.getBoolean(first + idx); + builder.appendBoolean(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - boolean result = v.getBoolean(first + idx); - builder.appendBoolean(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + boolean result = v.getBoolean(first + idx); + builder.appendBoolean(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBytesRefEvaluator.java index b88b2641952d7..6f1469e365336 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBytesRefEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxBytesRefEvaluator.java @@ -35,34 +35,32 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - BytesRef value = v.getBytesRef(first, firstScratch); - for (int i = first + 1; i < end; i++) { - BytesRef next = v.getBytesRef(i, nextScratch); - MvMax.process(value, next); - } - BytesRef result = value; - builder.appendBytesRef(result); + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + BytesRef value = v.getBytesRef(first, firstScratch); + for (int i = first + 1; i < end; i++) { + BytesRef next = v.getBytesRef(i, nextScratch); + MvMax.process(value, next); + } + BytesRef result = value; + builder.appendBytesRef(result); } + return builder.build(); } } @@ -70,78 +68,72 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - BytesRef value = v.getBytesRef(first, firstScratch); - for (int i = first + 1; i < end; i++) { - BytesRef next = v.getBytesRef(i, nextScratch); - MvMax.process(value, next); - } - BytesRef result = value; - builder.appendBytesRef(result); + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + BytesRef value = v.getBytesRef(first, firstScratch); + for (int i = first + 1; i < end; i++) { + BytesRef next = v.getBytesRef(i, nextScratch); + MvMax.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + BytesRef result = value; + builder.appendBytesRef(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - BytesRef result = v.getBytesRef(first + idx, firstScratch); - builder.appendBytesRef(result); + private Block evalAscendingNullable(Block fieldVal) { + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + BytesRef result = v.getBytesRef(first + idx, firstScratch); + builder.appendBytesRef(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - BytesRef result = v.getBytesRef(first + idx, firstScratch); - builder.appendBytesRef(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + BytesRef result = v.getBytesRef(first + idx, firstScratch); + builder.appendBytesRef(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxDoubleEvaluator.java index cdbca7f534cf3..34e51c2d6f221 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxDoubleEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - double value = v.getDouble(first); - for (int i = first + 1; i < end; i++) { - double next = v.getDouble(i); - value = MvMax.process(value, next); - } - double result = value; - builder.appendDouble(result); + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + double value = v.getDouble(first); + for (int i = first + 1; i < end; i++) { + double next = v.getDouble(i); + value = MvMax.process(value, next); + } + double result = value; + builder.appendDouble(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - double value = v.getDouble(first); - for (int i = first + 1; i < end; i++) { - double next = v.getDouble(i); - value = MvMax.process(value, next); - } - double result = value; - builder.appendDouble(result); + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + double value = v.getDouble(first); + for (int i = first + 1; i < end; i++) { + double next = v.getDouble(i); + value = MvMax.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + double result = value; + builder.appendDouble(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - double result = v.getDouble(first + idx); - builder.appendDouble(result); + private Block evalAscendingNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + double result = v.getDouble(first + idx); + builder.appendDouble(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - double result = v.getDouble(first + idx); - builder.appendDouble(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + double result = v.getDouble(first + idx); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxIntEvaluator.java index a670bb5e43044..5382f2dff2fd8 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxIntEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - int value = v.getInt(first); - for (int i = first + 1; i < end; i++) { - int next = v.getInt(i); - value = MvMax.process(value, next); - } - int result = value; - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + int value = v.getInt(first); + for (int i = first + 1; i < end; i++) { + int next = v.getInt(i); + value = MvMax.process(value, next); + } + int result = value; + builder.appendInt(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - int value = v.getInt(first); - for (int i = first + 1; i < end; i++) { - int next = v.getInt(i); - value = MvMax.process(value, next); - } - int result = value; - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + int value = v.getInt(first); + for (int i = first + 1; i < end; i++) { + int next = v.getInt(i); + value = MvMax.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + int result = value; + builder.appendInt(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - int result = v.getInt(first + idx); - builder.appendInt(result); + private Block evalAscendingNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + int result = v.getInt(first + idx); + builder.appendInt(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - int result = v.getInt(first + idx); - builder.appendInt(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + int result = v.getInt(first + idx); + builder.appendInt(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxLongEvaluator.java index 5b713429785f0..331d070315ea6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxLongEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvMax.process(value, next); - } - long result = value; - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvMax.process(value, next); + } + long result = value; + builder.appendLong(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvMax.process(value, next); - } - long result = value; - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvMax.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + long result = value; + builder.appendLong(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - long result = v.getLong(first + idx); - builder.appendLong(result); + private Block evalAscendingNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + long result = v.getLong(first + idx); + builder.appendLong(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMax.ascendingIndex(valueCount); - long result = v.getLong(first + idx); - builder.appendLong(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMax.ascendingIndex(valueCount); + long result = v.getLong(first + idx); + builder.appendLong(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianDoubleEvaluator.java index 7049c6a10e0e1..4870712f8f2fb 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianDoubleEvaluator.java @@ -34,29 +34,27 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - MvMedian.Doubles work = new MvMedian.Doubles(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvMedian.process(work, value); - } - double result = MvMedian.finish(work); - builder.appendDouble(result); + public Block evalNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + MvMedian.Doubles work = new MvMedian.Doubles(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvMedian.process(work, value); + } + double result = MvMedian.finish(work); + builder.appendDouble(result); } + return builder.build(); } } @@ -64,25 +62,23 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - MvMedian.Doubles work = new MvMedian.Doubles(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvMedian.process(work, value); - } - double result = MvMedian.finish(work); - builder.appendDouble(result); + public Block evalNotNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + MvMedian.Doubles work = new MvMedian.Doubles(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvMedian.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvMedian.finish(work); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianIntEvaluator.java index 1b9d2bfcdc642..83376cb634a8f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianIntEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - MvMedian.Ints work = new MvMedian.Ints(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - int value = v.getInt(i); - MvMedian.process(work, value); - } - int result = MvMedian.finish(work); - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + MvMedian.Ints work = new MvMedian.Ints(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + int value = v.getInt(i); + MvMedian.process(work, value); + } + int result = MvMedian.finish(work); + builder.appendInt(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - MvMedian.Ints work = new MvMedian.Ints(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - int value = v.getInt(i); - MvMedian.process(work, value); - } - int result = MvMedian.finish(work); - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + MvMedian.Ints work = new MvMedian.Ints(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + int value = v.getInt(i); + MvMedian.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + int result = MvMedian.finish(work); + builder.appendInt(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - MvMedian.Ints work = new MvMedian.Ints(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int result = MvMedian.ascending(v, first, valueCount); - builder.appendInt(result); + private Block evalAscendingNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + MvMedian.Ints work = new MvMedian.Ints(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int result = MvMedian.ascending(v, first, valueCount); + builder.appendInt(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - MvMedian.Ints work = new MvMedian.Ints(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int result = MvMedian.ascending(v, first, valueCount); - builder.appendInt(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + MvMedian.Ints work = new MvMedian.Ints(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int result = MvMedian.ascending(v, first, valueCount); + builder.appendInt(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianLongEvaluator.java index 6a2221366ea38..bf324d4db4f72 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianLongEvaluator.java @@ -34,32 +34,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvMedian.process(work, value); - } - long result = MvMedian.finish(work); - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvMedian.process(work, value); + } + long result = MvMedian.finish(work); + builder.appendLong(result); } + return builder.build(); } } @@ -67,72 +65,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvMedian.process(work, value); - } - long result = MvMedian.finish(work); - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvMedian.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + long result = MvMedian.finish(work); + builder.appendLong(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - long result = MvMedian.ascending(v, first, valueCount); - builder.appendLong(result); + private Block evalAscendingNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + long result = MvMedian.ascending(v, first, valueCount); + builder.appendLong(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - long result = MvMedian.ascending(v, first, valueCount); - builder.appendLong(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + long result = MvMedian.ascending(v, first, valueCount); + builder.appendLong(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianUnsignedLongEvaluator.java index 3864d0253ba5f..3f95ba060f825 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianUnsignedLongEvaluator.java @@ -34,32 +34,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvMedian.processUnsignedLong(work, value); - } - long result = MvMedian.finishUnsignedLong(work); - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvMedian.processUnsignedLong(work, value); + } + long result = MvMedian.finishUnsignedLong(work); + builder.appendLong(result); } + return builder.build(); } } @@ -67,72 +65,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - long value = v.getLong(i); - MvMedian.processUnsignedLong(work, value); - } - long result = MvMedian.finishUnsignedLong(work); - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + long value = v.getLong(i); + MvMedian.processUnsignedLong(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + long result = MvMedian.finishUnsignedLong(work); + builder.appendLong(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - long result = MvMedian.ascendingUnsignedLong(v, first, valueCount); - builder.appendLong(result); + private Block evalAscendingNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + long result = MvMedian.ascendingUnsignedLong(v, first, valueCount); + builder.appendLong(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - MvMedian.Longs work = new MvMedian.Longs(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - long result = MvMedian.ascendingUnsignedLong(v, first, valueCount); - builder.appendLong(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + MvMedian.Longs work = new MvMedian.Longs(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + long result = MvMedian.ascendingUnsignedLong(v, first, valueCount); + builder.appendLong(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBooleanEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBooleanEvaluator.java index 032607382ae6b..a8546837479a8 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBooleanEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBooleanEvaluator.java @@ -34,32 +34,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - boolean value = v.getBoolean(first); - for (int i = first + 1; i < end; i++) { - boolean next = v.getBoolean(i); - value = MvMin.process(value, next); - } - boolean result = value; - builder.appendBoolean(result); + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + boolean value = v.getBoolean(first); + for (int i = first + 1; i < end; i++) { + boolean next = v.getBoolean(i); + value = MvMin.process(value, next); + } + boolean result = value; + builder.appendBoolean(result); } + return builder.build(); } } @@ -67,72 +65,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - boolean value = v.getBoolean(first); - for (int i = first + 1; i < end; i++) { - boolean next = v.getBoolean(i); - value = MvMin.process(value, next); - } - boolean result = value; - builder.appendBoolean(result); + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + boolean value = v.getBoolean(first); + for (int i = first + 1; i < end; i++) { + boolean next = v.getBoolean(i); + value = MvMin.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + boolean result = value; + builder.appendBoolean(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - boolean result = v.getBoolean(first + idx); - builder.appendBoolean(result); + private Block evalAscendingNullable(Block fieldVal) { + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanBlock.Builder builder = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + boolean result = v.getBoolean(first + idx); + builder.appendBoolean(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - BooleanBlock v = (BooleanBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - boolean result = v.getBoolean(first + idx); - builder.appendBoolean(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + BooleanBlock v = (BooleanBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BooleanVector.FixedBuilder builder = driverContext.blockFactory().newBooleanVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + boolean result = v.getBoolean(first + idx); + builder.appendBoolean(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBytesRefEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBytesRefEvaluator.java index 05d792775af1d..f00e7272ae378 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBytesRefEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinBytesRefEvaluator.java @@ -35,34 +35,32 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - BytesRef value = v.getBytesRef(first, firstScratch); - for (int i = first + 1; i < end; i++) { - BytesRef next = v.getBytesRef(i, nextScratch); - MvMin.process(value, next); - } - BytesRef result = value; - builder.appendBytesRef(result); + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + BytesRef value = v.getBytesRef(first, firstScratch); + for (int i = first + 1; i < end; i++) { + BytesRef next = v.getBytesRef(i, nextScratch); + MvMin.process(value, next); + } + BytesRef result = value; + builder.appendBytesRef(result); } + return builder.build(); } } @@ -70,78 +68,72 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - BytesRef value = v.getBytesRef(first, firstScratch); - for (int i = first + 1; i < end; i++) { - BytesRef next = v.getBytesRef(i, nextScratch); - MvMin.process(value, next); - } - BytesRef result = value; - builder.appendBytesRef(result); + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + BytesRef value = v.getBytesRef(first, firstScratch); + for (int i = first + 1; i < end; i++) { + BytesRef next = v.getBytesRef(i, nextScratch); + MvMin.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + BytesRef result = value; + builder.appendBytesRef(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - BytesRef result = v.getBytesRef(first + idx, firstScratch); - builder.appendBytesRef(result); + private Block evalAscendingNullable(Block fieldVal) { + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + BytesRef result = v.getBytesRef(first + idx, firstScratch); + builder.appendBytesRef(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - BytesRefBlock v = (BytesRefBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { - BytesRef firstScratch = new BytesRef(); - BytesRef nextScratch = new BytesRef(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - BytesRef result = v.getBytesRef(first + idx, firstScratch); - builder.appendBytesRef(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + BytesRefBlock v = (BytesRefBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef firstScratch = new BytesRef(); + BytesRef nextScratch = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + BytesRef result = v.getBytesRef(first + idx, firstScratch); + builder.appendBytesRef(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinDoubleEvaluator.java index 9652d869e0726..5cd7ee9039a33 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinDoubleEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - double value = v.getDouble(first); - for (int i = first + 1; i < end; i++) { - double next = v.getDouble(i); - value = MvMin.process(value, next); - } - double result = value; - builder.appendDouble(result); + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + double value = v.getDouble(first); + for (int i = first + 1; i < end; i++) { + double next = v.getDouble(i); + value = MvMin.process(value, next); + } + double result = value; + builder.appendDouble(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - double value = v.getDouble(first); - for (int i = first + 1; i < end; i++) { - double next = v.getDouble(i); - value = MvMin.process(value, next); - } - double result = value; - builder.appendDouble(result); + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + double value = v.getDouble(first); + for (int i = first + 1; i < end; i++) { + double next = v.getDouble(i); + value = MvMin.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + double result = value; + builder.appendDouble(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - double result = v.getDouble(first + idx); - builder.appendDouble(result); + private Block evalAscendingNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + double result = v.getDouble(first + idx); + builder.appendDouble(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - double result = v.getDouble(first + idx); - builder.appendDouble(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + double result = v.getDouble(first + idx); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinIntEvaluator.java index ff52d235cecc5..93b4612f898ad 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinIntEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - int value = v.getInt(first); - for (int i = first + 1; i < end; i++) { - int next = v.getInt(i); - value = MvMin.process(value, next); - } - int result = value; - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + int value = v.getInt(first); + for (int i = first + 1; i < end; i++) { + int next = v.getInt(i); + value = MvMin.process(value, next); + } + int result = value; + builder.appendInt(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - int value = v.getInt(first); - for (int i = first + 1; i < end; i++) { - int next = v.getInt(i); - value = MvMin.process(value, next); - } - int result = value; - builder.appendInt(result); + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + int value = v.getInt(first); + for (int i = first + 1; i < end; i++) { + int next = v.getInt(i); + value = MvMin.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + int result = value; + builder.appendInt(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - int result = v.getInt(first + idx); - builder.appendInt(result); + private Block evalAscendingNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + int result = v.getInt(first + idx); + builder.appendInt(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - int result = v.getInt(first + idx); - builder.appendInt(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntVector.FixedBuilder builder = driverContext.blockFactory().newIntVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + int result = v.getInt(first + idx); + builder.appendInt(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinLongEvaluator.java index b940eead096a4..9c974caecc40d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinLongEvaluator.java @@ -33,32 +33,30 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNullable(ref); + public Block evalNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvMin.process(value, next); - } - long result = value; - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvMin.process(value, next); + } + long result = value; + builder.appendLong(result); } + return builder.build(); } } @@ -66,72 +64,66 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - if (ref.block().mvSortedAscending()) { - return evalAscendingNotNullable(ref); + public Block evalNotNullable(Block fieldVal) { + if (fieldVal.mvSortedAscending()) { + return evalAscendingNotNullable(fieldVal); } - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvMin.process(value, next); - } - long result = value; - builder.appendLong(result); + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvMin.process(value, next); } - return Block.Ref.floating(builder.build().asBlock()); + long result = value; + builder.appendLong(result); } + return builder.build().asBlock(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - long result = v.getLong(first + idx); - builder.appendLong(result); + private Block evalAscendingNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + long result = v.getLong(first + idx); + builder.appendLong(result); } + return builder.build(); } } /** * Evaluate blocks containing at least one multivalued field and all multivalued fields are in ascending order. */ - private Block.Ref evalAscendingNotNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int idx = MvMin.ascendingIndex(valueCount); - long result = v.getLong(first + idx); - builder.appendLong(result); - } - return Block.Ref.floating(builder.build().asBlock()); + private Block evalAscendingNotNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongVector.FixedBuilder builder = driverContext.blockFactory().newLongVectorFixedBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int idx = MvMin.ascendingIndex(valueCount); + long result = v.getLong(first + idx); + builder.appendLong(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumDoubleEvaluator.java index d5d29942491c6..cc54ebad77667 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumDoubleEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumDoubleEvaluator.java @@ -34,29 +34,27 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvSum.process(work, value); - } - double result = MvSum.finish(work); - builder.appendDouble(result); + public Block evalNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleBlock.Builder builder = driverContext.blockFactory().newDoubleBlockBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; } - return Block.Ref.floating(builder.build()); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvSum.process(work, value); + } + double result = MvSum.finish(work); + builder.appendDouble(result); } + return builder.build(); } } @@ -64,25 +62,23 @@ public Block.Ref evalNullable(Block.Ref ref) { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNotNullable(Block.Ref ref) { - try (ref) { - DoubleBlock v = (DoubleBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { - CompensatedSum work = new CompensatedSum(); - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - for (int i = first; i < end; i++) { - double value = v.getDouble(i); - MvSum.process(work, value); - } - double result = MvSum.finish(work); - builder.appendDouble(result); + public Block evalNotNullable(Block fieldVal) { + DoubleBlock v = (DoubleBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (DoubleVector.FixedBuilder builder = driverContext.blockFactory().newDoubleVectorFixedBuilder(positionCount)) { + CompensatedSum work = new CompensatedSum(); + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + for (int i = first; i < end; i++) { + double value = v.getDouble(i); + MvSum.process(work, value); } - return Block.Ref.floating(builder.build().asBlock()); + double result = MvSum.finish(work); + builder.appendDouble(result); } + return builder.build().asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java index 8375a303eb7cb..bd24d4a917e84 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumIntEvaluator.java @@ -39,34 +39,32 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - IntBlock v = (IntBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - try { - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - int value = v.getInt(first); - for (int i = first + 1; i < end; i++) { - int next = v.getInt(i); - value = MvSum.process(value, next); - } - int result = value; - builder.appendInt(result); - } catch (ArithmeticException e) { - warnings.registerException(e); - builder.appendNull(); + public Block evalNullable(Block fieldVal) { + IntBlock v = (IntBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (IntBlock.Builder builder = driverContext.blockFactory().newIntBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + try { + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + int value = v.getInt(first); + for (int i = first + 1; i < end; i++) { + int next = v.getInt(i); + value = MvSum.process(value, next); } + int result = value; + builder.appendInt(result); + } catch (ArithmeticException e) { + warnings.registerException(e); + builder.appendNull(); } - return Block.Ref.floating(builder.build()); } + return builder.build(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java index 625c3756bb5af..823d6fa17bee2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumLongEvaluator.java @@ -39,34 +39,32 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - try { - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvSum.process(value, next); - } - long result = value; - builder.appendLong(result); - } catch (ArithmeticException e) { - warnings.registerException(e); - builder.appendNull(); + public Block evalNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + try { + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvSum.process(value, next); } + long result = value; + builder.appendLong(result); + } catch (ArithmeticException e) { + warnings.registerException(e); + builder.appendNull(); } - return Block.Ref.floating(builder.build()); } + return builder.build(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java index 407b0be54b4a5..8203b46b57a51 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumUnsignedLongEvaluator.java @@ -39,34 +39,32 @@ public String name() { * Evaluate blocks containing at least one multivalued field. */ @Override - public Block.Ref evalNullable(Block.Ref ref) { - try (ref) { - LongBlock v = (LongBlock) ref.block(); - int positionCount = v.getPositionCount(); - try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { - for (int p = 0; p < positionCount; p++) { - int valueCount = v.getValueCount(p); - if (valueCount == 0) { - builder.appendNull(); - continue; - } - try { - int first = v.getFirstValueIndex(p); - int end = first + valueCount; - long value = v.getLong(first); - for (int i = first + 1; i < end; i++) { - long next = v.getLong(i); - value = MvSum.processUnsignedLong(value, next); - } - long result = value; - builder.appendLong(result); - } catch (ArithmeticException e) { - warnings.registerException(e); - builder.appendNull(); + public Block evalNullable(Block fieldVal) { + LongBlock v = (LongBlock) fieldVal; + int positionCount = v.getPositionCount(); + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = v.getValueCount(p); + if (valueCount == 0) { + builder.appendNull(); + continue; + } + try { + int first = v.getFirstValueIndex(p); + int end = first + valueCount; + long value = v.getLong(first); + for (int i = first + 1; i < end; i++) { + long next = v.getLong(i); + value = MvSum.processUnsignedLong(value, next); } + long result = value; + builder.appendLong(result); + } catch (ArithmeticException e) { + warnings.registerException(e); + builder.appendNull(); } - return Block.Ref.floating(builder.build()); } + return builder.build(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java index bb935ac2e91c6..2b3045d29c70f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatEvaluator.java @@ -38,22 +38,20 @@ public ConcatEvaluator(BreakingBytesRefBuilder scratch, EvalOperator.ExpressionE } @Override - public Block.Ref eval(Page page) { - Block.Ref[] valuesRefs = new Block.Ref[values.length]; - try (Releasable valuesRelease = Releasables.wrap(valuesRefs)) { - BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + public Block eval(Page page) { + BytesRefBlock[] valuesBlocks = new BytesRefBlock[values.length]; + try (Releasable valuesRelease = Releasables.wrap(valuesBlocks)) { for (int i = 0; i < valuesBlocks.length; i++) { - valuesRefs[i] = values[i].eval(page); - valuesBlocks[i] = (BytesRefBlock) valuesRefs[i].block(); + valuesBlocks[i] = (BytesRefBlock)values[i].eval(page); } BytesRefVector[] valuesVectors = new BytesRefVector[values.length]; for (int i = 0; i < valuesBlocks.length; i++) { valuesVectors[i] = valuesBlocks[i].asVector(); if (valuesVectors[i] == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valuesBlocks)); + return eval(page.getPositionCount(), valuesBlocks); } } - return Block.Ref.floating(eval(page.getPositionCount(), valuesVectors).asBlock()); + return eval(page.getPositionCount(), valuesVectors).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java index c0ac9f41fd99e..b1cadf96b80cd 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithEvaluator.java @@ -36,20 +36,18 @@ public EndsWithEvaluator(EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref suffixRef = suffix.eval(page)) { - BytesRefBlock suffixBlock = (BytesRefBlock) suffixRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (BytesRefBlock suffixBlock = (BytesRefBlock) suffix.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, suffixBlock)); + return eval(page.getPositionCount(), strBlock, suffixBlock); } BytesRefVector suffixVector = suffixBlock.asVector(); if (suffixVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, suffixBlock)); + return eval(page.getPositionCount(), strBlock, suffixBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, suffixVector).asBlock()); + return eval(page.getPositionCount(), strVector, suffixVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java index 5df651e0013f4..034cf5ddc5727 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrimEvaluator.java @@ -30,14 +30,13 @@ public LTrimEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driver } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java index 35b4efc290bbf..b2cbbc8ed9cf6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftEvaluator.java @@ -45,20 +45,18 @@ public LeftEvaluator(BytesRef out, UnicodeUtil.UTF8CodePoint cp, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref lengthRef = length.eval(page)) { - IntBlock lengthBlock = (IntBlock) lengthRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (IntBlock lengthBlock = (IntBlock) length.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, lengthBlock); } IntVector lengthVector = lengthBlock.asVector(); if (lengthVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, lengthBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, lengthVector).asBlock()); + return eval(page.getPositionCount(), strVector, lengthVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java index 45d4438bec986..2896de06f656d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthEvaluator.java @@ -32,14 +32,13 @@ public LengthEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext drive } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java index 86aa88b64bd8b..a2d1d6bb34384 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrimEvaluator.java @@ -30,14 +30,13 @@ public RTrimEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driver } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java index 929ed6b9b0d45..b3af24d2f6851 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceConstantEvaluator.java @@ -44,20 +44,18 @@ public ReplaceConstantEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref newStrRef = newStr.eval(page)) { - BytesRefBlock newStrBlock = (BytesRefBlock) newStrRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (BytesRefBlock newStrBlock = (BytesRefBlock) newStr.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, newStrBlock)); + return eval(page.getPositionCount(), strBlock, newStrBlock); } BytesRefVector newStrVector = newStrBlock.asVector(); if (newStrVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, newStrBlock)); + return eval(page.getPositionCount(), strBlock, newStrBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, newStrVector)); + return eval(page.getPositionCount(), strVector, newStrVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java index 9d742bd9fb7ef..89013fd3ca2f1 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReplaceEvaluator.java @@ -44,26 +44,23 @@ public ReplaceEvaluator(Source source, EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref regexRef = regex.eval(page)) { - BytesRefBlock regexBlock = (BytesRefBlock) regexRef.block(); - try (Block.Ref newStrRef = newStr.eval(page)) { - BytesRefBlock newStrBlock = (BytesRefBlock) newStrRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (BytesRefBlock regexBlock = (BytesRefBlock) regex.eval(page)) { + try (BytesRefBlock newStrBlock = (BytesRefBlock) newStr.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock)); + return eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock); } BytesRefVector regexVector = regexBlock.asVector(); if (regexVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock)); + return eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock); } BytesRefVector newStrVector = newStrBlock.asVector(); if (newStrVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock)); + return eval(page.getPositionCount(), strBlock, regexBlock, newStrBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, regexVector, newStrVector)); + return eval(page.getPositionCount(), strVector, regexVector, newStrVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java index a4e6083a26eb0..1e3094ed8d5d3 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightEvaluator.java @@ -45,20 +45,18 @@ public RightEvaluator(BytesRef out, UnicodeUtil.UTF8CodePoint cp, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref lengthRef = length.eval(page)) { - IntBlock lengthBlock = (IntBlock) lengthRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (IntBlock lengthBlock = (IntBlock) length.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, lengthBlock); } IntVector lengthVector = lengthBlock.asVector(); if (lengthVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, lengthBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, lengthVector).asBlock()); + return eval(page.getPositionCount(), strVector, lengthVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java index 257fa867c5f2c..bb5b3569934c0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitSingleByteEvaluator.java @@ -38,14 +38,13 @@ public SplitSingleByteEvaluator(EvalOperator.ExpressionEvaluator str, byte delim } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock)); + return eval(page.getPositionCount(), strBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector)); + return eval(page.getPositionCount(), strVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java index 4996cae1da7d3..d80d8d65c3606 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitVariableEvaluator.java @@ -38,20 +38,18 @@ public SplitVariableEvaluator(EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref delimRef = delim.eval(page)) { - BytesRefBlock delimBlock = (BytesRefBlock) delimRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (BytesRefBlock delimBlock = (BytesRefBlock) delim.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, delimBlock)); + return eval(page.getPositionCount(), strBlock, delimBlock); } BytesRefVector delimVector = delimBlock.asVector(); if (delimVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, delimBlock)); + return eval(page.getPositionCount(), strBlock, delimBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, delimVector)); + return eval(page.getPositionCount(), strVector, delimVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java index 7579b2aa2696d..564dd1b7760be 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithEvaluator.java @@ -36,20 +36,18 @@ public StartsWithEvaluator(EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref prefixRef = prefix.eval(page)) { - BytesRefBlock prefixBlock = (BytesRefBlock) prefixRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (BytesRefBlock prefixBlock = (BytesRefBlock) prefix.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, prefixBlock)); + return eval(page.getPositionCount(), strBlock, prefixBlock); } BytesRefVector prefixVector = prefixBlock.asVector(); if (prefixVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, prefixBlock)); + return eval(page.getPositionCount(), strBlock, prefixBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, prefixVector).asBlock()); + return eval(page.getPositionCount(), strVector, prefixVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java index 8b9b02ed7ec08..f0b4b0363ebc5 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringEvaluator.java @@ -40,26 +40,23 @@ public SubstringEvaluator(EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref startRef = start.eval(page)) { - IntBlock startBlock = (IntBlock) startRef.block(); - try (Block.Ref lengthRef = length.eval(page)) { - IntBlock lengthBlock = (IntBlock) lengthRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (IntBlock startBlock = (IntBlock) start.eval(page)) { + try (IntBlock lengthBlock = (IntBlock) length.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, startBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, startBlock, lengthBlock); } IntVector startVector = startBlock.asVector(); if (startVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, startBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, startBlock, lengthBlock); } IntVector lengthVector = lengthBlock.asVector(); if (lengthVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, startBlock, lengthBlock)); + return eval(page.getPositionCount(), strBlock, startBlock, lengthBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, startVector, lengthVector).asBlock()); + return eval(page.getPositionCount(), strVector, startVector, lengthVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java index 27d2465875dce..a410df8bbdc69 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringNoLengthEvaluator.java @@ -36,20 +36,18 @@ public SubstringNoLengthEvaluator(EvalOperator.ExpressionEvaluator str, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref strRef = str.eval(page)) { - BytesRefBlock strBlock = (BytesRefBlock) strRef.block(); - try (Block.Ref startRef = start.eval(page)) { - IntBlock startBlock = (IntBlock) startRef.block(); + public Block eval(Page page) { + try (BytesRefBlock strBlock = (BytesRefBlock) str.eval(page)) { + try (IntBlock startBlock = (IntBlock) start.eval(page)) { BytesRefVector strVector = strBlock.asVector(); if (strVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, startBlock)); + return eval(page.getPositionCount(), strBlock, startBlock); } IntVector startVector = startBlock.asVector(); if (startVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), strBlock, startBlock)); + return eval(page.getPositionCount(), strBlock, startBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), strVector, startVector).asBlock()); + return eval(page.getPositionCount(), strVector, startVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java index f9a6e19e5293a..38b42070e96a6 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/TrimEvaluator.java @@ -30,14 +30,13 @@ public TrimEvaluator(EvalOperator.ExpressionEvaluator val, DriverContext driverC } @Override - public Block.Ref eval(Page page) { - try (Block.Ref valRef = val.eval(page)) { - BytesRefBlock valBlock = (BytesRefBlock) valRef.block(); + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { BytesRefVector valVector = valBlock.asVector(); if (valVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), valBlock)); + return eval(page.getPositionCount(), valBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), valVector).asBlock()); + return eval(page.getPositionCount(), valVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java index d24c4700a2dde..f484a77c30ed2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDatetimesEvaluator.java @@ -41,14 +41,13 @@ public AddDatetimesEvaluator(Source source, EvalOperator.ExpressionEvaluator dat } @Override - public Block.Ref eval(Page page) { - try (Block.Ref datetimeRef = datetime.eval(page)) { - LongBlock datetimeBlock = (LongBlock) datetimeRef.block(); + public Block eval(Page page) { + try (LongBlock datetimeBlock = (LongBlock) datetime.eval(page)) { LongVector datetimeVector = datetimeBlock.asVector(); if (datetimeVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), datetimeBlock)); + return eval(page.getPositionCount(), datetimeBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), datetimeVector)); + return eval(page.getPositionCount(), datetimeVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java index 1029295315ec3..1e9cf33ae39e2 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddDoublesEvaluator.java @@ -33,20 +33,18 @@ public AddDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java index 276a8d9c477b6..e7a3b57479b99 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddIntsEvaluator.java @@ -39,20 +39,18 @@ public AddIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java index fc39808f7c2f1..d2e029ff276b8 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddLongsEvaluator.java @@ -39,20 +39,18 @@ public AddLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java index 0d8bae4fdb2d2..54b7b8df88178 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddUnsignedLongsEvaluator.java @@ -39,20 +39,18 @@ public AddUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java index 4a7caa5a488d9..f906d83b19ce4 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivDoublesEvaluator.java @@ -33,20 +33,18 @@ public DivDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java index 4bb8c64959635..53cfbd8540e33 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivIntsEvaluator.java @@ -39,20 +39,18 @@ public DivIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java index 133c6c9bc5503..31f62d3d729c5 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivLongsEvaluator.java @@ -39,20 +39,18 @@ public DivLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java index af160626c730d..104208de1e13f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivUnsignedLongsEvaluator.java @@ -39,20 +39,18 @@ public DivUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java index f194c1f1a1fbb..6d4f2d08b0b6e 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModDoublesEvaluator.java @@ -33,20 +33,18 @@ public ModDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java index 2cac7f276c1ce..1f6979179627d 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModIntsEvaluator.java @@ -39,20 +39,18 @@ public ModIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java index 37e2d31928ab9..3bc252c5cd059 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModLongsEvaluator.java @@ -39,20 +39,18 @@ public ModLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java index 97e386e1921af..a18a99c7e220f 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModUnsignedLongsEvaluator.java @@ -39,20 +39,18 @@ public ModUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java index d9f3ae0729c46..4ab6801f66b92 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulDoublesEvaluator.java @@ -33,20 +33,18 @@ public MulDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java index b100f23023a43..9926668c5e505 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulIntsEvaluator.java @@ -39,20 +39,18 @@ public MulIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java index 33fe21db3b478..8be74005e1940 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulLongsEvaluator.java @@ -39,20 +39,18 @@ public MulLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java index 4f6df9f79786e..4ba489dc65f06 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulUnsignedLongsEvaluator.java @@ -39,20 +39,18 @@ public MulUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java index 4705bd343547c..330b3afa3df19 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegDoublesEvaluator.java @@ -29,14 +29,13 @@ public NegDoublesEvaluator(EvalOperator.ExpressionEvaluator v, DriverContext dri } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - DoubleBlock vBlock = (DoubleBlock) vRef.block(); + public Block eval(Page page) { + try (DoubleBlock vBlock = (DoubleBlock) v.eval(page)) { DoubleVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector).asBlock()); + return eval(page.getPositionCount(), vVector).asBlock(); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java index 8783f91eed95a..9691099b03924 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegIntsEvaluator.java @@ -36,14 +36,13 @@ public NegIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - IntBlock vBlock = (IntBlock) vRef.block(); + public Block eval(Page page) { + try (IntBlock vBlock = (IntBlock) v.eval(page)) { IntVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector)); + return eval(page.getPositionCount(), vVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java index 9f4dc7d8c9ed5..4d8ee14d4569b 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegLongsEvaluator.java @@ -36,14 +36,13 @@ public NegLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator v, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref vRef = v.eval(page)) { - LongBlock vBlock = (LongBlock) vRef.block(); + public Block eval(Page page) { + try (LongBlock vBlock = (LongBlock) v.eval(page)) { LongVector vVector = vBlock.asVector(); if (vVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), vBlock)); + return eval(page.getPositionCount(), vBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), vVector)); + return eval(page.getPositionCount(), vVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java index ec8125ebde0ea..de81736c42abf 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDatetimesEvaluator.java @@ -41,14 +41,13 @@ public SubDatetimesEvaluator(Source source, EvalOperator.ExpressionEvaluator dat } @Override - public Block.Ref eval(Page page) { - try (Block.Ref datetimeRef = datetime.eval(page)) { - LongBlock datetimeBlock = (LongBlock) datetimeRef.block(); + public Block eval(Page page) { + try (LongBlock datetimeBlock = (LongBlock) datetime.eval(page)) { LongVector datetimeVector = datetimeBlock.asVector(); if (datetimeVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), datetimeBlock)); + return eval(page.getPositionCount(), datetimeBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), datetimeVector)); + return eval(page.getPositionCount(), datetimeVector); } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java index 76c06dd137a18..6609d6cfbb4ae 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubDoublesEvaluator.java @@ -33,20 +33,18 @@ public SubDoublesEvaluator(EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - DoubleBlock lhsBlock = (DoubleBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - DoubleBlock rhsBlock = (DoubleBlock) rhsRef.block(); + public Block eval(Page page) { + try (DoubleBlock lhsBlock = (DoubleBlock) lhs.eval(page)) { + try (DoubleBlock rhsBlock = (DoubleBlock) rhs.eval(page)) { DoubleVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } DoubleVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector).asBlock()); + return eval(page.getPositionCount(), lhsVector, rhsVector).asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java index 842a8a5b42ea1..4013cdd240dd0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubIntsEvaluator.java @@ -39,20 +39,18 @@ public SubIntsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - IntBlock lhsBlock = (IntBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - IntBlock rhsBlock = (IntBlock) rhsRef.block(); + public Block eval(Page page) { + try (IntBlock lhsBlock = (IntBlock) lhs.eval(page)) { + try (IntBlock rhsBlock = (IntBlock) rhs.eval(page)) { IntVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } IntVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java index 79e2879f2861a..7528750da15f8 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubLongsEvaluator.java @@ -39,20 +39,18 @@ public SubLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator lhs, } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java index c71855990c257..6c2a31db0a6f0 100644 --- a/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java +++ b/x-pack/plugin/esql/src/main/java/generated/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubUnsignedLongsEvaluator.java @@ -39,20 +39,18 @@ public SubUnsignedLongsEvaluator(Source source, EvalOperator.ExpressionEvaluator } @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhsRef = lhs.eval(page)) { - LongBlock lhsBlock = (LongBlock) lhsRef.block(); - try (Block.Ref rhsRef = rhs.eval(page)) { - LongBlock rhsBlock = (LongBlock) rhsRef.block(); + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + try (LongBlock rhsBlock = (LongBlock) rhs.eval(page)) { LongVector lhsVector = lhsBlock.asVector(); if (lhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } LongVector rhsVector = rhsBlock.asVector(); if (rhsVector == null) { - return Block.Ref.floating(eval(page.getPositionCount(), lhsBlock, rhsBlock)); + return eval(page.getPositionCount(), lhsBlock, rhsBlock); } - return Block.Ref.floating(eval(page.getPositionCount(), lhsVector, rhsVector)); + return eval(page.getPositionCount(), lhsVector, rhsVector); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index b9ef94d587556..132df0d3a5afd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -86,14 +86,14 @@ record BooleanLogicExpressionEvaluator(BinaryLogic bl, ExpressionEvaluator leftE implements ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - try (Block.Ref lhs = leftEval.eval(page); Block.Ref rhs = rightEval.eval(page)) { - Vector lhsVector = lhs.block().asVector(); - Vector rhsVector = rhs.block().asVector(); + public Block eval(Page page) { + try (Block lhs = leftEval.eval(page); Block rhs = rightEval.eval(page)) { + Vector lhsVector = lhs.asVector(); + Vector rhsVector = rhs.asVector(); if (lhsVector != null && rhsVector != null) { - return Block.Ref.floating(eval((BooleanVector) lhsVector, (BooleanVector) rhsVector)); + return eval((BooleanVector) lhsVector, (BooleanVector) rhsVector); } - return Block.Ref.floating(eval(lhs.block(), rhs.block())); + return eval(lhs, rhs); } } @@ -165,8 +165,10 @@ static class Attributes extends ExpressionMapper { public ExpressionEvaluator.Factory map(Attribute attr, Layout layout) { record Attribute(int channel) implements ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - return new Block.Ref(page.getBlock(channel), page); + public Block eval(Page page) { + Block block = page.getBlock(channel); + block.incRef(); + return block; } @Override @@ -193,8 +195,8 @@ static class Literals extends ExpressionMapper { public ExpressionEvaluator.Factory map(Literal lit, Layout layout) { record LiteralsEvaluator(DriverContext context, Literal lit) implements ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - return Block.Ref.floating(block(lit, context.blockFactory(), page.getPositionCount())); + public Block eval(Page page) { + return block(lit, context.blockFactory(), page.getPositionCount()); } @Override @@ -261,12 +263,10 @@ public String toString() { record IsNullEvaluator(DriverContext driverContext, EvalOperator.ExpressionEvaluator field) implements ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldBlock = field.eval(page)) { - if (fieldBlock.block().asVector() != null) { - return Block.Ref.floating( - BooleanBlock.newConstantBlockWith(false, page.getPositionCount(), driverContext.blockFactory()) - ); + public Block eval(Page page) { + try (Block fieldBlock = field.eval(page)) { + if (fieldBlock.asVector() != null) { + return BooleanBlock.newConstantBlockWith(false, page.getPositionCount(), driverContext.blockFactory()); } try ( BooleanVector.FixedBuilder builder = BooleanVector.newVectorFixedBuilder( @@ -275,9 +275,9 @@ public Block.Ref eval(Page page) { ) ) { for (int p = 0; p < page.getPositionCount(); p++) { - builder.appendBoolean(fieldBlock.block().isNull(p)); + builder.appendBoolean(fieldBlock.isNull(p)); } - return Block.Ref.floating(builder.build().asBlock()); + return builder.build().asBlock(); } } } @@ -317,12 +317,10 @@ record IsNotNullEvaluator(DriverContext driverContext, EvalOperator.ExpressionEv implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { - try (Block.Ref fieldBlock = field.eval(page)) { - if (fieldBlock.block().asVector() != null) { - return Block.Ref.floating( - BooleanBlock.newConstantBlockWith(true, page.getPositionCount(), driverContext.blockFactory()) - ); + public Block eval(Page page) { + try (Block fieldBlock = field.eval(page)) { + if (fieldBlock.asVector() != null) { + return BooleanBlock.newConstantBlockWith(true, page.getPositionCount(), driverContext.blockFactory()); } try ( BooleanVector.FixedBuilder builder = BooleanVector.newVectorFixedBuilder( @@ -331,9 +329,9 @@ public Block.Ref eval(Page page) { ) ) { for (int p = 0; p < page.getPositionCount(); p++) { - builder.appendBoolean(fieldBlock.block().isNull(p) == false); + builder.appendBoolean(fieldBlock.isNull(p) == false); } - return Block.Ref.floating(builder.build().asBlock()); + return builder.build().asBlock(); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java index 90699bfaf83df..3ab555799ee34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java @@ -37,8 +37,8 @@ public interface EvaluatorMapper { default Object fold() { return toJavaObject(toEvaluator(e -> driverContext -> new ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { - return Block.Ref.floating(fromArrayRow(driverContext.blockFactory(), e.fold())[0]); + public Block eval(Page page) { + return fromArrayRow(driverContext.blockFactory(), e.fold())[0]; } @Override @@ -49,6 +49,6 @@ public void close() {} // TODO maybe this should have a small fixed limit? new BlockFactory(new NoopCircuitBreaker(CircuitBreaker.REQUEST), BigArrays.NON_RECYCLING_INSTANCE) ) - ).eval(new Page(1)).block(), 0); + ).eval(new Page(1)), 0); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java index 2be60292dff6f..6ef37abf5a9b4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/InMapper.java @@ -13,7 +13,6 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BooleanVector; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.Releasables; @@ -47,7 +46,7 @@ public ExpressionEvaluator.Factory map(In in, Layout layout) { record InExpressionEvaluator(List listEvaluators) implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { int positionCount = page.getPositionCount(); boolean[] values = new boolean[positionCount]; BitSet nulls = new BitSet(positionCount); // at least one evaluation resulted in NULL on a row @@ -55,22 +54,21 @@ public Block.Ref eval(Page page) { for (int i = 0; i < listEvaluators().size(); i++) { var evaluator = listEvaluators.get(i); - try (Block.Ref ref = evaluator.eval(page)) { - - Vector vector = ref.block().asVector(); + try (BooleanBlock block = (BooleanBlock) evaluator.eval(page)) { + BooleanVector vector = block.asVector(); if (vector != null) { - updateValues((BooleanVector) vector, values); + updateValues(vector, values); } else { - if (ref.block().areAllValuesNull()) { + if (block.areAllValuesNull()) { nullInValues = true; } else { - updateValues((BooleanBlock) ref.block(), values, nulls); + updateValues(block, values, nulls); } } } } - return Block.Ref.floating(evalWithNulls(values, nulls, nullInValues)); + return evalWithNulls(values, nulls, nullInValues); } private static void updateValues(BooleanVector vector, boolean[] values) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index a3d08e4cb6306..caef1fe0de627 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -212,7 +212,7 @@ private record CaseEvaluator( EvalOperator.ExpressionEvaluator elseVal ) implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { /* * We have to evaluate lazily so any errors or warnings that would be * produced by the right hand side are avoided. And so if anything @@ -231,26 +231,25 @@ public Block.Ref eval(Page page) { ); try (Releasable ignored = limited::releaseBlocks) { for (ConditionEvaluator condition : conditions) { - try (Block.Ref conditionRef = condition.condition.eval(limited)) { - BooleanBlock b = (BooleanBlock) conditionRef.block(); + try (BooleanBlock b = (BooleanBlock) condition.condition.eval(limited)) { if (b.isNull(0)) { continue; } if (false == b.getBoolean(b.getFirstValueIndex(0))) { continue; } - try (Block.Ref valueRef = condition.value.eval(limited)) { - result.copyFrom(valueRef.block(), 0, 1); + try (Block values = condition.value.eval(limited)) { + result.copyFrom(values, 0, 1); continue position; } } } - try (Block.Ref elseRef = elseVal.eval(limited)) { - result.copyFrom(elseRef.block(), 0, 1); + try (Block values = elseVal.eval(limited)) { + result.copyFrom(values, 0, 1); } } } - return Block.Ref.floating(result.build()); + return result.build(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index c717fafa877a9..0da3717f758bf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -100,18 +100,21 @@ protected AbstractEvaluator(DriverContext driverContext, EvalOperator.Expression /** * Called when evaluating a {@link Block} that contains null values. + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ protected abstract Block evalBlock(Block b); /** * Called when evaluating a {@link Block} that does not contain null values. + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ protected abstract Block evalVector(Vector v); - public Block.Ref eval(Page page) { - try (Block.Ref ref = fieldEvaluator.eval(page)) { - Vector vector = ref.block().asVector(); - return Block.Ref.floating(vector == null ? evalBlock(ref.block()) : evalVector(vector)); + @Override + public final Block eval(Page page) { + try (Block block = fieldEvaluator.eval(page)) { + Vector vector = block.asVector(); + return vector == null ? evalBlock(block) : evalVector(vector); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java index 196137336bee5..e3bb8212aebab 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java @@ -64,30 +64,37 @@ protected AbstractEvaluator(EvalOperator.ExpressionEvaluator field) { * that it's producing an "array vector" because it only ever emits single * valued fields and no null values. Building an array vector directly is * generally faster than building it via a {@link Block.Builder}. + * + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ - protected abstract Block.Ref evalNotNullable(Block.Ref fieldVal); + protected abstract Block evalNotNullable(Block fieldVal); /** - * Called to evaluate single valued fields when the target block does not - * have null values. + * Called to evaluate single valued fields when the target block does not have null values. + * + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ - protected Block.Ref evalSingleValuedNotNullable(Block.Ref fieldRef) { + protected Block evalSingleValuedNotNullable(Block fieldRef) { + fieldRef.incRef(); return fieldRef; } @Override - public final Block.Ref eval(Page page) { - Block.Ref ref = field.eval(page); - if (ref.block().mayHaveMultivaluedFields() == false) { - if (ref.block().mayHaveNulls()) { - return evalSingleValuedNullable(ref); + public final Block eval(Page page) { + try (Block block = field.eval(page)) { + if (block.mayHaveMultivaluedFields()) { + if (block.mayHaveNulls()) { + return evalNullable(block); + } else { + return evalNotNullable(block); + } + } + if (block.mayHaveNulls()) { + return evalSingleValuedNullable(block); + } else { + return evalSingleValuedNotNullable(block); } - return evalSingleValuedNotNullable(ref); - } - if (ref.block().mayHaveNulls()) { - return evalNullable(ref); } - return evalNotNullable(ref); } } @@ -105,21 +112,28 @@ protected AbstractNullableEvaluator(EvalOperator.ExpressionEvaluator field) { /** * Called when evaluating a {@link Block} that contains null values. + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ - protected abstract Block.Ref evalNullable(Block.Ref fieldVal); + protected abstract Block evalNullable(Block fieldVal); /** - * Called to evaluate single valued fields when the target block has null - * values. + * Called to evaluate single valued fields when the target block has null values. + * @return the returned Block has its own reference and the caller is responsible for releasing it. */ - protected Block.Ref evalSingleValuedNullable(Block.Ref fieldRef) { + protected Block evalSingleValuedNullable(Block fieldRef) { + fieldRef.incRef(); return fieldRef; } @Override - public Block.Ref eval(Page page) { - Block.Ref ref = field.eval(page); - return ref.block().mayHaveMultivaluedFields() ? evalNullable(ref) : evalSingleValuedNullable(ref); + public Block eval(Page page) { + try (Block block = field.eval(page)) { + if (block.mayHaveMultivaluedFields()) { + return evalNullable(block); + } else { + return evalSingleValuedNullable(block); + } + } } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index a187bb41ee235..9f5c492d7fe7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -120,11 +120,8 @@ private static class Evaluator implements ExpressionEvaluator { } @Override - public final Block.Ref eval(Page page) { - try (Block.Ref fieldRef = field.eval(page); Block.Ref delimRef = delim.eval(page)) { - BytesRefBlock fieldVal = (BytesRefBlock) fieldRef.block(); - BytesRefBlock delimVal = (BytesRefBlock) delimRef.block(); - + public final Block eval(Page page) { + try (BytesRefBlock fieldVal = (BytesRefBlock) field.eval(page); BytesRefBlock delimVal = (BytesRefBlock) delim.eval(page)) { int positionCount = page.getPositionCount(); try (BytesRefBlock.Builder builder = BytesRefBlock.newBlockBuilder(positionCount, context.blockFactory())) { BytesRefBuilder work = new BytesRefBuilder(); // TODO BreakingBytesRefBuilder so we don't blow past circuit breakers @@ -155,7 +152,7 @@ public final Block.Ref eval(Page page) { } builder.appendBytesRef(work.get()); } - return Block.Ref.floating(builder.build()); + return builder.build(); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java index 528c0b6d5f0cb..6d1446f4cccf4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java @@ -95,46 +95,38 @@ protected String name() { } @Override - protected Block.Ref evalNullable(Block.Ref ref) { - try (ref; IntBlock.Builder builder = IntBlock.newBlockBuilder(ref.block().getPositionCount(), driverContext.blockFactory())) { - for (int p = 0; p < ref.block().getPositionCount(); p++) { - int valueCount = ref.block().getValueCount(p); + protected Block evalNullable(Block block) { + try (var builder = IntBlock.newBlockBuilder(block.getPositionCount(), driverContext.blockFactory())) { + for (int p = 0; p < block.getPositionCount(); p++) { + int valueCount = block.getValueCount(p); if (valueCount == 0) { builder.appendNull(); continue; } builder.appendInt(valueCount); } - return Block.Ref.floating(builder.build()); + return builder.build(); } } @Override - protected Block.Ref evalNotNullable(Block.Ref ref) { - try ( - ref; - IntVector.FixedBuilder builder = IntVector.newVectorFixedBuilder( - ref.block().getPositionCount(), - driverContext.blockFactory() - ) - ) { - for (int p = 0; p < ref.block().getPositionCount(); p++) { - builder.appendInt(ref.block().getValueCount(p)); + protected Block evalNotNullable(Block block) { + try (var builder = IntVector.newVectorFixedBuilder(block.getPositionCount(), driverContext.blockFactory())) { + for (int p = 0; p < block.getPositionCount(); p++) { + builder.appendInt(block.getValueCount(p)); } - return Block.Ref.floating(builder.build().asBlock()); + return builder.build().asBlock(); } } @Override - protected Block.Ref evalSingleValuedNullable(Block.Ref ref) { + protected Block evalSingleValuedNullable(Block ref) { return evalNullable(ref); } @Override - protected Block.Ref evalSingleValuedNotNullable(Block.Ref ref) { - try (ref) { - return Block.Ref.floating(driverContext.blockFactory().newConstantIntBlockWith(1, ref.block().getPositionCount())); - } + protected Block evalSingleValuedNotNullable(Block ref) { + return driverContext.blockFactory().newConstantIntBlockWith(1, ref.getPositionCount()); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index d0fe387d680db..ea95971e1b7b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -143,7 +143,7 @@ private record CoalesceEvaluator(DriverContext driverContext, ElementType result implements EvalOperator.ExpressionEvaluator { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { /* * We have to evaluate lazily so any errors or warnings that would be * produced by the right hand side are avoided. And so if anything @@ -163,9 +163,9 @@ public Block.Ref eval(Page page) { ); try (Releasable ignored = limited::releaseBlocks) { for (EvalOperator.ExpressionEvaluator eval : evaluators) { - try (Block.Ref ref = eval.eval(limited)) { - if (false == ref.block().isNull(0)) { - result.copyFrom(ref.block(), 0, 1); + try (Block block = eval.eval(limited)) { + if (false == block.isNull(0)) { + result.copyFrom(block, 0, 1); continue position; } } @@ -173,7 +173,7 @@ public Block.Ref eval(Page page) { result.appendNull(); } } - return Block.Ref.floating(result.build()); + return result.build(); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 35a63d720eb1d..64b93cc6eae5d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -233,8 +233,8 @@ private void testEvaluate(boolean readFloating) { // TODO should we convert unsigned_long into BigDecimal so it's easier to assert? Object result; try (ExpressionEvaluator evaluator = evaluator(expression).get(driverContext())) { - try (Block.Ref ref = evaluator.eval(row(testCase.getDataValues()))) { - result = toJavaObject(ref.block(), 0); + try (Block block = evaluator.eval(row(testCase.getDataValues()))) { + result = toJavaObject(block, 0); } } assertThat(result, not(equalTo(Double.NaN))); @@ -385,18 +385,17 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con } } Expression expression = readFloating ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase); - try (ExpressionEvaluator eval = evaluator(expression).get(context); Block.Ref ref = eval.eval(new Page(manyPositionsBlocks))) { - assertThat(ref.block().getPositionCount(), equalTo(ref.block().getPositionCount())); + try (ExpressionEvaluator eval = evaluator(expression).get(context); Block block = eval.eval(new Page(manyPositionsBlocks))) { for (int p = 0; p < positions; p++) { if (nullPositions.contains(p)) { - assertThat(toJavaObject(ref.block(), p), allNullsMatcher()); + assertThat(toJavaObject(block, p), allNullsMatcher()); continue; } - assertThat(toJavaObject(ref.block(), p), testCase.getMatcher()); + assertThat(toJavaObject(block, p), testCase.getMatcher()); } assertThat( "evaluates to tracked block", - ref.block().blockFactory(), + block.blockFactory(), either(sameInstance(context.blockFactory())).or(sameInstance(inputBlockFactory)) ); } @@ -428,8 +427,8 @@ public final void testSimpleWithNulls() { // TODO replace this with nulls insert data.add(simpleData.get(b)); } } - try (Block.Ref ref = eval.eval(new Page(blocks))) { - assertSimpleWithNulls(data, ref.block(), i); + try (Block block = eval.eval(new Page(blocks))) { + assertSimpleWithNulls(data, block, i); } } } @@ -456,8 +455,8 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru futures.add(exec.submit(() -> { try (EvalOperator.ExpressionEvaluator eval = evalSupplier.get(driverContext())) { for (int c = 0; c < count; c++) { - try (Block.Ref ref = eval.eval(page)) { - assertThat(toJavaObject(ref.block(), 0), testCase.getMatcher()); + try (Block block = eval.eval(page)) { + assertThat(toJavaObject(block, 0), testCase.getMatcher()); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java index 4f217a68b8535..5b67011a818ab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java @@ -37,9 +37,9 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( private final EvalOperator.ExpressionEvaluator child = childEval.get(ctx); @Override - public Block.Ref eval(Page page) { - try (Block.Ref ref = child.eval(page)) { - return Block.Ref.floating(BlockUtils.deepCopyOf(ref.block(), ctx.blockFactory())); + public Block eval(Page page) { + try (Block block = child.eval(page)) { + return BlockUtils.deepCopyOf(block, ctx.blockFactory()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index f776e1a4655d2..838044c8b90f6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -90,9 +90,9 @@ public void testEvalCase() { testCase(caseExpr -> { try ( EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(child -> evaluator(child)).get(driverContext()); - Block.Ref ref = eval.eval(new Page(IntBlock.newConstantBlockWith(0, 1))) + Block block = eval.eval(new Page(IntBlock.newConstantBlockWith(0, 1))) ) { - return toJavaObject(ref.block(), 0); + return toJavaObject(block, 0); } }); } @@ -148,12 +148,12 @@ public void testCaseWithIncompatibleTypes() { public void testCaseIsLazy() { Case caseExpr = caseExpr(true, 1, true, 2); - try (Block.Ref ref = caseExpr.toEvaluator(child -> { + try (Block block = caseExpr.toEvaluator(child -> { Object value = child.fold(); if (value != null && value.equals(2)) { return dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { fail("Unexpected evaluation of 4th argument"); return null; } @@ -164,7 +164,7 @@ public void close() {} } return evaluator(child); }).get(driverContext()).eval(new Page(IntBlock.newConstantBlockWith(0, 1)))) { - assertEquals(1, toJavaObject(ref.block(), 0)); + assertEquals(1, toJavaObject(block, 0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java index 8f10e2d11c5f1..5ef485e8ba441 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/RoundTests.java @@ -113,19 +113,19 @@ public void testExamples() { private Object process(Number val) { try ( - Block.Ref ref = evaluator(new Round(Source.EMPTY, field("val", typeOf(val)), null)).get(driverContext()).eval(row(List.of(val))) + Block block = evaluator(new Round(Source.EMPTY, field("val", typeOf(val)), null)).get(driverContext()).eval(row(List.of(val))) ) { - return toJavaObject(ref.block(), 0); + return toJavaObject(block, 0); } } private Object process(Number val, int decimals) { try ( - Block.Ref ref = evaluator(new Round(Source.EMPTY, field("val", typeOf(val)), field("decimals", DataTypes.INTEGER))).get( + Block block = evaluator(new Round(Source.EMPTY, field("val", typeOf(val)), field("decimals", DataTypes.INTEGER))).get( driverContext() ).eval(row(List.of(val, decimals))) ) { - return toJavaObject(ref.block(), 0); + return toJavaObject(block, 0); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 8db6b1bbd0c93..328f94b9c87e7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -89,7 +89,7 @@ public void testCoalesceIsLazy() { if (child == evil) { return dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override - public Block.Ref eval(Page page) { + public Block eval(Page page) { throw new AssertionError("shouldn't be called"); } @@ -101,9 +101,9 @@ public void close() {} }; try ( EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(map).get(driverContext()); - Block.Ref ref = eval.eval(row(testCase.getDataValues())) + Block block = eval.eval(row(testCase.getDataValues())) ) { - assertThat(toJavaObject(ref.block(), 0), testCase.getMatcher()); + assertThat(toJavaObject(block, 0), testCase.getMatcher()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index 32e894e5282d5..8b05b7478f570 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -135,9 +135,9 @@ public void testSomeConstant() { try ( EvalOperator.ExpressionEvaluator eval = evaluator(expression).get(driverContext()); - Block.Ref ref = eval.eval(row(fieldValues)) + Block block = eval.eval(row(fieldValues)) ) { - assertThat(toJavaObject(ref.block(), 0), testCase.getMatcher()); + assertThat(toJavaObject(block, 0), testCase.getMatcher()); } } @@ -150,7 +150,7 @@ private void testOversized(int totalLen, List mix, List fiel Exception e = expectThrows(EsqlClientException.class, () -> { try ( EvalOperator.ExpressionEvaluator eval = evaluator(expression).get(driverContext()); - Block.Ref ref = eval.eval(row(fieldValues)); + Block block = eval.eval(row(fieldValues)); ) {} }); assertThat(e.getMessage(), is("concatenating more than [1048576] bytes is not supported")); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index edbfec2bc5d85..316bb679f2b70 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -201,12 +201,12 @@ private String process(String str, int length) { EvalOperator.ExpressionEvaluator eval = evaluator( new Left(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, length, DataTypes.INTEGER)) ).get(driverContext()); - Block.Ref ref = eval.eval(row(List.of(new BytesRef(str)))) + Block block = eval.eval(row(List.of(new BytesRef(str)))) ) { - if (ref.block().isNull(0)) { + if (block.isNull(0)) { return null; } - BytesRef resultByteRef = ((BytesRef) toJavaObject(ref.block(), 0)); + BytesRef resultByteRef = ((BytesRef) toJavaObject(block, 0)); return resultByteRef == null ? null : resultByteRef.utf8ToString(); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index 04b8dd4079028..0eeb312512b30 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -203,12 +203,12 @@ private String process(String str, int length) { EvalOperator.ExpressionEvaluator eval = evaluator( new Right(Source.EMPTY, field("str", DataTypes.KEYWORD), new Literal(Source.EMPTY, length, DataTypes.INTEGER)) ).get(driverContext()); - Block.Ref ref = eval.eval(row(List.of(new BytesRef(str)))) + Block block = eval.eval(row(List.of(new BytesRef(str)))) ) { - if (ref.block().isNull(0)) { + if (block.isNull(0)) { return null; } - BytesRef resultByteRef = ((BytesRef) toJavaObject(ref.block(), 0)); + BytesRef resultByteRef = ((BytesRef) toJavaObject(block, 0)); return resultByteRef == null ? null : resultByteRef.utf8ToString(); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index 3926cc46dd883..abaa382637882 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -96,8 +96,8 @@ public void testConstantDelimiter() { */ assert ':' == 58; assertThat(eval.toString(), equalTo("SplitSingleByteEvaluator[str=Attribute[channel=0], delim=58]")); - try (Block.Ref ref = eval.eval(new Page(BytesRefBlock.newConstantBlockWith(new BytesRef("foo:bar"), 1)))) { - assertThat(toJavaObject(ref.block(), 0), equalTo(List.of(new BytesRef("foo"), new BytesRef("bar")))); + try (Block block = eval.eval(new Page(BytesRefBlock.newConstantBlockWith(new BytesRef("foo:bar"), 1)))) { + assertThat(toJavaObject(block, 0), equalTo(List.of(new BytesRef("foo"), new BytesRef("bar")))); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index 722b2bea8060a..fd9cb29ec62c4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -139,9 +139,9 @@ private String process(String str, int start, Integer length) { length == null ? null : new Literal(Source.EMPTY, length, DataTypes.INTEGER) ) ).get(driverContext()); - Block.Ref ref = eval.eval(row(List.of(new BytesRef(str)))) + Block block = eval.eval(row(List.of(new BytesRef(str)))) ) { - return ref.block().isNull(0) ? null : ((BytesRef) toJavaObject(ref.block(), 0)).utf8ToString(); + return block.isNull(0) ? null : ((BytesRef) toJavaObject(block, 0)).utf8ToString(); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java index 6b81fbb1a2fdd..cc677787c50c6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/AbstractBinaryOperatorTestCase.java @@ -94,8 +94,8 @@ public final void testApplyToAllTypes() { Source src = new Source(Location.EMPTY, lhsType.typeName() + " " + rhsType.typeName()); if (isRepresentable(lhsType) && isRepresentable(rhsType)) { op = build(src, field("lhs", lhsType), field("rhs", rhsType)); - try (Block.Ref ref = evaluator(op).get(driverContext()).eval(row(List.of(lhs.value(), rhs.value())))) { - result = toJavaObject(ref.block(), 0); + try (Block block = evaluator(op).get(driverContext()).eval(row(List.of(lhs.value(), rhs.value())))) { + result = toJavaObject(block, 0); } } else { op = build(src, lhs, rhs); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index dc365234351c6..727f0dcb9804d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -180,8 +180,8 @@ public void testEdgeCases() { private Object process(Object val) { if (testCase.allTypesAreRepresentable()) { Neg neg = new Neg(Source.EMPTY, field("val", typeOf(val))); - try (Block.Ref ref = evaluator(neg).get(driverContext()).eval(row(List.of(val)))) { - return toJavaObject(ref.block(), 0); + try (Block block = evaluator(neg).get(driverContext()).eval(row(List.of(val)))) { + return toJavaObject(block, 0); } } else { // just fold if type is not representable Neg neg = new Neg(Source.EMPTY, new Literal(Source.EMPTY, val, typeOf(val))); From 0ecb2af13d9910f01f077f02a95c5eb3e5dfab53 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 23 Nov 2023 20:47:06 +0000 Subject: [PATCH 10/35] AwaitsFix for #101095 --- .../java/org/elasticsearch/compute/operator/DriverTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java index 8f0d0e37ede23..39177c57fbffd 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/DriverTests.java @@ -39,6 +39,7 @@ public class DriverTests extends ESTestCase { + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101095") public void testThreadContext() throws Exception { DriverContext driverContext = driverContext(); ThreadPool threadPool = threadPool(); From b2127ec2f96674b1ec5b2d08a2be9c8ccf0226a1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 23 Nov 2023 21:40:43 +0000 Subject: [PATCH 11/35] Introduce RefCounted#mustIncRef (#102515) In several places we acquire a ref to a resource that we are certain is not closed, so this commit adds a utility for asserting this to be the case. This also helps a little with mocks since boolean methods like `tryIncRef()` return `false` on mock objects by default, but void methods like `mustIncRef()` default to being a no-op. --- .../core/AbstractRefCounted.java | 3 +- .../org/elasticsearch/core/RefCounted.java | 12 ++++++++ .../ingest/geoip/GeoIpDownloaderTests.java | 28 +++++-------------- .../repositories/s3/S3Service.java | 2 +- .../action/support/RefCountingRunnable.java | 14 ++++------ .../node/TransportBroadcastByNodeAction.java | 4 +-- .../master/TransportMasterNodeAction.java | 2 +- .../support/tasks/TransportTasksAction.java | 2 +- .../internal/support/AbstractClient.java | 3 +- .../coordination/JoinValidationService.java | 3 +- .../util/CancellableSingleObjectCache.java | 2 +- .../util/concurrent/ThrottledIterator.java | 4 +-- .../transport/ClusterConnectionManager.java | 2 +- .../transport/InboundHandler.java | 4 +-- .../transport/TransportActionProxy.java | 2 +- .../transport/TransportService.java | 4 +-- .../support/RefCountingListenerTests.java | 5 ++-- .../support/RefCountingRunnableTests.java | 5 ++-- .../transport/DisruptableMockTransport.java | 2 +- .../BatchedDocumentsIteratorTests.java | 5 +--- .../SecurityServerTransportInterceptor.java | 2 +- .../xpack/watcher/WatcherServiceTests.java | 1 - .../execution/TriggeredWatchStoreTests.java | 2 -- 23 files changed, 51 insertions(+), 62 deletions(-) diff --git a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java index 04cd4375a42be..ca5704fa9866d 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/AbstractRefCounted.java @@ -19,6 +19,7 @@ public abstract class AbstractRefCounted implements RefCounted { public static final String ALREADY_CLOSED_MESSAGE = "already closed, can't increment ref count"; + public static final String INVALID_DECREF_MESSAGE = "invalid decRef call: already closed"; private static final VarHandle VH_REFCOUNT_FIELD; @@ -63,7 +64,7 @@ public final boolean tryIncRef() { public final boolean decRef() { touch(); int i = (int) VH_REFCOUNT_FIELD.getAndAdd(this, -1); - assert i > 0 : "invalid decRef call: already closed"; + assert i > 0 : INVALID_DECREF_MESSAGE; if (i == 1) { try { closeInternal(); diff --git a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java index 49c030609951b..1f725ac48a16f 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java @@ -62,4 +62,16 @@ public interface RefCounted { * @return whether there are currently any active references to this object. */ boolean hasReferences(); + + /** + * Similar to {@link #incRef()} except that it also asserts that it managed to acquire the ref, for use in situations where it is a bug + * if all refs have been released. + */ + default void mustIncRef() { + if (tryIncRef()) { + return; + } + assert false : AbstractRefCounted.ALREADY_CLOSED_MESSAGE; + incRef(); // throws an ISE + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index db8e157d1576f..5fedb357fff8e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -180,15 +180,11 @@ public int read() throws IOException { public void testIndexChunksNoData() throws IOException { client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var flushResponse = mock(FlushResponse.class); - when(flushResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(flushResponse); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); }); client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var refreshResponse = mock(RefreshResponse.class); - when(refreshResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(refreshResponse); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); }); InputStream empty = new ByteArrayInputStream(new byte[0]); @@ -198,15 +194,11 @@ public void testIndexChunksNoData() throws IOException { public void testIndexChunksMd5Mismatch() { client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var flushResponse = mock(FlushResponse.class); - when(flushResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(flushResponse); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); }); client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var refreshResponse = mock(RefreshResponse.class); - when(refreshResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(refreshResponse); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); }); IOException exception = expectThrows( @@ -238,21 +230,15 @@ public void testIndexChunks() throws IOException { assertEquals("test", source.get("name")); assertArrayEquals(chunksData[chunk], (byte[]) source.get("data")); assertEquals(chunk + 15, source.get("chunk")); - var indexResponse = mock(IndexResponse.class); - when(indexResponse.hasReferences()).thenReturn(true); - listener.onResponse(indexResponse); + listener.onResponse(mock(IndexResponse.class)); }); client.addHandler(FlushAction.INSTANCE, (FlushRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var flushResponse = mock(FlushResponse.class); - when(flushResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(flushResponse); + flushResponseActionListener.onResponse(mock(FlushResponse.class)); }); client.addHandler(RefreshAction.INSTANCE, (RefreshRequest request, ActionListener flushResponseActionListener) -> { assertArrayEquals(new String[] { GeoIpDownloader.DATABASES_INDEX }, request.indices()); - var refreshResponse = mock(RefreshResponse.class); - when(refreshResponse.hasReferences()).thenReturn(true); - flushResponseActionListener.onResponse(refreshResponse); + flushResponseActionListener.onResponse(mock(RefreshResponse.class)); }); InputStream big = new ByteArrayInputStream(bigArray); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index e33bfbff141b2..3a8c35fffafae 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -135,7 +135,7 @@ public AmazonS3Reference client(RepositoryMetadata repositoryMetadata) { return existing; } final AmazonS3Reference clientReference = new AmazonS3Reference(buildClient(clientSettings)); - clientReference.incRef(); + clientReference.mustIncRef(); clientsCache = Maps.copyMapWithAddedEntry(clientsCache, clientSettings, clientReference); return clientReference; } diff --git a/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java index d05f698749a3b..8dcc801f10c30 100644 --- a/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java +++ b/server/src/main/java/org/elasticsearch/action/support/RefCountingRunnable.java @@ -63,7 +63,6 @@ public final class RefCountingRunnable implements Releasable { private static final Logger logger = LogManager.getLogger(RefCountingRunnable.class); - static final String ALREADY_CLOSED_MESSAGE = "already closed, cannot acquire or release any further refs"; private final RefCounted refCounted; @@ -86,14 +85,11 @@ public RefCountingRunnable(Runnable delegate) { * will be ignored otherwise. This deviates from the contract of {@link java.io.Closeable}. */ public Releasable acquire() { - if (refCounted.tryIncRef()) { - // All refs are considered equal so there's no real need to allocate a new object here, although note that this deviates - // (subtly) from the docs for Closeable#close() which indicate that it should be idempotent. But only if assertions are - // disabled, and if assertions are enabled then we are asserting that we never double-close these things anyway. - return Releasables.assertOnce(this); - } - assert false : ALREADY_CLOSED_MESSAGE; - throw new IllegalStateException(ALREADY_CLOSED_MESSAGE); + refCounted.mustIncRef(); + // All refs are considered equal so there's no real need to allocate a new object here, although note that this deviates (subtly) + // from the docs for Closeable#close() which indicate that it should be idempotent. But only if assertions are disabled, and if + // assertions are enabled then we are asserting that we never double-close these things anyway. + return Releasables.assertOnce(this); } /** diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java index 194b4852c16d7..19c7561ccdb15 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java @@ -228,7 +228,7 @@ protected String[] resolveConcreteIndexNames(ClusterState clusterState, Request @Override protected void doExecute(Task task, Request request, ActionListener listener) { // workaround for https://github.com/elastic/elasticsearch/issues/97916 - TODO remove this when we can - request.incRef(); + request.mustIncRef(); executor.execute(ActionRunnable.wrapReleasing(listener, request::decRef, l -> doExecuteForked(task, request, listener))); } @@ -474,7 +474,7 @@ class NodeRequest extends TransportRequest implements IndicesRequest { } NodeRequest(Request indicesLevelRequest, List shards, String nodeId) { - indicesLevelRequest.incRef(); + indicesLevelRequest.mustIncRef(); this.indicesLevelRequest = indicesLevelRequest; this.shards = shards; this.nodeId = nodeId; diff --git a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index 71964d737e8d2..b771f6cc512d1 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -169,7 +169,7 @@ protected void doExecute(Task task, final Request request, ActionListener extends PlainActionF @Override public final void onResponse(R result) { - assert result.hasReferences(); if (set(result)) { - result.incRef(); + result.mustIncRef(); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index dc18a7950394a..6ba35d6aec25a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -363,8 +363,7 @@ protected void doRun() { ); return; } - assert bytes.hasReferences() : "already closed"; - bytes.incRef(); + bytes.mustIncRef(); transportService.sendRequest( connection, JOIN_VALIDATE_ACTION_NAME, diff --git a/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java b/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java index 467efff9e72c4..081dec3f6b7db 100644 --- a/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java +++ b/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java @@ -192,7 +192,7 @@ private final class CachedItem extends AbstractRefCounted { CachedItem(Key key) { this.key = key; - incRef(); // start with a refcount of 2 so we're not closed while adding the first listener + mustIncRef(); // start with a refcount of 2 so we're not closed while adding the first listener this.future.addListener(new ActionListener<>() { @Override public void onResponse(Value value) { diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledIterator.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledIterator.java index d6ac42a9211c9..34236b957dea2 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledIterator.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThrottledIterator.java @@ -88,7 +88,7 @@ private void run() { } } try (var itemRefs = new ItemRefCounted()) { - itemRefs.incRef(); + itemRefs.mustIncRef(); itemConsumer.accept(Releasables.releaseOnce(itemRefs::decRef), item); } catch (Exception e) { logger.error(Strings.format("exception when processing [%s] with [%s]", item, itemConsumer), e); @@ -108,7 +108,7 @@ private class ItemRefCounted extends AbstractRefCounted implements Releasable { private boolean isRecursive = true; ItemRefCounted() { - refs.incRef(); + refs.mustIncRef(); } @Override diff --git a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java index 419cbf97dfa09..4d6a66b6ec075 100644 --- a/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java +++ b/server/src/main/java/org/elasticsearch/transport/ClusterConnectionManager.java @@ -223,7 +223,7 @@ private void connectToNodeOrRetry( IOUtils.closeWhileHandlingException(conn); } else { logger.debug("connected to node [{}]", node); - managerRefs.incRef(); + managerRefs.mustIncRef(); try { connectionListener.onNodeConnected(node, conn); } finally { diff --git a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java index 5d47c79abfd61..1686213139722 100644 --- a/server/src/main/java/org/elasticsearch/transport/InboundHandler.java +++ b/server/src/main/java/org/elasticsearch/transport/InboundHandler.java @@ -293,7 +293,7 @@ private static void doHandleRequest(RequestHandlerR private void handleRequestForking(T request, RequestHandlerRegistry reg, TransportChannel channel) { boolean success = false; - request.incRef(); + request.mustIncRef(); try { reg.getExecutor().execute(threadPool.getThreadContext().preserveContextWithTracing(new AbstractRunnable() { @Override @@ -381,7 +381,7 @@ private void handleResponse( // no need to provide a buffer release here, we never escape the buffer when handling directly doHandleResponse(handler, remoteAddress, stream, inboundMessage.getHeader(), () -> {}); } else { - inboundMessage.incRef(); + inboundMessage.mustIncRef(); // release buffer once we deserialize the message, but have a fail-safe in #onAfter below in case that didn't work out final Releasable releaseBuffer = Releasables.releaseOnce(inboundMessage::decRef); executor.execute(new ForkingResponseHandlerRunnable(handler, null, threadPool) { diff --git a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java index aa28f8a76b58e..ecd4ec6e4fc1b 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportActionProxy.java @@ -65,7 +65,7 @@ public Executor executor(ThreadPool threadPool) { @Override public void handleResponse(TransportResponse response) { try { - response.incRef(); + response.mustIncRef(); channel.sendResponse(response); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/server/src/main/java/org/elasticsearch/transport/TransportService.java b/server/src/main/java/org/elasticsearch/transport/TransportService.java index 7550435ac0bb8..8e6c1e67fcd10 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportService.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportService.java @@ -1013,7 +1013,7 @@ private void sendLocalRequest(long requestId, final String action, final Transpo } } else { boolean success = false; - request.incRef(); + request.mustIncRef(); try { executor.execute(threadPool.getThreadContext().preserveContextWithTracing(new AbstractRunnable() { @Override @@ -1479,7 +1479,7 @@ public void sendResponse(TransportResponse response) throws IOException { if (executor == EsExecutors.DIRECT_EXECUTOR_SERVICE) { processResponse(handler, response); } else { - response.incRef(); + response.mustIncRef(); executor.execute(new ForkingResponseHandlerRunnable(handler, null, threadPool) { @Override protected void doRun() { diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java index fe1d45e6a5002..6e2e984c060fb 100644 --- a/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingListenerTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.RunOnce; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ReachabilityChecker; @@ -174,10 +175,10 @@ public void testValidation() { final String expectedMessage; if (randomBoolean()) { throwingRunnable = refs::acquire; - expectedMessage = RefCountingRunnable.ALREADY_CLOSED_MESSAGE; + expectedMessage = AbstractRefCounted.ALREADY_CLOSED_MESSAGE; } else { throwingRunnable = refs::close; - expectedMessage = "invalid decRef call: already closed"; + expectedMessage = AbstractRefCounted.INVALID_DECREF_MESSAGE; } assertEquals(expectedMessage, expectThrows(AssertionError.class, throwingRunnable).getMessage()); diff --git a/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java index b5ccc4f50969b..1018b073adb9d 100644 --- a/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/RefCountingRunnableTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.AbstractRefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.test.ESTestCase; @@ -166,10 +167,10 @@ public void testValidation() { final String expectedMessage; if (randomBoolean()) { throwingRunnable = randomBoolean() ? refs::acquire : refs::acquireListener; - expectedMessage = RefCountingRunnable.ALREADY_CLOSED_MESSAGE; + expectedMessage = AbstractRefCounted.ALREADY_CLOSED_MESSAGE; } else { throwingRunnable = refs::close; - expectedMessage = "invalid decRef call: already closed"; + expectedMessage = AbstractRefCounted.INVALID_DECREF_MESSAGE; } assertEquals(expectedMessage, expectThrows(AssertionError.class, throwingRunnable).getMessage()); diff --git a/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java index 05d6eca0d021d..eb85323caf5a1 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/DisruptableMockTransport.java @@ -150,7 +150,7 @@ protected void onSendRequest( assert destinationTransport.getLocalNode().equals(getLocalNode()) == false : "non-local message from " + getLocalNode() + " to itself"; - request.incRef(); + request.mustIncRef(); destinationTransport.execute(new RebootSensitiveRunnable() { @Override diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java index 166624a77a352..d0efe69e8ac49 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/BatchedDocumentsIteratorTests.java @@ -140,9 +140,7 @@ private void givenClearScrollRequest() { doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2]; wasScrollCleared = true; - var clearScrollResponse = mock(ClearScrollResponse.class); - when(clearScrollResponse.hasReferences()).thenReturn(true); - listener.onResponse(clearScrollResponse); + listener.onResponse(mock(ClearScrollResponse.class)); return null; }).when(client).execute(eq(ClearScrollAction.INSTANCE), any(), any()); } @@ -173,7 +171,6 @@ ResponsesMocker addBatch(String... hits) { protected SearchResponse createSearchResponseWithHits(String... hits) { SearchHits searchHits = createHits(hits); SearchResponse searchResponse = mock(SearchResponse.class); - when(searchResponse.hasReferences()).thenReturn(true); when(searchResponse.getScrollId()).thenReturn(SCROLL_ID); when(searchResponse.getHits()).thenReturn(searchHits); return searchResponse; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 123814ec38c6f..53dd31fe46793 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -543,7 +543,7 @@ public static class ProfileSecuredRequestHandler imp AbstractRunnable getReceiveRunnable(T request, TransportChannel channel, Task task) { final Runnable releaseRequest = new RunOnce(request::decRef); - request.incRef(); + request.mustIncRef(); return new AbstractRunnable() { @Override public boolean isForceExecution() { diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index 623c72ee93c2f..87c7c6e9748f8 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -163,7 +163,6 @@ void stopExecutor() {} // response setup, successful refresh response RefreshResponse refreshResponse = mock(RefreshResponse.class); - when(refreshResponse.hasReferences()).thenReturn(true); when(refreshResponse.getSuccessfulShards()).thenReturn( clusterState.getMetadata().getIndices().get(Watch.INDEX).getNumberOfShards() ); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index 385ee448e2b47..01547b898e4b4 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -210,7 +210,6 @@ public void testFindTriggeredWatchesGoodCase() { SearchResponse searchResponse1 = mock(SearchResponse.class); when(searchResponse1.getSuccessfulShards()).thenReturn(1); when(searchResponse1.getTotalShards()).thenReturn(1); - when(searchResponse1.hasReferences()).thenReturn(true); BytesArray source = new BytesArray("{}"); SearchHit hit = new SearchHit(0, "first_foo"); hit.version(1L); @@ -513,7 +512,6 @@ private RefreshResponse mockRefreshResponse(int total, int successful) { RefreshResponse refreshResponse = mock(RefreshResponse.class); when(refreshResponse.getTotalShards()).thenReturn(total); when(refreshResponse.getSuccessfulShards()).thenReturn(successful); - when(refreshResponse.hasReferences()).thenReturn(true); return refreshResponse; } } From 62be62cec0c31246ca1447400131fc5142429cd9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Nov 2023 00:32:10 +0100 Subject: [PATCH 12/35] Check docker env by probing docker (#102558) For now we probe ddocker functionality when using minio testcontainer. We need to adjust our docker availability matrix. Should mute tests and address #102532 --- .../s3/S3RepositoryThirdPartyTests.java | 2 ++ .../s3/RepositoryS3ClientYamlTestSuiteIT.java | 2 ++ .../DockerEnvironmentAwareTestContainer.java | 28 +++++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java index 18f5de496f9bb..b11120e068d14 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3RepositoryThirdPartyTests.java @@ -12,6 +12,7 @@ import com.amazonaws.services.s3.model.ListMultipartUploadsRequest; import com.amazonaws.services.s3.model.MultipartUpload; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -54,6 +55,7 @@ import static org.hamcrest.Matchers.not; @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase { static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("tests.use.fixture", "true")); diff --git a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java index d4b964fb3a7f0..2f2f42974f131 100644 --- a/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java +++ b/modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java @@ -15,6 +15,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter; @@ -24,6 +25,7 @@ import org.junit.rules.TestRule; @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT { public static final S3HttpFixture s3Fixture = new S3HttpFixture(); diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java index 588fbf3d6a615..250e56fda4105 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -12,6 +12,7 @@ import org.junit.Assume; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.GenericContainer; import org.testcontainers.images.builder.ImageFromDockerfile; @@ -30,29 +31,44 @@ public class DockerEnvironmentAwareTestContainer extends GenericContainerhttps://github.com/elastic/elasticsearch/issues/102532 + * */ + private static boolean isDockerAvailable() { + try { + DockerClientFactory.instance().client(); + return true; + } catch (Throwable ex) { + LOGGER.warn("Probing docker has failed; disabling test", ex); + return false; + } + } + public DockerEnvironmentAwareTestContainer(ImageFromDockerfile imageFromDockerfile) { super(imageFromDockerfile); } @Override public void start() { - Assume.assumeTrue("Docker is not available", shouldDockerBeAvailable()); + Assume.assumeFalse("Docker support excluded on OS", EXCLUDED_OS); + Assume.assumeTrue("Docker probing succesful", DOCKER_PROBING_SUCCESSFUL); super.start(); } - private boolean shouldDockerBeAvailable() { - return CI == false || (EXCLUDED_OS == false); - } - static String deriveId(Map values) { return values.get("ID") + "-" + values.get("VERSION_ID"); } private static boolean isExcludedOs() { + if (CI) { + // we dont exclude OS outside of CI environment + return false; + } if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { return true; } From 774a05cc0ad0cc6ec142af5564f57acac10080bc Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 23 Nov 2023 17:33:38 -0800 Subject: [PATCH 13/35] Update doc fields for pow and date_extract function (#102554) These two generated files have not been updated and committed. --- .../esql/functions/signature/date_extract.svg | 1 - docs/reference/esql/functions/types/pow.asciidoc | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 docs/reference/esql/functions/signature/date_extract.svg diff --git a/docs/reference/esql/functions/signature/date_extract.svg b/docs/reference/esql/functions/signature/date_extract.svg deleted file mode 100644 index ec69633c02e8b..0000000000000 --- a/docs/reference/esql/functions/signature/date_extract.svg +++ /dev/null @@ -1 +0,0 @@ -DATE_EXTRACT(arg1,arg2) \ No newline at end of file diff --git a/docs/reference/esql/functions/types/pow.asciidoc b/docs/reference/esql/functions/types/pow.asciidoc index 3965732945c50..0e22c123ebf53 100644 --- a/docs/reference/esql/functions/types/pow.asciidoc +++ b/docs/reference/esql/functions/types/pow.asciidoc @@ -3,8 +3,18 @@ base | exponent | result double | double | double double | integer | double +double | long | double +double | unsigned_long | double integer | double | double integer | integer | double +integer | long | double +integer | unsigned_long | double long | double | double long | integer | double +long | long | double +long | unsigned_long | double +unsigned_long | double | double +unsigned_long | integer | double +unsigned_long | long | double +unsigned_long | unsigned_long | double |=== From 5bc01896d8a02df768cda3848257538522479b23 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Nov 2023 08:47:12 +0100 Subject: [PATCH 14/35] Fix Nullpointer in test container tests and setup logging (#102561) related to #102532 --- modules/repository-s3/build.gradle | 1 + .../testcontainers/DockerEnvironmentAwareTestContainer.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/repository-s3/build.gradle b/modules/repository-s3/build.gradle index 645a989872f49..8b1f30a1bba61 100644 --- a/modules/repository-s3/build.gradle +++ b/modules/repository-s3/build.gradle @@ -48,6 +48,7 @@ dependencies { internalClusterTestImplementation project(':test:fixtures:minio-fixture') yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" + internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" } restResources { diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java index 250e56fda4105..c0fb83e5206f4 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/testcontainers/DockerEnvironmentAwareTestContainer.java @@ -28,20 +28,22 @@ import java.util.stream.Collectors; public class DockerEnvironmentAwareTestContainer extends GenericContainer { + protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class); + private static final String DOCKER_ON_LINUX_EXCLUSIONS_FILE = ".ci/dockerOnLinuxExclusions"; private static final boolean CI = Boolean.parseBoolean(System.getProperty("CI", "false")); private static final boolean EXCLUDED_OS = isExcludedOs(); private static final boolean DOCKER_PROBING_SUCCESSFUL = isDockerAvailable(); - protected static final Logger LOGGER = LoggerFactory.getLogger(DockerEnvironmentAwareTestContainer.class); - /** * see https://github.com/elastic/elasticsearch/issues/102532 * */ private static boolean isDockerAvailable() { try { + LOGGER.info("Probing docker environment..."); DockerClientFactory.instance().client(); + LOGGER.info("Probing docker environment successful"); return true; } catch (Throwable ex) { LOGGER.warn("Probing docker has failed; disabling test", ex); From 6b4cf5757301258dffee709636774c19683cc6b4 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Fri, 24 Nov 2023 09:17:07 +0100 Subject: [PATCH 15/35] Update metric names to follow open telemetry conventions (#102529) Update metric names to follow conventions described in https://opentelemetry.io/docs/specs/semconv/general/metrics/ --- .../allocator/DesiredBalanceReconciler.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index def92e7700d0b..dc3cbfa8b5ae8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -98,26 +98,26 @@ public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool thre unassignedShards = LongGaugeMetric.create( meterRegistry, - "es.allocator.desired_balance.unassigned_shards", - "Current unassigned shards count", - "count" + "es.allocator.desired_balance.shards.unassigned", + "Unassigned shards count", + "{shard}" ); totalAllocations = LongGaugeMetric.create( meterRegistry, - "es.allocator.desired_balance.total_allocations", - "Current total shards count in cluster", - "count" + "es.allocator.desired_balance.shards.count", + "Total shards count", + "{shard}" ); undesiredAllocations = LongGaugeMetric.create( meterRegistry, - "es.allocator.desired_balance.undesired_allocations", - "Current number of shards allocated on undesired nodes", - "count" + "es.allocator.desired_balance.allocations.undesired", + "Count of shards allocated on undesired nodes", + "{shard}" ); undesiredAllocationsRatio = meterRegistry.registerDoubleGauge( - "es.allocator.desired_balance.undesired_allocations_ratio", - "Current undesired_allocations / allocations ratio", - "count", + "es.allocator.desired_balance.allocations.undesired_ratio", + "Ratio of undesired allocations to shard count", + "1", () -> { var total = totalAllocations.get(); var undesired = undesiredAllocations.get(); From 51eef046a0a7a31014d556c28f5374874c80fea3 Mon Sep 17 00:00:00 2001 From: Sanjay J Date: Fri, 24 Nov 2023 14:03:21 +0530 Subject: [PATCH 16/35] Add support for configuring proxy scheme in S3 client settings and EC2 discovery plugin (#102495) Closes #101873 --- docs/changelog/102495.yaml | 6 +++++ docs/plugins/discovery-ec2.asciidoc | 5 ++++ .../snapshot-restore/repository-s3.asciidoc | 6 +++++ .../repositories/s3/S3ClientSettings.java | 18 +++++++++++++ .../repositories/s3/S3RepositoryPlugin.java | 1 + .../repositories/s3/S3Service.java | 1 + .../s3/S3ClientSettingsTests.java | 1 + .../discovery/ec2/AwsEc2ServiceImpl.java | 1 + .../discovery/ec2/Ec2ClientSettings.java | 16 +++++++++++- .../discovery/ec2/Ec2DiscoveryPlugin.java | 1 + .../discovery/ec2/AwsEc2ServiceImplTests.java | 25 +++++++++++++++++-- .../ec2/Ec2DiscoveryPluginTests.java | 6 +++++ 12 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/102495.yaml diff --git a/docs/changelog/102495.yaml b/docs/changelog/102495.yaml new file mode 100644 index 0000000000000..77ae42f7eebcb --- /dev/null +++ b/docs/changelog/102495.yaml @@ -0,0 +1,6 @@ +pr: 102495 +summary: "Add support for configuring proxy scheme in S3 client settings and EC2 discovery plugin" +area: Distributed +type: enhancement +issues: + - 101873 diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 3947ed71ea9ae..44acba4752aaa 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -97,6 +97,11 @@ The available settings for the EC2 discovery plugin are as follows. this setting determines the port to use to connect to the proxy. Defaults to `80`. +`discovery.ec2.proxy.scheme`:: + + The scheme to use when connecting to the EC2 service endpoint through proxy specified + in `discovery.ec2.proxy.host`. Valid values are `http` or `https`. Defaults to `http`. + `discovery.ec2.proxy.username` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable]):: When the address of an HTTP proxy is given in `discovery.ec2.proxy.host`, diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 3f2210f51cbb5..032d4f47bf678 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -133,6 +133,12 @@ settings belong in the `elasticsearch.yml` file. The port of a proxy to connect to S3 through. +`proxy.scheme`:: + + The scheme to use for the proxy connection to S3. Valid values are either `http` or `https`. + Defaults to `http`. This setting allows to specify the protocol used for communication with the + proxy server + `proxy.username` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable]):: The username to connect to the `proxy.host` with. diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index 7d1b495a0f008..ab322786fcd43 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -96,6 +96,13 @@ final class S3ClientSettings { key -> Setting.intSetting(key, 80, 0, 1 << 16, Property.NodeScope) ); + /** The proxy scheme for connecting to S3 through a proxy. */ + static final Setting.AffixSetting PROXY_SCHEME_SETTING = Setting.affixKeySetting( + PREFIX, + "proxy.scheme", + key -> new Setting<>(key, "http", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope) + ); + /** The username of a proxy to connect to s3 through. */ static final Setting.AffixSetting PROXY_USERNAME_SETTING = Setting.affixKeySetting( PREFIX, @@ -174,6 +181,9 @@ final class S3ClientSettings { /** The port number the proxy host should be connected on. */ final int proxyPort; + /** The proxy scheme to use for connecting to s3 through a proxy. */ + final Protocol proxyScheme; + // these should be "secure" yet the api for the s3 client only takes String, so storing them // as SecureString here won't really help with anything /** An optional username for the proxy host, for basic authentication. */ @@ -209,6 +219,7 @@ private S3ClientSettings( Protocol protocol, String proxyHost, int proxyPort, + Protocol proxyScheme, String proxyUsername, String proxyPassword, int readTimeoutMillis, @@ -224,6 +235,7 @@ private S3ClientSettings( this.protocol = protocol; this.proxyHost = proxyHost; this.proxyPort = proxyPort; + this.proxyScheme = proxyScheme; this.proxyUsername = proxyUsername; this.proxyPassword = proxyPassword; this.readTimeoutMillis = readTimeoutMillis; @@ -252,6 +264,7 @@ S3ClientSettings refine(Settings repositorySettings) { final Protocol newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol); final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost); final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxyPort); + final Protocol newProxyScheme = getRepoSettingOrDefault(PROXY_SCHEME_SETTING, normalizedSettings, proxyScheme); final int newReadTimeoutMillis = Math.toIntExact( getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis() ); @@ -275,6 +288,7 @@ S3ClientSettings refine(Settings repositorySettings) { && protocol == newProtocol && Objects.equals(proxyHost, newProxyHost) && proxyPort == newProxyPort + && proxyScheme == newProxyScheme && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries @@ -291,6 +305,7 @@ S3ClientSettings refine(Settings repositorySettings) { newProtocol, newProxyHost, newProxyPort, + newProxyScheme, proxyUsername, proxyPassword, newReadTimeoutMillis, @@ -398,6 +413,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String getConfigValue(settings, clientName, PROTOCOL_SETTING), getConfigValue(settings, clientName, PROXY_HOST_SETTING), getConfigValue(settings, clientName, PROXY_PORT_SETTING), + getConfigValue(settings, clientName, PROXY_SCHEME_SETTING), proxyUsername.toString(), proxyPassword.toString(), Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), @@ -428,6 +444,7 @@ public boolean equals(final Object o) { && Objects.equals(endpoint, that.endpoint) && protocol == that.protocol && Objects.equals(proxyHost, that.proxyHost) + && proxyScheme == that.proxyScheme && Objects.equals(proxyUsername, that.proxyUsername) && Objects.equals(proxyPassword, that.proxyPassword) && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding) @@ -443,6 +460,7 @@ public int hashCode() { protocol, proxyHost, proxyPort, + proxyScheme, proxyUsername, proxyPassword, readTimeoutMillis, diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 4042d414048d9..f85a66c5eb367 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -119,6 +119,7 @@ public List> getSettings() { S3ClientSettings.PROTOCOL_SETTING, S3ClientSettings.PROXY_HOST_SETTING, S3ClientSettings.PROXY_PORT_SETTING, + S3ClientSettings.PROXY_SCHEME_SETTING, S3ClientSettings.PROXY_USERNAME_SETTING, S3ClientSettings.PROXY_PASSWORD_SETTING, S3ClientSettings.READ_TIMEOUT_SETTING, diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 3a8c35fffafae..195a18891ebd0 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -221,6 +221,7 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // TODO: remove this leniency, these settings should exist together and be validated clientConfiguration.setProxyHost(clientSettings.proxyHost); clientConfiguration.setProxyPort(clientSettings.proxyPort); + clientConfiguration.setProxyProtocol(clientSettings.proxyScheme); clientConfiguration.setProxyUsername(clientSettings.proxyUsername); clientConfiguration.setProxyPassword(clientSettings.proxyPassword); } diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 8bff849ca26c2..c48e0dc337d30 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -37,6 +37,7 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); assertThat(defaultSettings.proxyHost, is(emptyString())); assertThat(defaultSettings.proxyPort, is(80)); + assertThat(defaultSettings.proxyScheme, is(Protocol.HTTP)); assertThat(defaultSettings.proxyUsername, is(emptyString())); assertThat(defaultSettings.proxyPassword, is(emptyString())); assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index ff32759508038..94aa05288a55c 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -62,6 +62,7 @@ static ClientConfiguration buildConfiguration(Ec2ClientSettings clientSettings) // TODO: remove this leniency, these settings should exist together and be validated clientConfiguration.setProxyHost(clientSettings.proxyHost); clientConfiguration.setProxyPort(clientSettings.proxyPort); + clientConfiguration.setProxyProtocol(clientSettings.proxyScheme); clientConfiguration.setProxyUsername(clientSettings.proxyUsername); clientConfiguration.setProxyPassword(clientSettings.proxyPassword); } diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java index 043114b26c81b..3a1cd1f1d33e6 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java @@ -48,6 +48,14 @@ final class Ec2ClientSettings { /** The port of a proxy to connect to ec2 through. */ static final Setting PROXY_PORT_SETTING = Setting.intSetting("discovery.ec2.proxy.port", 80, 0, 1 << 16, Property.NodeScope); + /** The scheme to use for the proxy connection to ec2. Defaults to "http". */ + static final Setting PROXY_SCHEME_SETTING = new Setting<>( + "discovery.ec2.proxy.scheme", + "http", + s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), + Property.NodeScope + ); + /** An override for the ec2 endpoint to connect to. */ static final Setting ENDPOINT_SETTING = new Setting<>( "discovery.ec2.endpoint", @@ -56,7 +64,7 @@ final class Ec2ClientSettings { Property.NodeScope ); - /** The protocol to use to connect to to ec2. */ + /** The protocol to use to connect to ec2. */ static final Setting PROTOCOL_SETTING = new Setting<>( "discovery.ec2.protocol", "https", @@ -99,6 +107,9 @@ final class Ec2ClientSettings { /** The port number the proxy host should be connected on. */ final int proxyPort; + /** The scheme to use for the proxy connection to ec2 */ + final Protocol proxyScheme; + // these should be "secure" yet the api for the ec2 client only takes String, so // storing them // as SecureString here won't really help with anything @@ -117,6 +128,7 @@ protected Ec2ClientSettings( Protocol protocol, String proxyHost, int proxyPort, + Protocol proxyScheme, String proxyUsername, String proxyPassword, int readTimeoutMillis @@ -126,6 +138,7 @@ protected Ec2ClientSettings( this.protocol = protocol; this.proxyHost = proxyHost; this.proxyPort = proxyPort; + this.proxyScheme = proxyScheme; this.proxyUsername = proxyUsername; this.proxyPassword = proxyPassword; this.readTimeoutMillis = readTimeoutMillis; @@ -196,6 +209,7 @@ static Ec2ClientSettings getClientSettings(Settings settings) { PROTOCOL_SETTING.get(settings), PROXY_HOST_SETTING.get(settings), PROXY_PORT_SETTING.get(settings), + PROXY_SCHEME_SETTING.get(settings), proxyUsername.toString(), proxyPassword.toString(), (int) READ_TIMEOUT_SETTING.get(settings).millis() diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java index 08cf7ea559bf7..69447e800d4ac 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java @@ -104,6 +104,7 @@ public List> getSettings() { Ec2ClientSettings.PROTOCOL_SETTING, Ec2ClientSettings.PROXY_HOST_SETTING, Ec2ClientSettings.PROXY_PORT_SETTING, + Ec2ClientSettings.PROXY_SCHEME_SETTING, Ec2ClientSettings.PROXY_USERNAME_SETTING, Ec2ClientSettings.PROXY_PASSWORD_SETTING, Ec2ClientSettings.READ_TIMEOUT_SETTING, diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java index bb73b951ca4f7..aa4a5bd6e54ea 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java @@ -119,7 +119,16 @@ public void testRejectionOfLoneSessionToken() { } public void testAWSDefaultConfiguration() { - launchAWSConfigurationTest(Settings.EMPTY, Protocol.HTTPS, null, -1, null, null, ClientConfiguration.DEFAULT_SOCKET_TIMEOUT); + launchAWSConfigurationTest( + Settings.EMPTY, + Protocol.HTTPS, + null, + -1, + Protocol.HTTP, + null, + null, + ClientConfiguration.DEFAULT_SOCKET_TIMEOUT + ); } public void testAWSConfigurationWithAwsSettings() { @@ -130,10 +139,20 @@ public void testAWSConfigurationWithAwsSettings() { .put("discovery.ec2.protocol", "http") .put("discovery.ec2.proxy.host", "aws_proxy_host") .put("discovery.ec2.proxy.port", 8080) + .put("discovery.ec2.proxy.scheme", "http") .put("discovery.ec2.read_timeout", "10s") .setSecureSettings(secureSettings) .build(); - launchAWSConfigurationTest(settings, Protocol.HTTP, "aws_proxy_host", 8080, "aws_proxy_username", "aws_proxy_password", 10000); + launchAWSConfigurationTest( + settings, + Protocol.HTTP, + "aws_proxy_host", + 8080, + Protocol.HTTP, + "aws_proxy_username", + "aws_proxy_password", + 10000 + ); } protected void launchAWSConfigurationTest( @@ -141,6 +160,7 @@ protected void launchAWSConfigurationTest( Protocol expectedProtocol, String expectedProxyHost, int expectedProxyPort, + Protocol expectedProxyScheme, String expectedProxyUsername, String expectedProxyPassword, int expectedReadTimeout @@ -151,6 +171,7 @@ protected void launchAWSConfigurationTest( assertThat(configuration.getProtocol(), is(expectedProtocol)); assertThat(configuration.getProxyHost(), is(expectedProxyHost)); assertThat(configuration.getProxyPort(), is(expectedProxyPort)); + assertThat(configuration.getProxyProtocol(), is(expectedProxyScheme)); assertThat(configuration.getProxyUsername(), is(expectedProxyUsername)); assertThat(configuration.getProxyPassword(), is(expectedProxyPassword)); assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout)); diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 93ff42fb50218..b9bea564e2720 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.discovery.ec2; import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; @@ -160,6 +161,7 @@ public void testClientSettingsReInit() throws IOException { final Settings settings1 = Settings.builder() .put(Ec2ClientSettings.PROXY_HOST_SETTING.getKey(), "proxy_host_1") .put(Ec2ClientSettings.PROXY_PORT_SETTING.getKey(), 881) + .put(Ec2ClientSettings.PROXY_SCHEME_SETTING.getKey(), "http") .put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), "ec2_endpoint_1") .setSecureSettings(mockSecure1) .build(); @@ -175,6 +177,7 @@ public void testClientSettingsReInit() throws IOException { final Settings settings2 = Settings.builder() .put(Ec2ClientSettings.PROXY_HOST_SETTING.getKey(), "proxy_host_2") .put(Ec2ClientSettings.PROXY_PORT_SETTING.getKey(), 882) + .put(Ec2ClientSettings.PROXY_SCHEME_SETTING.getKey(), "http") .put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), "ec2_endpoint_2") .setSecureSettings(mockSecure2) .build(); @@ -194,6 +197,7 @@ public void testClientSettingsReInit() throws IOException { assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP)); assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); } // reload secure settings2 @@ -211,6 +215,7 @@ public void testClientSettingsReInit() throws IOException { assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP)); assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); } } @@ -228,6 +233,7 @@ public void testClientSettingsReInit() throws IOException { assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(882)); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP)); assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_2")); } } From 4faa02f68cd48223cbd17761425558512566eb50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= Date: Fri, 24 Nov 2023 09:47:24 +0100 Subject: [PATCH 17/35] Rename the inference rescorer into learn_to_rank (#102540) --- .../core/src/main/java/module-info.java | 1 + .../MlLTRNamedXContentProvider.java | 2 +- .../xpack/core/ml/utils/QueryProvider.java | 7 +- .../InferenceConfigItemTestCase.java | 1 + .../trainedmodel/LearnToRankConfigTests.java | 2 +- .../LearnToRankConfigUpdateTests.java | 2 +- .../xpack/ml/integration/MlRescorerIT.java | 164 +++++------ .../ml/integration/InferenceRescorerIT.java | 277 ----------------- .../ml/integration/LearnToRankRescorerIT.java | 278 ++++++++++++++++++ .../xpack/ml/MachineLearning.java | 12 +- .../{rescorer => ltr}/FeatureExtractor.java | 2 +- .../FieldValueFeatureExtractor.java | 2 +- .../InferenceRescorerFeature.java | 4 +- .../LearnToRankRescorer.java} | 18 +- .../LearnToRankRescorerBuilder.java} | 65 ++-- .../LearnToRankRescorerContext.java} | 34 +-- .../QueryFeatureExtractor.java | 2 +- ...arnToRankRescorerBuilderRewriteTests.java} | 67 +++-- ...ankRescorerBuilderSerializationTests.java} | 40 +-- .../QueryFeatureExtractorTests.java | 2 +- .../test/ml/inference_rescore.yml | 12 +- 21 files changed, 506 insertions(+), 488 deletions(-) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/{inference => ltr}/MlLTRNamedXContentProvider.java (98%) delete mode 100644 x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceRescorerIT.java create mode 100644 x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearnToRankRescorerIT.java rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer => ltr}/FeatureExtractor.java (91%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer => ltr}/FieldValueFeatureExtractor.java (97%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer => ltr}/InferenceRescorerFeature.java (89%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer/InferenceRescorer.java => ltr/LearnToRankRescorer.java} (91%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer/InferenceRescorerBuilder.java => ltr/LearnToRankRescorerBuilder.java} (85%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer/InferenceRescorerContext.java => ltr/LearnToRankRescorerContext.java} (74%) rename x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/{rescorer => ltr}/QueryFeatureExtractor.java (98%) rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/{rescorer/InferenceRescorerBuilderRewriteTests.java => ltr/LearnToRankRescorerBuilderRewriteTests.java} (81%) rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/{rescorer/InferenceRescorerBuilderSerializationTests.java => ltr/LearnToRankRescorerBuilderSerializationTests.java} (80%) rename x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/{rescorer => ltr}/QueryFeatureExtractorTests.java (99%) diff --git a/x-pack/plugin/core/src/main/java/module-info.java b/x-pack/plugin/core/src/main/java/module-info.java index f77b2eba19e6d..eb1271edd3b06 100644 --- a/x-pack/plugin/core/src/main/java/module-info.java +++ b/x-pack/plugin/core/src/main/java/module-info.java @@ -223,6 +223,7 @@ exports org.elasticsearch.xpack.core.watcher.trigger; exports org.elasticsearch.xpack.core.watcher.watch; exports org.elasticsearch.xpack.core.watcher; + exports org.elasticsearch.xpack.core.ml.ltr; provides org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber with diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlLTRNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/ltr/MlLTRNamedXContentProvider.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlLTRNamedXContentProvider.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/ltr/MlLTRNamedXContentProvider.java index dbfae12413632..2770b1bf93a97 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlLTRNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/ltr/MlLTRNamedXContentProvider.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.ml.inference; +package org.elasticsearch.xpack.core.ml.ltr; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.plugins.spi.NamedXContentProvider; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java index da50b1eb64b50..5b22165b57443 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/QueryProvider.java @@ -30,10 +30,9 @@ public class QueryProvider implements Writeable, ToXContentObject, Rewriteable { private static final Logger logger = LogManager.getLogger(QueryProvider.class); - - private Exception parsingException; - private QueryBuilder parsedQuery; - private Map query; + private final Exception parsingException; + private final QueryBuilder parsedQuery; + private final Map query; public static QueryProvider defaultQuery() { return new QueryProvider( diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java index ffa9334551360..a5ba16474fcbb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigTests; +import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; import java.util.ArrayList; import java.util.Collections; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigTests.java index ba971d535bf92..16e56b5dc73bd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigTests.java @@ -21,9 +21,9 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.inference.InferenceConfigItemTestCase; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.inference.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ltr.LearnToRankFeatureExtractorBuilder; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ltr.QueryExtractorBuilderTests; +import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigUpdateTests.java index 30befc767300b..6957357d4bb05 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/LearnToRankConfigUpdateTests.java @@ -16,10 +16,10 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.inference.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ltr.LearnToRankFeatureExtractorBuilder; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ltr.QueryExtractorBuilder; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ltr.QueryExtractorBuilderTests; +import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.utils.QueryProvider; import java.io.IOException; diff --git a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlRescorerIT.java b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlRescorerIT.java index 8dec8dcdb020e..a597a57be224b 100644 --- a/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlRescorerIT.java +++ b/x-pack/plugin/ml/qa/basic-multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlRescorerIT.java @@ -29,7 +29,7 @@ public class MlRescorerIT extends ESRestTestCase { @Before public void setupModelAndData() throws IOException { putRegressionModel(MODEL_ID, """ - { + { "description": "super complex model for tests", "input": {"field_names": ["cost", "product"]}, "inference_config": { @@ -37,7 +37,7 @@ public void setupModelAndData() throws IOException { "feature_extractors": [{ "query_extractor": { "feature_name": "two", - "query": {"script_score": {"query": {"match_all":{}}, "script": {"source": "return 2.0;"}}} + "query": { "script_score": { "query": { "match_all":{} }, "script": { "source": "return 2.0;" } } } } }] } @@ -174,11 +174,13 @@ public void setupModelAndData() throws IOException { } } } - }"""); + } + """); createIndex(INDEX_NAME, Settings.builder().put("number_of_shards", randomIntBetween(1, 3)).build(), """ - "properties":{ - "product":{"type": "keyword"}, - "cost":{"type": "integer"}}"""); + "properties": { + "product": {"type": "keyword"}, + "cost": {"type": "integer"} + }"""); indexData("{ \"product\": \"TV\", \"cost\": 300 }"); indexData("{ \"product\": \"TV\", \"cost\": 400 }"); indexData("{ \"product\": \"VCR\", \"cost\": 150 }"); @@ -190,18 +192,18 @@ public void setupModelAndData() throws IOException { @SuppressWarnings("unchecked") public void testLtrSimple() throws Exception { Response searchResponse = search(""" - { - "query": { - "match": { "product": { "query": "TV"}} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model" - } + { + "query": { + "match": { "product": { "query": "TV" } } + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model" + } } - - }"""); + } + """); Map response = responseAsMap(searchResponse); assertThat((List) XContentMapValues.extractValue("hits.hits._score", response), contains(20.0, 20.0)); @@ -211,23 +213,22 @@ public void testLtrSimple() throws Exception { public void testLtrSimpleDFS() throws Exception { Response searchResponse = searchDfs(""" { - "query": { - "match": { "product": { "query": "TV"}} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model", - "inference_config": { - "learn_to_rank": { - "feature_extractors":[ - {"query_extractor": {"feature_name": "product_bm25", "query": {"term": {"product": "TV"}}}} - ] - } - } - } + "query": { + "match": { "product": { "query": "TV" } } + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model", + "inference_config": { + "learn_to_rank": { + "feature_extractors":[ + { "query_extractor": { "feature_name": "product_bm25", "query": { "term": { "product": "TV" } } } } + ] + } + } } - + } }"""); Map response = responseAsMap(searchResponse); @@ -235,20 +236,19 @@ public void testLtrSimpleDFS() throws Exception { searchResponse = searchDfs(""" { - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model", - "inference_config": { - "learn_to_rank": { - "feature_extractors":[ - {"query_extractor": {"feature_name": "product_bm25", "query": {"term": {"product": "TV"}}}} - ] - } - } - } + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model", + "inference_config": { + "learn_to_rank": { + "feature_extractors":[ + { "query_extractor": { "feature_name": "product_bm25", "query": { "term": { "product": "TV" } } } } + ] + } + } } - + } }"""); response = responseAsMap(searchResponse); @@ -262,16 +262,16 @@ public void testLtrSimpleDFS() throws Exception { @SuppressWarnings("unchecked") public void testLtrSimpleEmpty() throws Exception { Response searchResponse = search(""" - { "query": { - "term": { "product": "computer"} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model" - } + { + "query": { + "term": { "product": "computer" } + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model" } - + } }"""); Map response = responseAsMap(searchResponse); @@ -281,16 +281,16 @@ public void testLtrSimpleEmpty() throws Exception { @SuppressWarnings("unchecked") public void testLtrEmptyDFS() throws Exception { Response searchResponse = searchDfs(""" - { "query": { - "match": { "product": { "query": "computer"}} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model" - } + { + "query": { + "match": { "product": { "query": "computer"} } + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model" } - + } }"""); Map response = responseAsMap(searchResponse); @@ -300,32 +300,32 @@ public void testLtrEmptyDFS() throws Exception { @SuppressWarnings("unchecked") public void testLtrCanMatch() throws Exception { Response searchResponse = searchCanMatch(""" - { "query": { - "match": { "product": { "query": "TV"}} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model" - } + { + "query": { + "match": { "product": { "query": "TV"}} + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model" } - + } }""", false); Map response = responseAsMap(searchResponse); assertThat(response.toString(), (List) XContentMapValues.extractValue("hits.hits._score", response), contains(20.0, 20.0)); searchResponse = searchCanMatch(""" - { "query": { - "match": { "product": { "query": "TV"}} - }, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "basic-ltr-model" - } + { + "query": { + "match": { "product": { "query": "TV"} } + }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "basic-ltr-model" } - + } }""", true); response = responseAsMap(searchResponse); diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceRescorerIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceRescorerIT.java deleted file mode 100644 index f51d2915d903e..0000000000000 --- a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/InferenceRescorerIT.java +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ml.integration; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.junit.Before; - -import java.io.IOException; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; - -public class InferenceRescorerIT extends InferenceTestCase { - - private static final String MODEL_ID = "ltr-model"; - private static final String INDEX_NAME = "store"; - - @Before - public void setupModelAndData() throws IOException { - putRegressionModel(MODEL_ID, """ - { - "description": "super complex model for tests", - "input": {"field_names": ["cost", "product"]}, - "inference_config": { - "learn_to_rank": { - "feature_extractors": [{ - "query_extractor": { - "feature_name": "two", - "query": {"script_score": {"query": {"match_all":{}}, "script": {"source": "return 2.0;"}}} - } - }] - } - }, - "definition": { - "preprocessors" : [{ - "one_hot_encoding": { - "field": "product", - "hot_map": { - "TV": "type_tv", - "VCR": "type_vcr", - "Laptop": "type_laptop" - } - } - }], - "trained_model": { - "ensemble": { - "feature_names": ["cost", "type_tv", "type_vcr", "type_laptop", "two", "product_bm25"], - "target_type": "regression", - "trained_models": [ - { - "tree": { - "feature_names": [ - "cost" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 400, - "decision_type": "lte", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 5.0 - }, - { - "node_index": 2, - "leaf_value": 2.0 - } - ], - "target_type": "regression" - } - }, - { - "tree": { - "feature_names": [ - "type_tv" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 1, - "decision_type": "lt", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 1.0 - }, - { - "node_index": 2, - "leaf_value": 12.0 - } - ], - "target_type": "regression" - } - }, - { - "tree": { - "feature_names": [ - "two" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 1, - "decision_type": "lt", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 1.0 - }, - { - "node_index": 2, - "leaf_value": 2.0 - } - ], - "target_type": "regression" - } - }, - { - "tree": { - "feature_names": [ - "product_bm25" - ], - "tree_structure": [ - { - "node_index": 0, - "split_feature": 0, - "split_gain": 12, - "threshold": 1, - "decision_type": "lt", - "default_left": true, - "left_child": 1, - "right_child": 2 - }, - { - "node_index": 1, - "leaf_value": 1.0 - }, - { - "node_index": 2, - "leaf_value": 4.0 - } - ], - "target_type": "regression" - } - } - ] - } - } - } - }"""); - createIndex(INDEX_NAME, Settings.EMPTY, """ - "properties":{ - "product":{"type": "keyword"}, - "cost":{"type": "integer"}}"""); - indexData("{ \"product\": \"TV\", \"cost\": 300}"); - indexData("{ \"product\": \"TV\", \"cost\": 400}"); - indexData("{ \"product\": \"TV\", \"cost\": 600}"); - indexData("{ \"product\": \"VCR\", \"cost\": 15}"); - indexData("{ \"product\": \"VCR\", \"cost\": 350}"); - indexData("{ \"product\": \"VCR\", \"cost\": 580}"); - indexData("{ \"product\": \"Laptop\", \"cost\": 100}"); - indexData("{ \"product\": \"Laptop\", \"cost\": 300}"); - indexData("{ \"product\": \"Laptop\", \"cost\": 500}"); - adminClient().performRequest(new Request("POST", INDEX_NAME + "/_refresh")); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") - public void testInferenceRescore() throws Exception { - Request request = new Request("GET", "store/_search?size=3&error_trace"); - request.setJsonEntity(""" - { - "rescore": { - "window_size": 10, - "inference": { "model_id": "ltr-model" } - } - }"""); - assertHitScores(client().performRequest(request), List.of(20.0, 20.0, 17.0)); - request.setJsonEntity(""" - { - "query": {"term": {"product": "Laptop"}}, - "rescore": { - "window_size": 10, - "inference": { - "model_id": "ltr-model", - "inference_config": { - "learn_to_rank": { - "feature_extractors":[{ - "query_extractor": {"feature_name": "product_bm25", "query": {"term": {"product": "Laptop"}}} - }] - } - } - } - } - }"""); - assertHitScores(client().performRequest(request), List.of(12.0, 12.0, 9.0)); - request.setJsonEntity(""" - { - "query": {"term": {"product": "Laptop"}}, - "rescore": { - "window_size": 10, - "inference": { "model_id": "ltr-model"} - } - }"""); - assertHitScores(client().performRequest(request), List.of(9.0, 9.0, 6.0)); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") - public void testInferenceRescoreSmallWindow() throws Exception { - Request request = new Request("GET", "store/_search?size=5"); - request.setJsonEntity(""" - { - "rescore": { - "window_size": 2, - "inference": { "model_id": "ltr-model" } - } - }"""); - assertHitScores(client().performRequest(request), List.of(20.0, 20.0, 1.0, 1.0, 1.0)); - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") - public void testInferenceRescorerWithChainedRescorers() throws IOException { - Request request = new Request("GET", "store/_search?size=5"); - request.setJsonEntity(""" - { - "rescore": [ - { - "window_size": 4, - "query": { "rescore_query":{ "script_score": {"query": {"match_all": {}}, "script": {"source": "return 4"}}}} - }, - { - "window_size": 3, - "inference": { "model_id": "ltr-model" } - }, - { - "window_size": 2, - "query": { "rescore_query": { "script_score": {"query": {"match_all": {}}, "script": {"source": "return 20"}}}} - } - ] - }"""); - assertHitScores(client().performRequest(request), List.of(40.0, 40.0, 17.0, 5.0, 1.0)); - } - - private void indexData(String data) throws IOException { - Request request = new Request("POST", INDEX_NAME + "/_doc"); - request.setJsonEntity(data); - client().performRequest(request); - } - - @SuppressWarnings("unchecked") - private static void assertHitScores(Response response, List expectedScores) throws IOException { - assertThat((List) XContentMapValues.extractValue("hits.hits._score", responseAsMap(response)), equalTo(expectedScores)); - } -} diff --git a/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearnToRankRescorerIT.java b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearnToRankRescorerIT.java new file mode 100644 index 0000000000000..caec6d2cd7659 --- /dev/null +++ b/x-pack/plugin/ml/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/LearnToRankRescorerIT.java @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class LearnToRankRescorerIT extends InferenceTestCase { + + private static final String MODEL_ID = "ltr-model"; + private static final String INDEX_NAME = "store"; + + @Before + public void setupModelAndData() throws IOException { + putRegressionModel(MODEL_ID, """ + { + "description": "super complex model for tests", + "input": {"field_names": ["cost", "product"]}, + "inference_config": { + "learn_to_rank": { + "feature_extractors": [{ + "query_extractor": { + "feature_name": "two", + "query": { "script_score": { "query": { "match_all":{} }, "script": { "source": "return 2.0;" } } } + } + }] + } + }, + "definition": { + "preprocessors" : [{ + "one_hot_encoding": { + "field": "product", + "hot_map": { + "TV": "type_tv", + "VCR": "type_vcr", + "Laptop": "type_laptop" + } + } + }], + "trained_model": { + "ensemble": { + "feature_names": ["cost", "type_tv", "type_vcr", "type_laptop", "two", "product_bm25"], + "target_type": "regression", + "trained_models": [ + { + "tree": { + "feature_names": [ + "cost" + ], + "tree_structure": [ + { + "node_index": 0, + "split_feature": 0, + "split_gain": 12, + "threshold": 400, + "decision_type": "lte", + "default_left": true, + "left_child": 1, + "right_child": 2 + }, + { + "node_index": 1, + "leaf_value": 5.0 + }, + { + "node_index": 2, + "leaf_value": 2.0 + } + ], + "target_type": "regression" + } + }, + { + "tree": { + "feature_names": [ + "type_tv" + ], + "tree_structure": [ + { + "node_index": 0, + "split_feature": 0, + "split_gain": 12, + "threshold": 1, + "decision_type": "lt", + "default_left": true, + "left_child": 1, + "right_child": 2 + }, + { + "node_index": 1, + "leaf_value": 1.0 + }, + { + "node_index": 2, + "leaf_value": 12.0 + } + ], + "target_type": "regression" + } + }, + { + "tree": { + "feature_names": [ + "two" + ], + "tree_structure": [ + { + "node_index": 0, + "split_feature": 0, + "split_gain": 12, + "threshold": 1, + "decision_type": "lt", + "default_left": true, + "left_child": 1, + "right_child": 2 + }, + { + "node_index": 1, + "leaf_value": 1.0 + }, + { + "node_index": 2, + "leaf_value": 2.0 + } + ], + "target_type": "regression" + } + }, + { + "tree": { + "feature_names": [ + "product_bm25" + ], + "tree_structure": [ + { + "node_index": 0, + "split_feature": 0, + "split_gain": 12, + "threshold": 1, + "decision_type": "lt", + "default_left": true, + "left_child": 1, + "right_child": 2 + }, + { + "node_index": 1, + "leaf_value": 1.0 + }, + { + "node_index": 2, + "leaf_value": 4.0 + } + ], + "target_type": "regression" + } + } + ] + } + } + } + }"""); + createIndex(INDEX_NAME, Settings.EMPTY, """ + "properties": { + "product": { "type": "keyword" }, + "cost": { "type": "integer" } + }"""); + indexData("{ \"product\": \"TV\", \"cost\": 300}"); + indexData("{ \"product\": \"TV\", \"cost\": 400}"); + indexData("{ \"product\": \"TV\", \"cost\": 600}"); + indexData("{ \"product\": \"VCR\", \"cost\": 15}"); + indexData("{ \"product\": \"VCR\", \"cost\": 350}"); + indexData("{ \"product\": \"VCR\", \"cost\": 580}"); + indexData("{ \"product\": \"Laptop\", \"cost\": 100}"); + indexData("{ \"product\": \"Laptop\", \"cost\": 300}"); + indexData("{ \"product\": \"Laptop\", \"cost\": 500}"); + adminClient().performRequest(new Request("POST", INDEX_NAME + "/_refresh")); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") + public void testInferenceRescore() throws Exception { + Request request = new Request("GET", "store/_search?size=3&error_trace"); + request.setJsonEntity(""" + { + "rescore": { + "window_size": 10, + "learn_to_rank": { "model_id": "ltr-model" } + } + }"""); + assertHitScores(client().performRequest(request), List.of(20.0, 20.0, 17.0)); + request.setJsonEntity(""" + { + "query": { "term": { "product": "Laptop" } }, + "rescore": { + "window_size": 10, + "learn_to_rank": { + "model_id": "ltr-model", + "inference_config": { + "learn_to_rank": { + "feature_extractors":[ + { "query_extractor": { "feature_name": "product_bm25", "query": { "term": { "product": "Laptop" } } } } + ] + } + } + } + } + }"""); + assertHitScores(client().performRequest(request), List.of(12.0, 12.0, 9.0)); + request.setJsonEntity(""" + { + "query": { "term": { "product": "Laptop" } }, + "rescore": { + "window_size": 10, + "learn_to_rank": { "model_id": "ltr-model"} + } + }"""); + assertHitScores(client().performRequest(request), List.of(9.0, 9.0, 6.0)); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") + public void testInferenceRescoreSmallWindow() throws Exception { + Request request = new Request("GET", "store/_search?size=5"); + request.setJsonEntity(""" + { + "rescore": { + "window_size": 2, + "learn_to_rank": { "model_id": "ltr-model" } + } + }"""); + assertHitScores(client().performRequest(request), List.of(20.0, 20.0, 1.0, 1.0, 1.0)); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98372") + public void testInferenceRescorerWithChainedRescorers() throws IOException { + Request request = new Request("GET", "store/_search?size=5"); + request.setJsonEntity(""" + { + "rescore": [ + { + "window_size": 4, + "query": { "rescore_query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 4"} } } } + }, + { + "window_size": 3, + "learn_to_rank": { "model_id": "ltr-model" } + }, + { + "window_size": 2, + "query": { "rescore_query": { "script_score": { "query": { "match_all": {} }, "script": { "source": "return 20" } } } } + } + ] + }"""); + assertHitScores(client().performRequest(request), List.of(40.0, 40.0, 17.0, 5.0, 1.0)); + } + + private void indexData(String data) throws IOException { + Request request = new Request("POST", INDEX_NAME + "/_doc"); + request.setJsonEntity(data); + client().performRequest(request); + } + + @SuppressWarnings("unchecked") + private static void assertHitScores(Response response, List expectedScores) throws IOException { + assertThat((List) XContentMapValues.extractValue("hits.hits._score", responseAsMap(response)), equalTo(expectedScores)); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index f4bce4906c0b0..538f02b3f9092 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -185,12 +185,12 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.MlEvaluationNamedXContentProvider; import org.elasticsearch.xpack.core.ml.dataframe.stats.AnalysisStatsNamedWriteablesProvider; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.inference.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.config.JobTaskState; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeTaskParams; import org.elasticsearch.xpack.core.ml.job.snapshot.upgrade.SnapshotUpgradeTaskState; +import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.template.TemplateUtils; import org.elasticsearch.xpack.ml.action.TransportAuditMlNotificationAction; @@ -322,13 +322,13 @@ import org.elasticsearch.xpack.ml.inference.deployment.DeploymentManager; import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; +import org.elasticsearch.xpack.ml.inference.ltr.InferenceRescorerFeature; +import org.elasticsearch.xpack.ml.inference.ltr.LearnToRankRescorerBuilder; import org.elasticsearch.xpack.ml.inference.modelsize.MlModelSizeNamedXContentProvider; import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider; import org.elasticsearch.xpack.ml.inference.pytorch.process.BlackHolePyTorchProcess; import org.elasticsearch.xpack.ml.inference.pytorch.process.NativePyTorchProcessFactory; import org.elasticsearch.xpack.ml.inference.pytorch.process.PyTorchProcessFactory; -import org.elasticsearch.xpack.ml.inference.rescorer.InferenceRescorerBuilder; -import org.elasticsearch.xpack.ml.inference.rescorer.InferenceRescorerFeature; import org.elasticsearch.xpack.ml.job.JobManager; import org.elasticsearch.xpack.ml.job.JobManagerHolder; import org.elasticsearch.xpack.ml.job.NodeLoadDetector; @@ -867,9 +867,9 @@ public List> getRescorers() { // Inference rescorer requires access to the model loading service return List.of( new RescorerSpec<>( - InferenceRescorerBuilder.NAME, - in -> new InferenceRescorerBuilder(in, modelLoadingService::get), - parser -> InferenceRescorerBuilder.fromXContent(parser, modelLoadingService::get) + LearnToRankRescorerBuilder.NAME, + in -> new LearnToRankRescorerBuilder(in, modelLoadingService::get), + parser -> LearnToRankRescorerBuilder.fromXContent(parser, modelLoadingService::get) ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FeatureExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FeatureExtractor.java similarity index 91% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FeatureExtractor.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FeatureExtractor.java index 36bf36ef99c52..90e462e3dc511 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FeatureExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FeatureExtractor.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.index.LeafReaderContext; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FieldValueFeatureExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FieldValueFeatureExtractor.java similarity index 97% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FieldValueFeatureExtractor.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FieldValueFeatureExtractor.java index 9f0ef84fc3575..5a2e3d29df949 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/FieldValueFeatureExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/FieldValueFeatureExtractor.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.index.mapper.MappedFieldType; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerFeature.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java similarity index 89% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerFeature.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java index 2b88faa3e4c14..8a26714c7c06b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerFeature.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/InferenceRescorerFeature.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.elasticsearch.common.util.FeatureFlag; @@ -14,7 +14,7 @@ * * Upon removal, ensure transport serialization is all corrected for future BWC. * - * See {@link InferenceRescorerBuilder} + * See {@link LearnToRankRescorerBuilder} */ public class InferenceRescorerFeature { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorer.java similarity index 91% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorer.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorer.java index df3e0756ea39a..dd1df7d8090d6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorer.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,17 +32,17 @@ import static java.util.stream.Collectors.toUnmodifiableSet; -public class InferenceRescorer implements Rescorer { +public class LearnToRankRescorer implements Rescorer { - public static final InferenceRescorer INSTANCE = new InferenceRescorer(); - private static final Logger logger = LogManager.getLogger(InferenceRescorer.class); + public static final LearnToRankRescorer INSTANCE = new LearnToRankRescorer(); + private static final Logger logger = LogManager.getLogger(LearnToRankRescorer.class); private static final Comparator SCORE_DOC_COMPARATOR = (o1, o2) -> { int cmp = Float.compare(o2.score, o1.score); return cmp == 0 ? Integer.compare(o1.doc, o2.doc) : cmp; }; - private InferenceRescorer() { + private LearnToRankRescorer() { } @@ -51,11 +51,11 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r if (topDocs.scoreDocs.length == 0) { return topDocs; } - InferenceRescorerContext ltrRescoreContext = (InferenceRescorerContext) rescoreContext; - if (ltrRescoreContext.inferenceDefinition == null) { + LearnToRankRescorerContext ltrRescoreContext = (LearnToRankRescorerContext) rescoreContext; + if (ltrRescoreContext.regressionModelDefinition == null) { throw new IllegalStateException("local model reference is null, missing rewriteAndFetch before rescore phase?"); } - LocalModel definition = ltrRescoreContext.inferenceDefinition; + LocalModel definition = ltrRescoreContext.regressionModelDefinition; // First take top slice of incoming docs, to be rescored: TopDocs topNFirstPass = topN(topDocs, rescoreContext.getWindowSize()); @@ -104,7 +104,7 @@ public TopDocs rescore(TopDocs topDocs, IndexSearcher searcher, RescoreContext r for (int i = 0; i < hitsToRescore.length; i++) { Map features = docFeatures.get(i); try { - InferenceResults results = definition.inferLtr(features, ltrRescoreContext.inferenceConfig); + InferenceResults results = definition.inferLtr(features, ltrRescoreContext.learnToRankConfig); if (results instanceof WarningInferenceResults warningInferenceResults) { logger.warn("Failure rescoring doc, warning returned [" + warningInferenceResults.getWarning() + "]"); } else if (results.predictedValue() instanceof Number prediction) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilder.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilder.java similarity index 85% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilder.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilder.java index 8d450a43722ed..39dbaa9f1c660 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilder.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilder.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.util.SetOnce; import org.elasticsearch.TransportVersion; @@ -41,9 +41,9 @@ import java.util.Optional; import java.util.function.Supplier; -public class InferenceRescorerBuilder extends RescorerBuilder { +public class LearnToRankRescorerBuilder extends RescorerBuilder { - public static final String NAME = "inference"; + public static final String NAME = "learn_to_rank"; private static final ParseField MODEL = new ParseField("model_id"); private static final ParseField INFERENCE_CONFIG = new ParseField("inference_config"); private static final ParseField INTERNAL_INFERENCE_CONFIG = new ParseField("_internal_inference_config"); @@ -62,7 +62,10 @@ public class InferenceRescorerBuilder extends RescorerBuilder modelLoadingServiceSupplier) { + public static LearnToRankRescorerBuilder fromXContent( + XContentParser parser, + Supplier modelLoadingServiceSupplier + ) { return PARSER.apply(parser, null).build(modelLoadingServiceSupplier); } @@ -75,7 +78,7 @@ public static InferenceRescorerBuilder fromXContent(XContentParser parser, Suppl private final Supplier inferenceConfigSupplier; private boolean rescoreOccurred; - public InferenceRescorerBuilder( + public LearnToRankRescorerBuilder( String modelId, LearnToRankConfigUpdate inferenceConfigUpdate, Supplier modelLoadingServiceSupplier @@ -89,7 +92,11 @@ public InferenceRescorerBuilder( this.inferenceConfig = null; } - InferenceRescorerBuilder(String modelId, LearnToRankConfig inferenceConfig, Supplier modelLoadingServiceSupplier) { + LearnToRankRescorerBuilder( + String modelId, + LearnToRankConfig inferenceConfig, + Supplier modelLoadingServiceSupplier + ) { this.modelId = Objects.requireNonNull(modelId); this.inferenceConfigUpdate = null; this.inferenceDefinition = null; @@ -99,7 +106,7 @@ public InferenceRescorerBuilder( this.inferenceConfig = Objects.requireNonNull(inferenceConfig); } - private InferenceRescorerBuilder( + private LearnToRankRescorerBuilder( String modelId, LearnToRankConfigUpdate update, Supplier modelLoadingServiceSupplier, @@ -114,7 +121,7 @@ private InferenceRescorerBuilder( this.inferenceConfig = null; } - private InferenceRescorerBuilder( + private LearnToRankRescorerBuilder( String modelId, LearnToRankConfig inferenceConfig, Supplier modelLoadingServiceSupplier, @@ -129,7 +136,7 @@ private InferenceRescorerBuilder( this.inferenceConfig = inferenceConfig; } - InferenceRescorerBuilder(String modelId, LearnToRankConfig inferenceConfig, LocalModel inferenceDefinition) { + LearnToRankRescorerBuilder(String modelId, LearnToRankConfig inferenceConfig, LocalModel inferenceDefinition) { this.modelId = modelId; this.inferenceConfigUpdate = null; this.inferenceDefinition = inferenceDefinition; @@ -139,7 +146,7 @@ private InferenceRescorerBuilder( this.inferenceConfig = inferenceConfig; } - public InferenceRescorerBuilder(StreamInput input, Supplier modelLoadingServiceSupplier) throws IOException { + public LearnToRankRescorerBuilder(StreamInput input, Supplier modelLoadingServiceSupplier) throws IOException { super(input); this.modelId = input.readString(); this.inferenceConfigUpdate = (LearnToRankConfigUpdate) input.readOptionalNamedWriteable(InferenceConfigUpdate.class); @@ -170,10 +177,10 @@ public TransportVersion getMinimalSupportedVersion() { * This can and be done on the coordinator as it not only validates if the stored model is of the appropriate type, it allows * any stored logic to rewrite on the coordinator level if possible. * @param ctx QueryRewriteContext - * @return rewritten InferenceRescorerBuilder or self if no changes + * @return rewritten LearnToRankRescorerBuilder or self if no changes * @throws IOException when rewrite fails */ - private RescorerBuilder doRewrite(QueryRewriteContext ctx) throws IOException { + private RescorerBuilder doRewrite(QueryRewriteContext ctx) throws IOException { // Awaiting fetch if (inferenceConfigSupplier != null && inferenceConfigSupplier.get() == null) { return this; @@ -183,7 +190,7 @@ private RescorerBuilder doRewrite(QueryRewriteContext if (rewrittenConfig == inferenceConfig) { return this; } - InferenceRescorerBuilder builder = new InferenceRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier); + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier); if (windowSize != null) { builder.windowSize(windowSize); } @@ -192,7 +199,7 @@ private RescorerBuilder doRewrite(QueryRewriteContext // We have requested for the stored config and fetch is completed, get the config and rewrite further if required if (inferenceConfigSupplier != null) { LearnToRankConfig rewrittenConfig = Rewriteable.rewrite(inferenceConfigSupplier.get(), ctx); - InferenceRescorerBuilder builder = new InferenceRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier); + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier); if (windowSize != null) { builder.windowSize(windowSize); } @@ -232,7 +239,7 @@ private RescorerBuilder doRewrite(QueryRewriteContext }, l::onFailure) ) ); - InferenceRescorerBuilder builder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder( modelId, inferenceConfigUpdate, modelLoadingServiceSupplier, @@ -249,7 +256,7 @@ private RescorerBuilder doRewrite(QueryRewriteContext * @param ctx Rewrite context * @return A rewritten rescorer with a model definition or a model definition supplier populated */ - private RescorerBuilder doDataNodeRewrite(QueryRewriteContext ctx) { + private RescorerBuilder doDataNodeRewrite(QueryRewriteContext ctx) { assert inferenceConfig != null; // We already have an inference definition, no need to do any rewriting if (inferenceDefinition != null) { @@ -261,7 +268,7 @@ private RescorerBuilder doDataNodeRewrite(QueryRewrite } if (inferenceDefinitionSupplier != null) { LocalModel inferenceDefinition = inferenceDefinitionSupplier.get(); - InferenceRescorerBuilder builder = new InferenceRescorerBuilder(modelId, inferenceConfig, inferenceDefinition); + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder(modelId, inferenceConfig, inferenceDefinition); if (windowSize() != null) { builder.windowSize(windowSize()); } @@ -275,7 +282,7 @@ private RescorerBuilder doDataNodeRewrite(QueryRewrite inferenceDefinitionSetOnce.set(lm); l.onResponse(null); }, l::onFailure))); - InferenceRescorerBuilder builder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder( modelId, inferenceConfig, modelLoadingServiceSupplier, @@ -293,7 +300,7 @@ private RescorerBuilder doDataNodeRewrite(QueryRewrite * @return A rewritten rescorer with a model definition or a model definition supplier populated * @throws IOException If fetching, parsing, or overall rewrite failures occur */ - private RescorerBuilder doSearchRewrite(QueryRewriteContext ctx) throws IOException { + private RescorerBuilder doSearchRewrite(QueryRewriteContext ctx) throws IOException { if (inferenceConfig == null) { return this; } @@ -301,9 +308,9 @@ private RescorerBuilder doSearchRewrite(QueryRewriteCo if (rewrittenConfig == inferenceConfig) { return this; } - InferenceRescorerBuilder builder = inferenceDefinition == null - ? new InferenceRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier) - : new InferenceRescorerBuilder(modelId, rewrittenConfig, inferenceDefinition); + LearnToRankRescorerBuilder builder = inferenceDefinition == null + ? new LearnToRankRescorerBuilder(modelId, rewrittenConfig, modelLoadingServiceSupplier) + : new LearnToRankRescorerBuilder(modelId, rewrittenConfig, inferenceDefinition); if (windowSize != null) { builder.windowSize(windowSize); } @@ -311,7 +318,7 @@ private RescorerBuilder doSearchRewrite(QueryRewriteCo } @Override - public RescorerBuilder rewrite(QueryRewriteContext ctx) throws IOException { + public RescorerBuilder rewrite(QueryRewriteContext ctx) throws IOException { if (ctx.convertToDataRewriteContext() != null) { return doDataNodeRewrite(ctx); } @@ -354,9 +361,9 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } @Override - protected InferenceRescorerContext innerBuildContext(int windowSize, SearchExecutionContext context) { + protected LearnToRankRescorerContext innerBuildContext(int windowSize, SearchExecutionContext context) { rescoreOccurred = true; - return new InferenceRescorerContext(windowSize, InferenceRescorer.INSTANCE, inferenceConfig, inferenceDefinition, context); + return new LearnToRankRescorerContext(windowSize, LearnToRankRescorer.INSTANCE, inferenceConfig, inferenceDefinition, context); } @Override @@ -364,7 +371,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (super.equals(o) == false) return false; - InferenceRescorerBuilder that = (InferenceRescorerBuilder) o; + LearnToRankRescorerBuilder that = (LearnToRankRescorerBuilder) o; return Objects.equals(modelId, that.modelId) && Objects.equals(inferenceDefinition, that.inferenceDefinition) && Objects.equals(inferenceConfigUpdate, that.inferenceConfigUpdate) @@ -437,12 +444,12 @@ void setInferenceConfig(InferenceConfig inferenceConfig) { ); } - InferenceRescorerBuilder build(Supplier modelLoadingServiceSupplier) { + LearnToRankRescorerBuilder build(Supplier modelLoadingServiceSupplier) { assert inferenceConfig == null || inferenceConfigUpdate == null; if (inferenceConfig != null) { - return new InferenceRescorerBuilder(modelId, inferenceConfig, modelLoadingServiceSupplier); + return new LearnToRankRescorerBuilder(modelId, inferenceConfig, modelLoadingServiceSupplier); } else { - return new InferenceRescorerBuilder(modelId, inferenceConfigUpdate, modelLoadingServiceSupplier); + return new LearnToRankRescorerBuilder(modelId, inferenceConfigUpdate, modelLoadingServiceSupplier); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerContext.java similarity index 74% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerContext.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerContext.java index fed3effdc06f6..844f96208cb35 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerContext.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -24,43 +24,43 @@ import java.util.ArrayList; import java.util.List; -public class InferenceRescorerContext extends RescoreContext { +public class LearnToRankRescorerContext extends RescoreContext { final SearchExecutionContext executionContext; - final LocalModel inferenceDefinition; - final LearnToRankConfig inferenceConfig; + final LocalModel regressionModelDefinition; + final LearnToRankConfig learnToRankConfig; /** * @param windowSize how many documents to rescore * @param rescorer The rescorer to apply - * @param inferenceConfig The inference config containing updated and rewritten parameters - * @param inferenceDefinition The local model inference definition, may be null during certain search phases. + * @param learnToRankConfig The inference config containing updated and rewritten parameters + * @param regressionModelDefinition The local model inference definition, may be null during certain search phases. * @param executionContext The local shard search context */ - public InferenceRescorerContext( + public LearnToRankRescorerContext( int windowSize, Rescorer rescorer, - LearnToRankConfig inferenceConfig, - LocalModel inferenceDefinition, + LearnToRankConfig learnToRankConfig, + LocalModel regressionModelDefinition, SearchExecutionContext executionContext ) { super(windowSize, rescorer); this.executionContext = executionContext; - this.inferenceDefinition = inferenceDefinition; - this.inferenceConfig = inferenceConfig; + this.regressionModelDefinition = regressionModelDefinition; + this.learnToRankConfig = learnToRankConfig; } List buildFeatureExtractors(IndexSearcher searcher) throws IOException { - assert this.inferenceDefinition != null && this.inferenceConfig != null; + assert this.regressionModelDefinition != null && this.learnToRankConfig != null; List featureExtractors = new ArrayList<>(); - if (this.inferenceDefinition.inputFields().isEmpty() == false) { + if (this.regressionModelDefinition.inputFields().isEmpty() == false) { featureExtractors.add( - new FieldValueFeatureExtractor(new ArrayList<>(this.inferenceDefinition.inputFields()), this.executionContext) + new FieldValueFeatureExtractor(new ArrayList<>(this.regressionModelDefinition.inputFields()), this.executionContext) ); } List weights = new ArrayList<>(); List queryFeatureNames = new ArrayList<>(); - for (LearnToRankFeatureExtractorBuilder featureExtractorBuilder : inferenceConfig.getFeatureExtractorBuilders()) { + for (LearnToRankFeatureExtractorBuilder featureExtractorBuilder : learnToRankConfig.getFeatureExtractorBuilders()) { if (featureExtractorBuilder instanceof QueryExtractorBuilder queryExtractorBuilder) { Query query = executionContext.toQuery(queryExtractorBuilder.query().getParsedQuery()).query(); Weight weight = searcher.rewrite(query).createWeight(searcher, ScoreMode.COMPLETE, 1f); @@ -77,11 +77,11 @@ List buildFeatureExtractors(IndexSearcher searcher) throws IOE @Override public List getParsedQueries() { - if (this.inferenceConfig == null) { + if (this.learnToRankConfig == null) { return List.of(); } List parsedQueries = new ArrayList<>(); - for (LearnToRankFeatureExtractorBuilder featureExtractorBuilder : inferenceConfig.getFeatureExtractorBuilders()) { + for (LearnToRankFeatureExtractorBuilder featureExtractorBuilder : learnToRankConfig.getFeatureExtractorBuilders()) { if (featureExtractorBuilder instanceof QueryExtractorBuilder queryExtractorBuilder) { parsedQueries.add(executionContext.toQuery(queryExtractorBuilder.query().getParsedQuery())); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java similarity index 98% rename from x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractor.java rename to x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java index 6920849e9af6d..bbc377a67ec0b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractor.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DisiPriorityQueue; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderRewriteTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderRewriteTests.java similarity index 81% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderRewriteTests.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderRewriteTests.java index b31c425ea1eb4..a847563e88240 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderRewriteTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderRewriteTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; @@ -67,7 +67,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class InferenceRescorerBuilderRewriteTests extends AbstractBuilderTestCase { +public class LearnToRankRescorerBuilderRewriteTests extends AbstractBuilderTestCase { private static final String GOOD_MODEL = "modelId"; private static final String BAD_MODEL = "badModel"; @@ -90,20 +90,23 @@ public class InferenceRescorerBuilderRewriteTests extends AbstractBuilderTestCas public void testMustRewrite() { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( GOOD_MODEL, LearnToRankConfigTests.randomLearnToRankConfig(), () -> testModelLoader ); SearchExecutionContext context = createSearchExecutionContext(); - InferenceRescorerContext inferenceRescorerContext = inferenceRescorerBuilder.innerBuildContext(randomIntBetween(1, 30), context); + LearnToRankRescorerContext learnToRankRescorerContext = learnToRankRescorerBuilder.innerBuildContext( + randomIntBetween(1, 30), + context + ); IllegalStateException e = expectThrows( IllegalStateException.class, - () -> inferenceRescorerContext.rescorer() + () -> learnToRankRescorerContext.rescorer() .rescore( new TopDocs(new TotalHits(10, TotalHits.Relation.EQUAL_TO), new ScoreDoc[10]), mock(IndexSearcher.class), - inferenceRescorerContext + learnToRankRescorerContext ) ); assertEquals("local model reference is null, missing rewriteAndFetch before rescore phase?", e.getMessage()); @@ -115,14 +118,14 @@ public void testRewriteOnCoordinator() throws IOException { 2, List.of(new QueryExtractorBuilder("all", QueryProvider.fromParsedQuery(QueryBuilders.matchAllQuery()))) ); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder(GOOD_MODEL, ltru, () -> testModelLoader); - inferenceRescorerBuilder.windowSize(4); + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder(GOOD_MODEL, ltru, () -> testModelLoader); + learnToRankRescorerBuilder.windowSize(4); CoordinatorRewriteContext context = createCoordinatorRewriteContext( new DateFieldMapper.DateFieldType("@timestamp"), randomIntBetween(0, 1_100_000), randomIntBetween(1_500_000, Integer.MAX_VALUE) ); - InferenceRescorerBuilder rewritten = rewriteAndFetch(inferenceRescorerBuilder, context); + LearnToRankRescorerBuilder rewritten = rewriteAndFetch(learnToRankRescorerBuilder, context); assertThat(rewritten.getInferenceConfig(), not(nullValue())); assertThat(rewritten.getInferenceConfig().getNumTopFeatureImportanceValues(), equalTo(2)); assertThat( @@ -143,7 +146,7 @@ public void testRewriteOnCoordinator() throws IOException { public void testRewriteOnCoordinatorWithBadModel() throws IOException { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( BAD_MODEL, randomBoolean() ? null : LearnToRankConfigUpdateTests.randomLearnToRankConfigUpdate(), () -> testModelLoader @@ -155,14 +158,14 @@ public void testRewriteOnCoordinatorWithBadModel() throws IOException { ); ElasticsearchStatusException ex = expectThrows( ElasticsearchStatusException.class, - () -> rewriteAndFetch(inferenceRescorerBuilder, context) + () -> rewriteAndFetch(learnToRankRescorerBuilder, context) ); assertThat(ex.status(), equalTo(RestStatus.BAD_REQUEST)); } public void testRewriteOnCoordinatorWithMissingModel() { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( "missing_model", randomBoolean() ? null : LearnToRankConfigUpdateTests.randomLearnToRankConfigUpdate(), () -> testModelLoader @@ -172,31 +175,35 @@ public void testRewriteOnCoordinatorWithMissingModel() { randomIntBetween(0, 1_100_000), randomIntBetween(1_500_000, Integer.MAX_VALUE) ); - expectThrows(ResourceNotFoundException.class, () -> rewriteAndFetch(inferenceRescorerBuilder, context)); + expectThrows(ResourceNotFoundException.class, () -> rewriteAndFetch(learnToRankRescorerBuilder, context)); } public void testSearchRewrite() throws IOException { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( GOOD_MODEL, LearnToRankConfigTests.randomLearnToRankConfig(), () -> testModelLoader ); QueryRewriteContext context = createSearchExecutionContext(); - InferenceRescorerBuilder rewritten = (InferenceRescorerBuilder) Rewriteable.rewrite(inferenceRescorerBuilder, context, true); + LearnToRankRescorerBuilder rewritten = (LearnToRankRescorerBuilder) Rewriteable.rewrite(learnToRankRescorerBuilder, context, true); assertThat(rewritten.modelLoadingServiceSupplier(), is(notNullValue())); - inferenceRescorerBuilder = new InferenceRescorerBuilder(GOOD_MODEL, LearnToRankConfigTests.randomLearnToRankConfig(), localModel()); + learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( + GOOD_MODEL, + LearnToRankConfigTests.randomLearnToRankConfig(), + localModel() + ); - rewritten = (InferenceRescorerBuilder) Rewriteable.rewrite(inferenceRescorerBuilder, context, true); + rewritten = (LearnToRankRescorerBuilder) Rewriteable.rewrite(learnToRankRescorerBuilder, context, true); assertThat(rewritten.modelLoadingServiceSupplier(), is(nullValue())); assertThat(rewritten.getInferenceDefinition(), is(notNullValue())); } - protected InferenceRescorerBuilder rewriteAndFetch(RescorerBuilder builder, QueryRewriteContext context) { - PlainActionFuture> future = new PlainActionFuture<>(); + protected LearnToRankRescorerBuilder rewriteAndFetch(RescorerBuilder builder, QueryRewriteContext context) { + PlainActionFuture> future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(builder, context, future); - return (InferenceRescorerBuilder) future.actionGet(); + return (LearnToRankRescorerBuilder) future.actionGet(); } @Override @@ -224,31 +231,33 @@ protected Object simulateMethod(Method method, Object[] args) { public void testRewriteOnShard() throws IOException { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( GOOD_MODEL, (LearnToRankConfig) GOOD_MODEL_CONFIG.getInferenceConfig(), () -> testModelLoader ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(); - InferenceRescorerBuilder rewritten = (InferenceRescorerBuilder) inferenceRescorerBuilder.rewrite(createSearchExecutionContext()); - assertSame(inferenceRescorerBuilder, rewritten); + LearnToRankRescorerBuilder rewritten = (LearnToRankRescorerBuilder) learnToRankRescorerBuilder.rewrite( + createSearchExecutionContext() + ); + assertSame(learnToRankRescorerBuilder, rewritten); assertFalse(searchExecutionContext.hasAsyncActions()); } public void testRewriteAndFetchOnDataNode() throws IOException { TestModelLoader testModelLoader = new TestModelLoader(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( GOOD_MODEL, (LearnToRankConfig) GOOD_MODEL_CONFIG.getInferenceConfig(), () -> testModelLoader ); boolean setWindowSize = randomBoolean(); if (setWindowSize) { - inferenceRescorerBuilder.windowSize(42); + learnToRankRescorerBuilder.windowSize(42); } DataRewriteContext rewriteContext = dataRewriteContext(); - InferenceRescorerBuilder rewritten = (InferenceRescorerBuilder) inferenceRescorerBuilder.rewrite(rewriteContext); - assertNotSame(inferenceRescorerBuilder, rewritten); + LearnToRankRescorerBuilder rewritten = (LearnToRankRescorerBuilder) learnToRankRescorerBuilder.rewrite(rewriteContext); + assertNotSame(learnToRankRescorerBuilder, rewritten); assertTrue(rewriteContext.hasAsyncActions()); if (setWindowSize) { assertThat(rewritten.windowSize(), equalTo(42)); @@ -260,12 +269,12 @@ public void testBuildContext() throws Exception { List inputFields = List.of(DOUBLE_FIELD_NAME, INT_FIELD_NAME); when(localModel.inputFields()).thenReturn(inputFields); SearchExecutionContext context = createSearchExecutionContext(); - InferenceRescorerBuilder inferenceRescorerBuilder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder learnToRankRescorerBuilder = new LearnToRankRescorerBuilder( GOOD_MODEL, (LearnToRankConfig) GOOD_MODEL_CONFIG.getInferenceConfig(), localModel ); - InferenceRescorerContext rescoreContext = inferenceRescorerBuilder.innerBuildContext(20, context); + LearnToRankRescorerContext rescoreContext = learnToRankRescorerBuilder.innerBuildContext(20, context); assertNotNull(rescoreContext); assertThat(rescoreContext.getWindowSize(), equalTo(20)); List featureExtractors = rescoreContext.buildFeatureExtractors(context.searcher()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderSerializationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderSerializationTests.java similarity index 80% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderSerializationTests.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderSerializationTests.java index f85d24770f70e..229157211848c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/InferenceRescorerBuilderSerializationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearnToRankRescorerBuilderSerializationTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.ParsingException; @@ -17,11 +17,11 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; -import org.elasticsearch.xpack.core.ml.inference.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigUpdateTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.LearnToRankConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.LearnToRankConfigUpdateTests; +import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService; import java.io.IOException; @@ -32,12 +32,12 @@ import static org.elasticsearch.search.rank.RankBuilder.WINDOW_SIZE_FIELD; -public class InferenceRescorerBuilderSerializationTests extends AbstractBWCSerializationTestCase { +public class LearnToRankRescorerBuilderSerializationTests extends AbstractBWCSerializationTestCase { @Override - protected InferenceRescorerBuilder doParseInstance(XContentParser parser) throws IOException { + protected LearnToRankRescorerBuilder doParseInstance(XContentParser parser) throws IOException { String fieldName = null; - InferenceRescorerBuilder rescorer = null; + LearnToRankRescorerBuilder rescorer = null; Integer windowSize = null; XContentParser.Token token = parser.nextToken(); assert token == XContentParser.Token.START_OBJECT; @@ -51,7 +51,7 @@ protected InferenceRescorerBuilder doParseInstance(XContentParser parser) throws throw new ParsingException(parser.getTokenLocation(), "rescore doesn't support [" + fieldName + "]"); } } else if (token == XContentParser.Token.START_OBJECT) { - rescorer = InferenceRescorerBuilder.fromXContent(parser, null); + rescorer = LearnToRankRescorerBuilder.fromXContent(parser, null); } else { throw new ParsingException(parser.getTokenLocation(), "unexpected token [" + token + "] after [" + fieldName + "]"); } @@ -66,19 +66,19 @@ protected InferenceRescorerBuilder doParseInstance(XContentParser parser) throws } @Override - protected Writeable.Reader instanceReader() { - return in -> new InferenceRescorerBuilder(in, null); + protected Writeable.Reader instanceReader() { + return in -> new LearnToRankRescorerBuilder(in, null); } @Override - protected InferenceRescorerBuilder createTestInstance() { - InferenceRescorerBuilder builder = randomBoolean() - ? new InferenceRescorerBuilder( + protected LearnToRankRescorerBuilder createTestInstance() { + LearnToRankRescorerBuilder builder = randomBoolean() + ? new LearnToRankRescorerBuilder( randomAlphaOfLength(10), randomBoolean() ? null : LearnToRankConfigUpdateTests.randomLearnToRankConfigUpdate(), null ) - : new InferenceRescorerBuilder( + : new LearnToRankRescorerBuilder( randomAlphaOfLength(10), LearnToRankConfigTests.randomLearnToRankConfig(), (Supplier) null @@ -90,11 +90,11 @@ protected InferenceRescorerBuilder createTestInstance() { } @Override - protected InferenceRescorerBuilder mutateInstance(InferenceRescorerBuilder instance) throws IOException { + protected LearnToRankRescorerBuilder mutateInstance(LearnToRankRescorerBuilder instance) throws IOException { int i = randomInt(3); return switch (i) { case 0 -> { - InferenceRescorerBuilder builder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder( randomValueOtherThan(instance.getModelId(), () -> randomAlphaOfLength(10)), instance.getInferenceConfigUpdate(), null @@ -104,11 +104,11 @@ protected InferenceRescorerBuilder mutateInstance(InferenceRescorerBuilder insta } yield builder; } - case 1 -> new InferenceRescorerBuilder(instance.getModelId(), instance.getInferenceConfigUpdate(), null).windowSize( + case 1 -> new LearnToRankRescorerBuilder(instance.getModelId(), instance.getInferenceConfigUpdate(), null).windowSize( randomValueOtherThan(instance.windowSize(), () -> randomIntBetween(1, 10000)) ); case 2 -> { - InferenceRescorerBuilder builder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder( instance.getModelId(), randomValueOtherThan(instance.getInferenceConfigUpdate(), LearnToRankConfigUpdateTests::randomLearnToRankConfigUpdate), null @@ -119,7 +119,7 @@ protected InferenceRescorerBuilder mutateInstance(InferenceRescorerBuilder insta yield builder; } case 3 -> { - InferenceRescorerBuilder builder = new InferenceRescorerBuilder( + LearnToRankRescorerBuilder builder = new LearnToRankRescorerBuilder( instance.getModelId(), randomValueOtherThan(instance.getInferenceConfig(), LearnToRankConfigTests::randomLearnToRankConfig), (Supplier) null @@ -134,12 +134,12 @@ protected InferenceRescorerBuilder mutateInstance(InferenceRescorerBuilder insta } @Override - protected InferenceRescorerBuilder mutateInstanceForVersion(InferenceRescorerBuilder instance, TransportVersion version) { + protected LearnToRankRescorerBuilder mutateInstanceForVersion(LearnToRankRescorerBuilder instance, TransportVersion version) { return instance; } public void testIncorrectInferenceConfigUpdateType() { - InferenceRescorerBuilder.Builder builder = new InferenceRescorerBuilder.Builder(); + LearnToRankRescorerBuilder.Builder builder = new LearnToRankRescorerBuilder.Builder(); expectThrows( IllegalArgumentException.class, () -> builder.setInferenceConfigUpdate(ClassificationConfigUpdateTests.randomClassificationConfigUpdate()) @@ -149,7 +149,7 @@ public void testIncorrectInferenceConfigUpdateType() { } public void testIncorrectInferenceConfigType() { - InferenceRescorerBuilder.Builder builder = new InferenceRescorerBuilder.Builder(); + LearnToRankRescorerBuilder.Builder builder = new LearnToRankRescorerBuilder.Builder(); expectThrows( IllegalArgumentException.class, () -> builder.setInferenceConfig(ClassificationConfigTests.randomClassificationConfig()) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java similarity index 99% rename from x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractorTests.java rename to x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java index bae0a6cf1c028..3878ce5dab087 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/rescorer/QueryFeatureExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/QueryFeatureExtractorTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.ml.inference.rescorer; +package org.elasticsearch.xpack.ml.inference.ltr; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/inference_rescore.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/inference_rescore.yml index 824999b3b3008..a0ae4b7c44316 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/inference_rescore.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/inference_rescore.yml @@ -146,7 +146,7 @@ setup: { "rescore": { "window_size": 10, - "inference": { "model_id": "ltr-model" } + "learn_to_rank": { "model_id": "ltr-model" } } } - match: { hits.hits.0._score: 17.0 } @@ -162,7 +162,7 @@ setup: "query": {"term": {"product": "Laptop"}}, "rescore": { "window_size": 10, - "inference": { "model_id": "ltr-model" } + "learn_to_rank": { "model_id": "ltr-model" } } } - match: { hits.hits.0._score: 6.0 } @@ -182,7 +182,7 @@ setup: { "rescore": { "window_size": 2, - "inference": { "model_id": "ltr-model" } + "learn_to_rank": { "model_id": "ltr-model" } } } - match: { hits.hits.0._score: 17.0 } @@ -209,7 +209,7 @@ setup: }, { "window_size": 3, - "inference": { "model_id": "ltr-model" } + "learn_to_rank": { "model_id": "ltr-model" } }, { "window_size": 2, @@ -232,7 +232,7 @@ setup: { "rescore": { "window_size": 10, - "inference": { "model_id": "ltr-missing" } + "learn_to_rank": { "model_id": "ltr-missing" } } } --- @@ -245,7 +245,7 @@ setup: "query": {"term": {"product": "Speaker"}}, "rescore": { "window_size": 10, - "inference": { "model_id": "ltr-model" } + "learn_to_rank": { "model_id": "ltr-model" } } } - length: { hits.hits: 0 } From 2410a083b5aee2a6c309b9326ea4baa2c844cecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 24 Nov 2023 10:04:18 +0100 Subject: [PATCH 18/35] Moving NodeInfo internal version storage to String (#100746) NodeInfo version as string. Moving from Version to Build.version(), which means also non-semantic versions. Particular care was needed for ESRestTestCase Version parsing, to make it work in a serverless environment, removing Version whenever possible and adding proper fallbacks when needed. --- .../org/elasticsearch/TransportVersions.java | 2 +- .../admin/cluster/node/info/NodeInfo.java | 33 ++++++++++++------- .../org/elasticsearch/node/NodeService.java | 3 +- .../cluster/node/info/NodeInfoTests.java | 3 +- .../remote/RemoteClusterNodesActionTests.java | 6 ++-- .../cluster/stats/ClusterStatsNodesTests.java | 3 +- .../ingest/ReservedPipelineActionTests.java | 3 +- .../TransportVersionsFixupListenerTests.java | 2 +- .../nodesinfo/NodeInfoStreamingTests.java | 2 +- .../action/cat/RestPluginsActionTests.java | 4 +-- .../AutoscalingNodesInfoServiceTests.java | 3 +- .../TransportNodeEnrollmentActionTests.java | 4 +-- ...InternalEnrollmentTokenGeneratorTests.java | 5 +-- 13 files changed, 40 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index a1675b3e2b908..13c3ea447aa3f 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -177,7 +177,7 @@ static TransportVersion def(int id) { public static final TransportVersion TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED = def(8_544_00_0); public static final TransportVersion GRANT_API_KEY_CLIENT_AUTHENTICATION_ADDED = def(8_545_00_0); public static final TransportVersion PIT_WITH_INDEX_FILTER = def(8_546_00_0); - + public static final TransportVersion NODE_INFO_VERSION_AS_STRING = def(8_547_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java index 6c10a6a07cba6..6e700ca4aecc3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java @@ -40,7 +40,7 @@ */ public class NodeInfo extends BaseNodeResponse { - private final Version version; + private final String version; private final TransportVersion transportVersion; private final IndexVersion indexVersion; private final Map componentVersions; @@ -61,16 +61,23 @@ public class NodeInfo extends BaseNodeResponse { public NodeInfo(StreamInput in) throws IOException { super(in); - version = Version.readVersion(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { + if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_VERSION_AS_STRING)) { + version = in.readString(); transportVersion = TransportVersion.readVersion(in); - } else { - transportVersion = TransportVersion.fromId(version.id); - } - if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_INDEX_VERSION_ADDED)) { indexVersion = IndexVersion.readVersion(in); } else { - indexVersion = IndexVersion.fromId(version.id); + Version legacyVersion = Version.readVersion(in); + version = legacyVersion.toString(); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { + transportVersion = TransportVersion.readVersion(in); + } else { + transportVersion = TransportVersion.fromId(legacyVersion.id); + } + if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_INDEX_VERSION_ADDED)) { + indexVersion = IndexVersion.readVersion(in); + } else { + indexVersion = IndexVersion.fromId(legacyVersion.id); + } } if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_COMPONENT_VERSIONS_ADDED)) { componentVersions = in.readImmutableMap(StreamInput::readString, StreamInput::readVInt); @@ -105,7 +112,7 @@ public NodeInfo(StreamInput in) throws IOException { } public NodeInfo( - Version version, + String version, TransportVersion transportVersion, IndexVersion indexVersion, Map componentVersions, @@ -156,7 +163,7 @@ public String getHostname() { * The current ES version */ public String getVersion() { - return version.toString(); + return version; } /** @@ -227,7 +234,11 @@ private void addInfoIfNonNull(Class clazz, @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - Version.writeVersion(version, out); + if (out.getTransportVersion().onOrAfter(TransportVersions.NODE_INFO_VERSION_AS_STRING)) { + out.writeString(version); + } else { + Version.writeVersion(Version.fromString(version), out); + } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { TransportVersion.writeVersion(transportVersion, out); } diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index 9b6e55383eea0..7f77bc2ab0e36 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -10,7 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.ComponentVersionNumber; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; @@ -118,7 +117,7 @@ public NodeInfo info( boolean indices ) { return new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), findComponentVersions(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java index 3e0460565dd61..8618ae516768f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfoTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -39,7 +38,7 @@ public class NodeInfoTests extends ESTestCase { */ public void testGetInfo() { NodeInfo nodeInfo = new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java index 91af3383f0670..a56eb50290ed9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/remote/RemoteClusterNodesActionTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.action.admin.cluster.remote; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -76,7 +76,7 @@ public void testDoExecuteForRemoteServerNodes() { } nodeInfos.add( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), @@ -154,7 +154,7 @@ public void testDoExecuteForRemoteNodes() { expectedRemoteNodes.add(node); nodeInfos.add( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java index f2b1cb28ce8b6..0ee3b244ecf45 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodesTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodeStatsTests; @@ -322,7 +321,7 @@ private static NodeInfo createNodeInfo(String nodeId, String transportType, Stri settings.put(randomFrom(NetworkModule.HTTP_TYPE_KEY, NetworkModule.HTTP_TYPE_DEFAULT_KEY), httpType); } return new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java index e02f5f66e6c80..b3ca89c9dc0bd 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/ReservedPipelineActionTests.java @@ -10,7 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.client.internal.Client; @@ -100,7 +99,7 @@ public void setup() { DiscoveryNode discoveryNode = DiscoveryNodeUtils.builder("_node_id").roles(emptySet()).build(); NodeInfo nodeInfo = new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java b/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java index ce78d603ea8d2..d89183d93205f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/TransportVersionsFixupListenerTests.java @@ -83,7 +83,7 @@ private static NodesInfoResponse getResponse(Map respo .stream() .map( e -> new NodeInfo( - null, + "", e.getValue(), null, null, diff --git a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java index 6ea93cc9c5940..d4668c7824ae4 100644 --- a/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java +++ b/server/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java @@ -239,7 +239,7 @@ private static NodeInfo createNodeInfo() { indexingBuffer = ByteSizeValue.ofBytes(random().nextLong() & ((1L << 40) - 1)); } return new NodeInfo( - VersionUtils.randomVersion(random()), + randomAlphaOfLengthBetween(6, 32), TransportVersionUtils.randomVersion(random()), IndexVersionUtils.randomVersion(random()), componentVersions, diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java index e58127e9a6d9b..c5256e0056873 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestPluginsActionTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.rest.action.cat; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules; @@ -64,7 +64,7 @@ private Table buildTable(List pluginDescriptor) { for (int i = 0; i < 3; i++) { nodeInfos.add( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java index c6fb0613b3e8e..5c47f5a9dc6a4 100644 --- a/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java +++ b/x-pack/plugin/autoscaling/src/test/java/org/elasticsearch/xpack/autoscaling/capacity/nodeinfo/AutoscalingNodesInfoServiceTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -449,7 +448,7 @@ private static NodeStats statsForNode(DiscoveryNode node, long memory) { private static org.elasticsearch.action.admin.cluster.node.info.NodeInfo infoForNode(DiscoveryNode node, int processors) { OsInfo osInfo = new OsInfo(randomLong(), processors, Processors.of((double) processors), null, null, null, null); return new org.elasticsearch.action.admin.cluster.node.info.NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java index 5c02f29ab7742..26bb64eb0b073 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/enrollment/TransportNodeEnrollmentActionTests.java @@ -7,8 +7,8 @@ package org.elasticsearch.xpack.security.action.enrollment; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; @@ -102,7 +102,7 @@ public void testDoExecute() throws Exception { DiscoveryNode n = node(i); nodeInfos.add( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java index 0b4f349422e8b..2abbb6a610170 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGeneratorTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.enrollment; +import org.elasticsearch.Build; import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -231,7 +232,7 @@ public Answer answerNullHttpInfo(InvocationOnMock invocationO new ClusterName("cluster_name"), List.of( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), @@ -266,7 +267,7 @@ private Answer answerWithInfo(InvocationOnMock invocationOnMo new ClusterName("cluster_name"), List.of( new NodeInfo( - Version.CURRENT, + Build.current().version(), TransportVersion.current(), IndexVersion.current(), Map.of(), From d127cb40922dfd7eaff46d24422e8d544cbc8867 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 24 Nov 2023 09:12:00 +0000 Subject: [PATCH 19/35] Move and consolidate more node construction logic (#102302) --- .../elasticsearch/node/NodeConstruction.java | 249 ++++++++++-------- 1 file changed, 144 insertions(+), 105 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 18f40af790f15..764d1b1366153 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -479,6 +479,7 @@ private SettingsModule validateSettings(Settings envSettings, Settings settings, additionalSettings, pluginsService.flatMap(Plugin::getSettingsFilter).toList() ); + modules.add(settingsModule); // creating `NodeEnvironment` breaks the ability to rollback to 7.x on an 8.0 upgrade (`upgradeLegacyNodeFolders`) so do this // after settings validation. @@ -604,16 +605,8 @@ private void construct( ).collect(Collectors.toSet()); final TaskManager taskManager = new TaskManager(settings, threadPool, taskHeaders, tracer); - final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool, taskManager); + ClusterService clusterService = createClusterService(settingsModule, threadPool, taskManager); clusterService.addStateApplier(scriptService); - resourcesToClose.add(clusterService); - - final Set> consistentSettings = settingsModule.getConsistentSettings(); - if (consistentSettings.isEmpty() == false) { - clusterService.addLocalNodeMasterListener( - new ConsistentSettingsService(settings, clusterService, consistentSettings).newHashPublisher() - ); - } Supplier documentParsingObserverSupplier = getDocumentParsingObserverSupplier(); @@ -628,32 +621,31 @@ private void construct( IngestService.createGrokThreadWatchdog(environment, threadPool), documentParsingObserverSupplier ); - final SetOnce repositoriesServiceReference = new SetOnce<>(); - final ClusterInfoService clusterInfoService = serviceProvider.newClusterInfoService( - pluginsService, - settings, - clusterService, - threadPool, - client - ); SystemIndices systemIndices = createSystemIndices(settings); - final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool); final FsHealthService fsHealthService = new FsHealthService( settings, clusterService.getClusterSettings(), threadPool, nodeEnvironment ); + + final SetOnce repositoriesServiceReference = new SetOnce<>(); final SetOnce rerouteServiceReference = new SetOnce<>(); + final ClusterInfoService clusterInfoService = serviceProvider.newClusterInfoService( + pluginsService, + settings, + clusterService, + threadPool, + client + ); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, clusterService, repositoriesServiceReference::get, rerouteServiceReference::get ); - final WriteLoadForecaster writeLoadForecaster = getWriteLoadForecaster(threadPool, settings, clusterService.getClusterSettings()); final ClusterModule clusterModule = new ClusterModule( settings, clusterService, @@ -662,7 +654,7 @@ private void construct( snapshotsInfoService, threadPool, systemIndices, - writeLoadForecaster, + getWriteLoadForecaster(threadPool, settings, clusterService.getClusterSettings()), telemetryProvider ); modules.add(clusterModule); @@ -679,16 +671,10 @@ private void construct( TransportVersion.current(), systemIndices.getMappingsVersions() ); + modules.add(loadPersistedClusterStateService(clusterService.getClusterSettings(), threadPool, compatibilityVersions)); PageCacheRecycler pageCacheRecycler = serviceProvider.newPageCacheRecycler(pluginsService, settings); BigArrays bigArrays = serviceProvider.newBigArrays(pluginsService, pageCacheRecycler, circuitBreakerService); - modules.add(settingsModule); final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry); - final PersistedClusterStateService persistedClusterStateService = newPersistedClusterStateService( - xContentRegistry, - clusterService.getClusterSettings(), - threadPool, - compatibilityVersions - ); FeatureService featureService = new FeatureService(pluginsService.loadServiceProviders(FeatureSpecification.class)); @@ -801,27 +787,6 @@ record PluginServiceInstances( Collection pluginComponents = pluginsService.flatMap(p -> p.createComponents(pluginServices)).toList(); - List> reservedStateHandlers = new ArrayList<>(); - - // add all reserved state handlers from server - reservedStateHandlers.add(new ReservedClusterSettingsAction(settingsModule.getClusterSettings())); - - var templateService = new MetadataIndexTemplateService( - clusterService, - metadataCreateIndexService, - indicesService, - settingsModule.getIndexScopedSettings(), - xContentRegistry, - systemIndices, - indexSettingProviders - ); - - reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); - - // add all reserved state handlers from plugins - pluginsService.loadServiceProviders(ReservedClusterStateHandlerProvider.class) - .forEach(h -> reservedStateHandlers.addAll(h.handlers())); - var terminationHandlers = pluginsService.loadServiceProviders(TerminationHandlerProvider.class) .stream() .map(TerminationHandlerProvider::handler); @@ -841,7 +806,14 @@ record PluginServiceInstances( systemIndices, tracer, clusterService, - reservedStateHandlers, + buildReservedStateHandlers( + settingsModule, + clusterService, + indicesService, + systemIndices, + indexSettingProviders, + metadataCreateIndexService + ), pluginsService.loadSingletonServiceProvider(RestExtension.class, RestExtension::allowAll) ); modules.add(actionModule); @@ -990,7 +962,7 @@ record PluginServiceInstances( nodeService = new NodeService( settings, threadPool, - monitorService, + new MonitorService(settings, nodeEnvironment, threadPool), discoveryModule.getCoordinator(), transportService, indicesService, @@ -1022,44 +994,18 @@ record PluginServiceInstances( tracer ); - final PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); - final SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( - client, - clusterService, - systemIndices, - metadataUpdateSettingsService, - metadataCreateIndexService, - settingsModule.getIndexScopedSettings() - ); - final HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create( - clusterService, - persistentTasksService, - featureService, - settings, - clusterService.getClusterSettings() - ); - final Stream> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor); - final Stream> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) - .map( - p -> p.getPersistentTasksExecutor( - clusterService, - threadPool, - client, - settingsModule, - clusterModule.getIndexNameExpressionResolver() - ) + modules.add( + loadPersistentTasksService( + settingsModule, + clusterService, + threadPool, + systemIndices, + featureService, + clusterModule.getIndexNameExpressionResolver(), + metadataUpdateSettingsService, + metadataCreateIndexService ) - .flatMap(List::stream); - final PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry( - Stream.concat(pluginTaskExecutors, builtinTaskExecutors).toList() - ); - final PersistentTasksClusterService persistentTasksClusterService = new PersistentTasksClusterService( - settings, - registry, - clusterService, - threadPool ); - resourcesToClose.add(persistentTasksClusterService); PluginShutdownService pluginShutdownService = new PluginShutdownService( pluginsService.filterPlugins(ShutdownAwarePlugin.class).toList() @@ -1110,7 +1056,6 @@ record PluginServiceInstances( b.bind(AggregationUsageService.class).toInstance(searchModule.getValuesSourceRegistry().getUsageService()); b.bind(MetadataUpgrader.class).toInstance(metadataUpgrader); b.bind(MetaStateService.class).toInstance(metaStateService); - b.bind(PersistedClusterStateService.class).toInstance(persistedClusterStateService); b.bind(IndicesService.class).toInstance(indicesService); b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService); b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService); @@ -1130,9 +1075,6 @@ record PluginServiceInstances( b.bind(Coordinator.class).toInstance(discoveryModule.getCoordinator()); b.bind(Reconfigurator.class).toInstance(discoveryModule.getReconfigurator()); b.bind(HttpServerTransport.class).toInstance(httpServerTransport); - b.bind(PersistentTasksService.class).toInstance(persistentTasksService); - b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService); - b.bind(PersistentTasksExecutorRegistry.class).toInstance(registry); b.bind(RepositoriesService.class).toInstance(repositoryService); b.bind(SnapshotsService.class).toInstance(snapshotsService); b.bind(SnapshotShardsService.class).toInstance(snapshotShardsService); @@ -1143,10 +1085,8 @@ record PluginServiceInstances( b.bind(PluginShutdownService.class).toInstance(pluginShutdownService); b.bind(IndexSettingProviders.class).toInstance(indexSettingProviders); b.bind(DesiredNodesSettingsValidator.class).toInstance(new DesiredNodesSettingsValidator()); - b.bind(HealthNodeTaskExecutor.class).toInstance(healthNodeTaskExecutor); b.bind(Tracer.class).toInstance(tracer); b.bind(FileSettingsService.class).toInstance(fileSettingsService); - b.bind(WriteLoadForecaster.class).toInstance(writeLoadForecaster); b.bind(CompatibilityVersions.class).toInstance(compatibilityVersions); }); @@ -1162,6 +1102,24 @@ record PluginServiceInstances( postInjection(clusterModule, actionModule, clusterService, transportService, featureService); } + private ClusterService createClusterService(SettingsModule settingsModule, ThreadPool threadPool, TaskManager taskManager) { + ClusterService clusterService = new ClusterService( + settingsModule.getSettings(), + settingsModule.getClusterSettings(), + threadPool, + taskManager + ); + resourcesToClose.add(clusterService); + + Set> consistentSettings = settingsModule.getConsistentSettings(); + if (consistentSettings.isEmpty() == false) { + clusterService.addLocalNodeMasterListener( + new ConsistentSettingsService(settingsModule.getSettings(), clusterService, consistentSettings).newHashPublisher() + ); + } + return clusterService; + } + private UsageService createUsageService() { UsageService usageService = new UsageService(); modules.bindToInstance(UsageService.class, usageService); @@ -1373,11 +1331,14 @@ private WriteLoadForecaster getWriteLoadForecaster(ThreadPool threadPool, Settin var writeLoadForecasters = pluginsService.filterPlugins(ClusterPlugin.class) .flatMap(clusterPlugin -> clusterPlugin.createWriteLoadForecasters(threadPool, settings, clusterSettings).stream()); - return getSinglePlugin(writeLoadForecasters, WriteLoadForecaster.class).orElse(WriteLoadForecaster.DEFAULT); + WriteLoadForecaster forecaster = getSinglePlugin(writeLoadForecasters, WriteLoadForecaster.class).orElse( + WriteLoadForecaster.DEFAULT + ); + modules.bindToInstance(WriteLoadForecaster.class, forecaster); + return forecaster; } - private PersistedClusterStateService newPersistedClusterStateService( - NamedXContentRegistry xContentRegistry, + private Module loadPersistedClusterStateService( ClusterSettings clusterSettings, ThreadPool threadPool, CompatibilityVersions compatibilityVersions @@ -1386,18 +1347,96 @@ private PersistedClusterStateService newPersistedClusterStateService( .map(ClusterCoordinationPlugin::getPersistedClusterStateServiceFactory) .flatMap(Optional::stream); - return getSinglePlugin(persistedClusterStateServiceFactories, ClusterCoordinationPlugin.PersistedClusterStateServiceFactory.class) - .map( - f -> f.newPersistedClusterStateService( - nodeEnvironment, - xContentRegistry, - clusterSettings, - threadPool, - compatibilityVersions - ) - ) + var service = getSinglePlugin( + persistedClusterStateServiceFactories, + ClusterCoordinationPlugin.PersistedClusterStateServiceFactory.class + ).map(f -> f.newPersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool, compatibilityVersions)) .orElseGet( () -> new PersistedClusterStateService(nodeEnvironment, xContentRegistry, clusterSettings, threadPool::relativeTimeInMillis) ); + + return b -> b.bind(PersistedClusterStateService.class).toInstance(service); + } + + private List> buildReservedStateHandlers( + SettingsModule settingsModule, + ClusterService clusterService, + IndicesService indicesService, + SystemIndices systemIndices, + IndexSettingProviders indexSettingProviders, + MetadataCreateIndexService metadataCreateIndexService + ) { + List> reservedStateHandlers = new ArrayList<>(); + + // add all reserved state handlers from server + reservedStateHandlers.add(new ReservedClusterSettingsAction(settingsModule.getClusterSettings())); + + var templateService = new MetadataIndexTemplateService( + clusterService, + metadataCreateIndexService, + indicesService, + settingsModule.getIndexScopedSettings(), + xContentRegistry, + systemIndices, + indexSettingProviders + ); + reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); + + // add all reserved state handlers from plugins + pluginsService.loadServiceProviders(ReservedClusterStateHandlerProvider.class) + .forEach(h -> reservedStateHandlers.addAll(h.handlers())); + + return reservedStateHandlers; + } + + private Module loadPersistentTasksService( + SettingsModule settingsModule, + ClusterService clusterService, + ThreadPool threadPool, + SystemIndices systemIndices, + FeatureService featureService, + IndexNameExpressionResolver indexNameExpressionResolver, + MetadataUpdateSettingsService metadataUpdateSettingsService, + MetadataCreateIndexService metadataCreateIndexService + ) { + PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); + SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( + client, + clusterService, + systemIndices, + metadataUpdateSettingsService, + metadataCreateIndexService, + settingsModule.getIndexScopedSettings() + ); + HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create( + clusterService, + persistentTasksService, + featureService, + settingsModule.getSettings(), + clusterService.getClusterSettings() + ); + Stream> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor); + + Stream> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) + .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver)) + .flatMap(List::stream); + + PersistentTasksExecutorRegistry registry = new PersistentTasksExecutorRegistry( + Stream.concat(pluginTaskExecutors, builtinTaskExecutors).toList() + ); + PersistentTasksClusterService persistentTasksClusterService = new PersistentTasksClusterService( + settingsModule.getSettings(), + registry, + clusterService, + threadPool + ); + resourcesToClose.add(persistentTasksClusterService); + + return b -> { + b.bind(PersistentTasksService.class).toInstance(persistentTasksService); + b.bind(HealthNodeTaskExecutor.class).toInstance(healthNodeTaskExecutor); + b.bind(PersistentTasksExecutorRegistry.class).toInstance(registry); + b.bind(PersistentTasksClusterService.class).toInstance(persistentTasksClusterService); + }; } } From 5021087f68010f5d6a506c58667b85619be80ed9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 24 Nov 2023 10:04:21 +0000 Subject: [PATCH 20/35] Renamings/cleanups in snapshots code (#102564) Extracted from #101717 --- .../cluster/SnapshotsInProgress.java | 7 ++----- .../snapshots/IndexShardSnapshotStatus.java | 9 +++++++++ .../snapshots/SnapshotShardsService.java | 16 ++++++++++------ .../snapshots/SnapshotsService.java | 12 ++++++------ 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java index 0f046d4ab94f1..34a0024c1a467 100644 --- a/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java +++ b/server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; @@ -221,11 +222,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - Iterators.single((builder, params) -> builder.startArray("snapshots")), - asStream().iterator(), - Iterators.single((builder, params) -> builder.endArray()) - ); + return Iterators.concat(ChunkedToXContentHelper.startArray("snapshots"), asStream().iterator(), ChunkedToXContentHelper.endArray()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index 140c4684d1a70..451af25dfa649 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -61,7 +61,16 @@ public enum Stage { * where an abort could have occurred. */ public enum AbortStatus { + /** + * The shard snapshot got past the stage where an abort or pause could have occurred, and is either complete or on its way to + * completion. + */ NO_ABORT, + + /** + * The shard snapshot stopped before completion, either because the whole snapshot was aborted or because this node is to be + * removed. + */ ABORTED } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index 134e76c57ed4c..cc390618e5ebb 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -249,7 +249,7 @@ private void handleUpdatedSnapshotsInProgressEntry(String localNodeId, Snapshots // due to CS batching we might have missed the INIT state and straight went into ABORTED // notify master that abort has completed by moving to FAILED if (shard.getValue().state() == ShardState.ABORTED && localNodeId.equals(shard.getValue().nodeId())) { - notifyFailedSnapshotShard(snapshot, sid, shard.getValue().reason(), shard.getValue().generation()); + notifyUnsuccessfulSnapshotShard(snapshot, sid, shard.getValue().reason(), shard.getValue().generation()); } } else { snapshotStatus.abortIfNotCompleted("snapshot has been aborted", notifyOnAbortTaskRunner::enqueueTask); @@ -343,7 +343,7 @@ public void onFailure(Exception e) { logger.warn(() -> format("[%s][%s] failed to snapshot shard", shardId, snapshot), e); } snapshotStatus.moveToFailed(threadPool.absoluteTimeInMillis(), failure); - notifyFailedSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation()); + notifyUnsuccessfulSnapshotShard(snapshot, shardId, failure, snapshotStatus.generation()); } }); } @@ -540,7 +540,7 @@ private void syncShardStatsOnNewMaster(List entries) snapshot.snapshot(), shardId ); - notifyFailedSnapshotShard( + notifyUnsuccessfulSnapshotShard( snapshot.snapshot(), shardId, indexShardSnapshotStatus.getFailure(), @@ -554,15 +554,19 @@ private void syncShardStatsOnNewMaster(List entries) } } - /** Notify the master node that the given shard has been successfully snapshotted **/ + /** + * Notify the master node that the given shard snapshot completed successfully. + */ private void notifySuccessfulSnapshotShard(final Snapshot snapshot, final ShardId shardId, ShardSnapshotResult shardSnapshotResult) { assert shardSnapshotResult != null; assert shardSnapshotResult.getGeneration() != null; sendSnapshotShardUpdate(snapshot, shardId, ShardSnapshotStatus.success(clusterService.localNode().getId(), shardSnapshotResult)); } - /** Notify the master node that the given shard failed to be snapshotted **/ - private void notifyFailedSnapshotShard( + /** + * Notify the master node that the given shard snapshot has completed but did not succeed + */ + private void notifyUnsuccessfulSnapshotShard( final Snapshot snapshot, final ShardId shardId, final String failure, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index e6b140a3e70b8..3006689e90cd8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -918,17 +918,17 @@ private static boolean assertNoDanglingSnapshots(ClusterState state) { * disconnect of a data node that was executing a snapshot) or a routing change that started shards whose snapshot state is * {@link SnapshotsInProgress.ShardState#WAITING}. * - * @param changedNodes true iff either a master fail-over occurred or a data node that was doing snapshot work got removed from the + * @param changedNodes true if either a master fail-over occurred or a data node that was doing snapshot work was removed from the * cluster - * @param startShards true iff any waiting shards were started due to a routing change + * @param changedShards true if any waiting shards changed state in the routing table */ - private void processExternalChanges(boolean changedNodes, boolean startShards) { - if (changedNodes == false && startShards == false) { + private void processExternalChanges(boolean changedNodes, boolean changedShards) { + if (changedNodes == false && changedShards == false) { // nothing to do, no relevant external change happened return; } - final String source = "update snapshot after shards started [" - + startShards + final String source = "update snapshot after shards changed [" + + changedShards + "] or node configuration changed [" + changedNodes + "]"; From 919df917fdffea709e750b9a02e98a686febc05d Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 24 Nov 2023 10:10:12 +0000 Subject: [PATCH 21/35] Check that every ISRT has a primary (#102560) --- .../elasticsearch/cluster/routing/IndexShardRoutingTable.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index bd15d924c9c19..8e257ff2c7a54 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -114,6 +114,7 @@ public class IndexShardRoutingTable { allShardsStarted = false; } } + assert primary != null || shards.isEmpty() : shards; this.primary = primary; this.replicas = CollectionUtils.wrapUnmodifiableOrEmptySingleton(replicas); this.activeShards = CollectionUtils.wrapUnmodifiableOrEmptySingleton(activeShards); From 2a24bd29fbb32a0309832798c4705bc2e25abac3 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 24 Nov 2023 11:18:25 +0100 Subject: [PATCH 22/35] Fix this-escape -Werror (#102574) Using newer java versions expose a -Werror failure in building MinioTestContainer --- .../elasticsearch/test/fixtures/minio/MinioTestContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index 2572f7986dd93..fcb95890ace31 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -12,7 +12,7 @@ import org.junit.rules.TestRule; import org.testcontainers.images.builder.ImageFromDockerfile; -public class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule { +public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer implements TestRule { private static final int servicePort = 9000; private final boolean enabled; From e1c96c4837482649391d5714cdd686f3cd8086e3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 24 Nov 2023 10:47:32 +0000 Subject: [PATCH 23/35] Add SnapshotInProgressAllocationDecider unit tests (#102572) This allocation decider has no unit tests today, but it would be useful to have some so that they can be used to support changes to its logic. This commit adds the missing test suite. --- ...pshotInProgressAllocationDeciderTests.java | 240 ++++++++++++++++++ .../cluster/routing/TestShardRouting.java | 2 + 2 files changed, 242 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java new file mode 100644 index 0000000000000..f7d6e9c9d623c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java @@ -0,0 +1,240 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.ClusterInfo; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.repositories.IndexId; +import org.elasticsearch.repositories.ShardGeneration; +import org.elasticsearch.repositories.ShardSnapshotResult; +import org.elasticsearch.snapshots.Snapshot; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotShardSizeInfo; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class SnapshotInProgressAllocationDeciderTests extends ESTestCase { + + private final SnapshotInProgressAllocationDecider decider = new SnapshotInProgressAllocationDecider(); + private final Index index = new Index(randomIdentifier(), randomUUID()); + private final ShardId shardId = new ShardId(index, 0); + private final String repositoryName = randomIdentifier(); + private final Snapshot snapshot = new Snapshot(repositoryName, new SnapshotId(randomIdentifier(), randomUUID())); + private final String nodeId = randomIdentifier(); + + public void testYesWhenSimulating() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + ClusterState.EMPTY_STATE, + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ).mutableCloneForSimulation(); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("allocation is always enabled when simulating", decision.getExplanation()); + } + + public void testYesWhenNotPrimary() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + ClusterState.EMPTY_STATE, + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, false, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("the shard is not being snapshotted", decision.getExplanation()); + } + + public void testYesWhenNoSnapshots() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + ClusterState.EMPTY_STATE, + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("no snapshots are currently running", decision.getExplanation()); + } + + public void testYesWhenNoShardSnapshot() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + // snapshot in progress but not targetting this shard + makeClusterState(new ShardId(randomIdentifier(), randomUUID(), 0), randomFrom(SnapshotsInProgress.ShardState.values())), + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("the shard is not being snapshotted", decision.getExplanation()); + } + + public void testYesWhenShardSnapshotComplete() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + // snapshot in progress but complete + makeClusterState( + shardId, + randomFrom( + Arrays.stream(SnapshotsInProgress.ShardState.values()).filter(SnapshotsInProgress.ShardState::completed).toList() + ) + ), + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("the shard is not being snapshotted", decision.getExplanation()); + } + + public void testYesWhenShardSnapshotOnDifferentNode() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + makeClusterState(shardId, randomFrom(SnapshotsInProgress.ShardState.values())), + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + // shard on a different node from the snapshot in progress one + TestShardRouting.newShardRouting(shardId, randomIdentifier(), true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(Decision.Type.YES, decision.type()); + assertEquals("the shard is not being snapshotted", decision.getExplanation()); + } + + public void testThrottleWhenSnapshotInProgress() { + final var routingAllocation = new RoutingAllocation( + new AllocationDeciders(List.of(decider)), + makeClusterState(shardId, SnapshotsInProgress.ShardState.INIT), + ClusterInfo.EMPTY, + SnapshotShardSizeInfo.EMPTY, + randomNonNegativeLong() + ); + routingAllocation.setDebugMode(RoutingAllocation.DebugMode.ON); + + final var decision = decider.canAllocate( + TestShardRouting.newShardRouting(shardId, nodeId, true, ShardRoutingState.STARTED), + null, + routingAllocation + ); + + assertEquals(decision.getExplanation(), Decision.Type.THROTTLE, decision.type()); + assertEquals( + "waiting for snapshotting of shard [" + shardId + "] to complete on this node [" + nodeId + "]", + decision.getExplanation() + ); + } + + private ClusterState makeClusterState(ShardId shardId, SnapshotsInProgress.ShardState shardState) { + return ClusterState.builder(ClusterName.DEFAULT) + .putCustom(SnapshotsInProgress.TYPE, makeSnapshotsInProgress(shardId, shardState)) + .build(); + } + + private SnapshotsInProgress makeSnapshotsInProgress(ShardId snapshotShardId, SnapshotsInProgress.ShardState shardState) { + final SnapshotsInProgress.ShardSnapshotStatus shardSnapshotStatus; + if (shardState == SnapshotsInProgress.ShardState.SUCCESS) { + shardSnapshotStatus = SnapshotsInProgress.ShardSnapshotStatus.success( + nodeId, + new ShardSnapshotResult(ShardGeneration.newGeneration(random()), ByteSizeValue.ZERO, 1) + ); + } else if (shardState == SnapshotsInProgress.ShardState.QUEUED) { + shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(null, shardState, null); + } else if (shardState.failed()) { + shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus( + nodeId, + shardState, + randomAlphaOfLength(10), + ShardGeneration.newGeneration(random()) + ); + } else { + shardSnapshotStatus = new SnapshotsInProgress.ShardSnapshotStatus(nodeId, shardState, ShardGeneration.newGeneration(random())); + } + return SnapshotsInProgress.EMPTY.withUpdatedEntriesForRepo( + repositoryName, + List.of( + SnapshotsInProgress.Entry.snapshot( + snapshot, + randomBoolean(), + randomBoolean(), + shardState.completed() ? SnapshotsInProgress.State.SUCCESS : SnapshotsInProgress.State.STARTED, + Map.of(snapshotShardId.getIndexName(), new IndexId(snapshotShardId.getIndexName(), randomUUID())), + List.of(), + List.of(), + randomNonNegativeLong(), + randomNonNegativeLong(), + Map.of(snapshotShardId, shardSnapshotStatus), + null, + Map.of(), + IndexVersion.current() + ) + ) + ); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 8306fded6c29d..89c8546d6b7d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -23,6 +23,7 @@ import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; +import static org.junit.Assert.assertNotEquals; /** * A helper that allows to create shard routing instances within tests, while not requiring to expose @@ -35,6 +36,7 @@ public static ShardRouting newShardRouting(String index, int shardId, String cur } public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) { + assertNotEquals(ShardRoutingState.RELOCATING, state); return new ShardRouting( shardId, currentNodeId, From e649a08a1b0881761ea3a54716ebb90308f7a688 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 24 Nov 2023 21:57:33 +1100 Subject: [PATCH 24/35] [Test mute] AwaitsFix #102547 --- .../javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java index c582191c085f4..6a0a7bfd9769c 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/HttpStatsIT.java @@ -52,6 +52,7 @@ public void testNodeHttpStats() throws IOException { assertHttpStats(new XContentTestUtils.JsonMapView((Map) nodesMap.get(nodeId))); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102547") @SuppressWarnings("unchecked") public void testClusterInfoHttpStats() throws IOException { internalCluster().ensureAtLeastNumDataNodes(3); From e499ea97451328ba34b59e15563f1c36efcd11a1 Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:09:24 +0100 Subject: [PATCH 25/35] Respect ChunkedDataExtractor's query in the wrapped ScrollDataExtractor. (#102492) * Respect ChunkedDataExtractor's query in the wrapped ScrollDataExtractor. * Update docs/changelog/102492.yaml * Add+fix ChunkedDataExtractorTests * Close the scroll in the preview datafeed endpoint * Add extraFilters as constructor param of the DataExtractors * Update docs/changelog/102492.yaml Co-authored-by: David Roberts --------- Co-authored-by: David Roberts --- docs/changelog/102492.yaml | 5 +++ .../ml/datafeed/extractor/DataExtractor.java | 5 +++ .../TransportPreviewDatafeedAction.java | 11 +++--- .../extractor/DataExtractorFactory.java | 36 +++++++++++++------ .../AbstractAggregationDataExtractor.java | 5 +++ .../AggregationDataExtractorFactory.java | 18 +++------- .../CompositeAggregationDataExtractor.java | 5 +++ ...positeAggregationDataExtractorFactory.java | 16 ++++----- .../RollupDataExtractorFactory.java | 23 +++++------- .../chunked/ChunkedDataExtractor.java | 8 +++++ .../chunked/ChunkedDataExtractorFactory.java | 21 +++++------ .../extractor/scroll/ScrollDataExtractor.java | 6 ++++ .../scroll/ScrollDataExtractorFactory.java | 25 ++++++------- .../TransportPreviewDatafeedActionTests.java | 6 ++-- .../AggregationDataExtractorFactoryTests.java | 1 + .../ChunkedDataExtractorFactoryTests.java | 1 + .../chunked/ChunkedDataExtractorTests.java | 8 +++-- 17 files changed, 118 insertions(+), 82 deletions(-) create mode 100644 docs/changelog/102492.yaml diff --git a/docs/changelog/102492.yaml b/docs/changelog/102492.yaml new file mode 100644 index 0000000000000..943d82873e0b6 --- /dev/null +++ b/docs/changelog/102492.yaml @@ -0,0 +1,5 @@ +pr: 102492 +summary: Ensure datafeed previews with no start or end time don't search the cold or frozen tiers +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/DataExtractor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/DataExtractor.java index e432c8d8a9e14..ba2043a17767f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/DataExtractor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/extractor/DataExtractor.java @@ -41,6 +41,11 @@ record Result(SearchInterval searchInterval, Optional data) {} */ void cancel(); + /** + * Cancels and immediately destroys the data extractor, releasing all its resources. + */ + void destroy(); + /** * @return the end time to which this extractor will search */ diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java index 5d4c809c9c83c..5dd671962569a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java @@ -132,6 +132,10 @@ private void previewDatafeed( PreviewDatafeedAction.Request request, ActionListener listener ) { + final QueryBuilder extraFilters = request.getStartTime().isPresent() || request.getEndTime().isPresent() + ? null + : QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(DataTierFieldMapper.NAME, "data_frozen", "data_cold")); + DatafeedConfig.Builder previewDatafeedBuilder = buildPreviewDatafeed(datafeedConfig); useSecondaryAuthIfAvailable(securityContext, () -> { previewDatafeedBuilder.setHeaders( @@ -144,6 +148,7 @@ private void previewDatafeed( DataExtractorFactory.create( new ParentTaskAssigningClient(client, parentTaskId), previewDatafeedConfig, + extraFilters, job, xContentRegistry, // Fake DatafeedTimingStatsReporter that does not have access to results index @@ -158,9 +163,7 @@ private void previewDatafeed( final long start = request.getStartTime().orElse(0); final long end = request.getEndTime() .orElse(isDateNanos ? DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli() : Long.MAX_VALUE); - DataExtractor dataExtractor = request.getStartTime().isPresent() || request.getEndTime().isPresent() - ? dataExtractorFactory.newExtractor(start, end) - : dataExtractorFactory.newExtractor(start, end, hotOnly); + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(start, end); threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(() -> previewDatafeed(dataExtractor, l)); }) ) @@ -226,7 +229,7 @@ static void previewDatafeed(DataExtractor dataExtractor, ActionListener listener + ) { + create(client, datafeed, null, job, xContentRegistry, timingStatsReporter, listener); + } /** * Creates a {@code DataExtractorFactory} for the given datafeed-job combination. @@ -46,6 +51,7 @@ public interface DataExtractorFactory { static void create( Client client, DatafeedConfig datafeed, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DatafeedTimingStatsReporter timingStatsReporter, @@ -56,7 +62,7 @@ static void create( ActionListener factoryHandler = ActionListener.wrap( factory -> listener.onResponse( datafeed.getChunkingConfig().isEnabled() - ? new ChunkedDataExtractorFactory(client, datafeed, job, xContentRegistry, factory, timingStatsReporter) + ? new ChunkedDataExtractorFactory(client, datafeed, extraFilters, job, xContentRegistry, factory, timingStatsReporter) : factory ), listener::onFailure @@ -69,14 +75,22 @@ static void create( return; } if (hasAggs == false) { - ScrollDataExtractorFactory.create(client, datafeed, job, xContentRegistry, timingStatsReporter, factoryHandler); + ScrollDataExtractorFactory.create( + client, + datafeed, + extraFilters, + job, + xContentRegistry, + timingStatsReporter, + factoryHandler + ); return; } if (hasRollup && datafeed.getRuntimeMappings().isEmpty() == false) { // TODO Rollup V2 will support runtime fields listener.onFailure( new IllegalArgumentException( - "The datafeed has runtime_mappings defined, " + "runtime fields are not supported in rollup searches" + "The datafeed has runtime_mappings defined, runtime fields are not supported in rollup searches" ) ); return; @@ -90,6 +104,7 @@ static void create( final DataExtractorFactory dataExtractorFactory = new CompositeAggregationDataExtractorFactory( client, datafeed, + extraFilters, job, xContentRegistry, timingStatsReporter, @@ -107,6 +122,7 @@ static void create( RollupDataExtractorFactory.create( client, datafeed, + extraFilters, job, response.getJobs(), xContentRegistry, @@ -115,7 +131,7 @@ static void create( ); } else { factoryHandler.onResponse( - new AggregationDataExtractorFactory(client, datafeed, job, xContentRegistry, timingStatsReporter) + new AggregationDataExtractorFactory(client, datafeed, extraFilters, job, xContentRegistry, timingStatsReporter) ); } }, e -> { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java index 0d4ba9fd2086d..421581e2622ab 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AbstractAggregationDataExtractor.java @@ -77,6 +77,11 @@ public void cancel() { hasNext = false; } + @Override + public void destroy() { + cancel(); + } + @Override public long getEndTime() { return context.end; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java index 5a1dfa29e7000..b4869555fae99 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java @@ -23,6 +23,7 @@ public record AggregationDataExtractorFactory( Client client, DatafeedConfig datafeedConfig, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DatafeedTimingStatsReporter timingStatsReporter @@ -37,19 +38,10 @@ public static AggregatedSearchRequestBuilder requestBuilder(Client client, Strin @Override public DataExtractor newExtractor(long start, long end) { - return buildExtractor(start, end, datafeedConfig.getParsedQuery(xContentRegistry)); - } - - @Override - public DataExtractor newExtractor(long start, long end, QueryBuilder queryBuilder) { - return buildExtractor( - start, - end, - QueryBuilders.boolQuery().filter(datafeedConfig.getParsedQuery(xContentRegistry)).filter(queryBuilder) - ); - } - - private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBuilder) { + QueryBuilder queryBuilder = datafeedConfig.getParsedQuery(xContentRegistry); + if (extraFilters != null) { + queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); + } long histogramInterval = datafeedConfig.getHistogramIntervalMillis(xContentRegistry); AggregationDataExtractorContext dataExtractorContext = new AggregationDataExtractorContext( job.getId(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java index 13a7e8cf9a89e..859dd506a7712 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractor.java @@ -90,6 +90,11 @@ public void cancel() { isCancelled = true; } + @Override + public void destroy() { + cancel(); + } + @Override public long getEndTime() { return context.end; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorFactory.java index 83d84fc0f12c5..373ca8f5ada64 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/CompositeAggregationDataExtractorFactory.java @@ -32,6 +32,7 @@ public class CompositeAggregationDataExtractorFactory implements DataExtractorFa private final Client client; private final DatafeedConfig datafeedConfig; + private final QueryBuilder extraFilters; private final Job job; private final DatafeedTimingStatsReporter timingStatsReporter; private final String compositeAggName; @@ -46,6 +47,7 @@ public class CompositeAggregationDataExtractorFactory implements DataExtractorFa public CompositeAggregationDataExtractorFactory( Client client, DatafeedConfig datafeedConfig, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DatafeedTimingStatsReporter timingStatsReporter, @@ -53,6 +55,7 @@ public CompositeAggregationDataExtractorFactory( ) { this.client = Objects.requireNonNull(client); this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.extraFilters = extraFilters; this.job = Objects.requireNonNull(job); this.timingStatsReporter = Objects.requireNonNull(timingStatsReporter); this.parsedQuery = datafeedConfig.getParsedQuery(xContentRegistry); @@ -92,15 +95,10 @@ public CompositeAggregationDataExtractorFactory( @Override public DataExtractor newExtractor(long start, long end) { - return buildNewExtractor(start, end, parsedQuery); - } - - @Override - public DataExtractor newExtractor(long start, long end, QueryBuilder queryBuilder) { - return buildNewExtractor(start, end, QueryBuilders.boolQuery().filter(parsedQuery).filter(queryBuilder)); - } - - private DataExtractor buildNewExtractor(long start, long end, QueryBuilder queryBuilder) { + QueryBuilder queryBuilder = parsedQuery; + if (extraFilters != null) { + queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); + } CompositeAggregationBuilder compositeAggregationBuilder = new CompositeAggregationBuilder( compositeAggName, compositeValuesSourceBuilders diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java index ab87f36c89aab..40a578430a033 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractorFactory.java @@ -50,6 +50,7 @@ public class RollupDataExtractorFactory implements DataExtractorFactory { private final Client client; private final DatafeedConfig datafeedConfig; + private final QueryBuilder extraFilters; private final Job job; private final NamedXContentRegistry xContentRegistry; private final DatafeedTimingStatsReporter timingStatsReporter; @@ -57,12 +58,14 @@ public class RollupDataExtractorFactory implements DataExtractorFactory { private RollupDataExtractorFactory( Client client, DatafeedConfig datafeedConfig, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DatafeedTimingStatsReporter timingStatsReporter ) { this.client = Objects.requireNonNull(client); this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.extraFilters = extraFilters; this.job = Objects.requireNonNull(job); this.xContentRegistry = xContentRegistry; this.timingStatsReporter = Objects.requireNonNull(timingStatsReporter); @@ -80,19 +83,10 @@ public static AggregatedSearchRequestBuilder requestBuilder(Client client, Strin @Override public DataExtractor newExtractor(long start, long end) { - return buildExtractor(start, end, datafeedConfig.getParsedQuery(xContentRegistry)); - } - - @Override - public DataExtractor newExtractor(long start, long end, QueryBuilder queryBuilder) { - return buildExtractor( - start, - end, - QueryBuilders.boolQuery().filter(datafeedConfig.getParsedQuery(xContentRegistry)).filter(queryBuilder) - ); - } - - private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBuilder) { + QueryBuilder queryBuilder = datafeedConfig.getParsedQuery(xContentRegistry); + if (extraFilters != null) { + queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); + } long histogramInterval = datafeedConfig.getHistogramIntervalMillis(xContentRegistry); AggregationDataExtractorContext dataExtractorContext = new AggregationDataExtractorContext( job.getId(), @@ -114,6 +108,7 @@ private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBui public static void create( Client client, DatafeedConfig datafeed, + QueryBuilder extraFilters, Job job, Map rollupJobsWithCaps, NamedXContentRegistry xContentRegistry, @@ -171,7 +166,7 @@ public static void create( return; } - listener.onResponse(new RollupDataExtractorFactory(client, datafeed, job, xContentRegistry, timingStatsReporter)); + listener.onResponse(new RollupDataExtractorFactory(client, datafeed, extraFilters, job, xContentRegistry, timingStatsReporter)); } private static boolean validInterval(long datafeedInterval, ParsedRollupCaps rollupJobGroupConfig) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java index 090c85cb8ba36..669370f1f0175 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -197,6 +197,14 @@ public void cancel() { isCancelled = true; } + @Override + public void destroy() { + cancel(); + if (currentExtractor != null) { + currentExtractor.destroy(); + } + } + @Override public long getEndTime() { return context.end; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java index 65d082ffeadea..d0fbeb03150ed 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java @@ -23,6 +23,8 @@ public class ChunkedDataExtractorFactory implements DataExtractorFactory { private final Client client; private final DatafeedConfig datafeedConfig; + + private final QueryBuilder extraFilters; private final Job job; private final DataExtractorFactory dataExtractorFactory; private final NamedXContentRegistry xContentRegistry; @@ -31,6 +33,7 @@ public class ChunkedDataExtractorFactory implements DataExtractorFactory { public ChunkedDataExtractorFactory( Client client, DatafeedConfig datafeedConfig, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DataExtractorFactory dataExtractorFactory, @@ -38,6 +41,7 @@ public ChunkedDataExtractorFactory( ) { this.client = Objects.requireNonNull(client); this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.extraFilters = extraFilters; this.job = Objects.requireNonNull(job); this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); this.xContentRegistry = xContentRegistry; @@ -46,19 +50,10 @@ public ChunkedDataExtractorFactory( @Override public DataExtractor newExtractor(long start, long end) { - return buildExtractor(start, end, datafeedConfig.getParsedQuery(xContentRegistry)); - } - - @Override - public DataExtractor newExtractor(long start, long end, QueryBuilder queryBuilder) { - return buildExtractor( - start, - end, - QueryBuilders.boolQuery().filter(datafeedConfig.getParsedQuery(xContentRegistry)).filter(queryBuilder) - ); - } - - private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBuilder) { + QueryBuilder queryBuilder = datafeedConfig.getParsedQuery(xContentRegistry); + if (extraFilters != null) { + queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); + } ChunkedDataExtractorContext.TimeAligner timeAligner = newTimeAligner(); ChunkedDataExtractorContext dataExtractorContext = new ChunkedDataExtractorContext( job.getId(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index e7aba2211b2df..f633a69bf807f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -84,6 +84,12 @@ public void cancel() { isCancelled = true; } + @Override + public void destroy() { + cancel(); + clearScroll(); + } + @Override public long getEndTime() { return context.end; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java index 7a485d08d0643..27838795aedd2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java @@ -38,6 +38,7 @@ public class ScrollDataExtractorFactory implements DataExtractorFactory { private final Client client; private final DatafeedConfig datafeedConfig; + private final QueryBuilder extraFilters; private final Job job; private final TimeBasedExtractedFields extractedFields; private final NamedXContentRegistry xContentRegistry; @@ -46,6 +47,7 @@ public class ScrollDataExtractorFactory implements DataExtractorFactory { private ScrollDataExtractorFactory( Client client, DatafeedConfig datafeedConfig, + QueryBuilder extraFilters, Job job, TimeBasedExtractedFields extractedFields, NamedXContentRegistry xContentRegistry, @@ -53,6 +55,7 @@ private ScrollDataExtractorFactory( ) { this.client = Objects.requireNonNull(client); this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.extraFilters = extraFilters; this.job = Objects.requireNonNull(job); this.extractedFields = Objects.requireNonNull(extractedFields); this.xContentRegistry = xContentRegistry; @@ -61,19 +64,10 @@ private ScrollDataExtractorFactory( @Override public DataExtractor newExtractor(long start, long end) { - return buildExtractor(start, end, datafeedConfig.getParsedQuery(xContentRegistry)); - } - - @Override - public DataExtractor newExtractor(long start, long end, QueryBuilder queryBuilder) { - return buildExtractor( - start, - end, - QueryBuilders.boolQuery().filter(datafeedConfig.getParsedQuery(xContentRegistry)).filter(queryBuilder) - ); - } - - private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBuilder) { + QueryBuilder queryBuilder = datafeedConfig.getParsedQuery(xContentRegistry); + if (extraFilters != null) { + queryBuilder = QueryBuilders.boolQuery().filter(queryBuilder).filter(extraFilters); + } ScrollDataExtractorContext dataExtractorContext = new ScrollDataExtractorContext( job.getId(), extractedFields, @@ -93,6 +87,7 @@ private DataExtractor buildExtractor(long start, long end, QueryBuilder queryBui public static void create( Client client, DatafeedConfig datafeed, + QueryBuilder extraFilters, Job job, NamedXContentRegistry xContentRegistry, DatafeedTimingStatsReporter timingStatsReporter, @@ -123,7 +118,9 @@ public static void create( return; } TimeBasedExtractedFields fields = TimeBasedExtractedFields.build(job, datafeed, fieldCapabilitiesResponse); - listener.onResponse(new ScrollDataExtractorFactory(client, datafeed, job, fields, xContentRegistry, timingStatsReporter)); + listener.onResponse( + new ScrollDataExtractorFactory(client, datafeed, extraFilters, job, fields, xContentRegistry, timingStatsReporter) + ); }, e -> { Throwable cause = ExceptionsHelper.unwrapCause(e); if (cause instanceof IndexNotFoundException notFound) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedActionTests.java index 4ee8409bbd996..a15bec8c110d6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedActionTests.java @@ -96,7 +96,7 @@ public void testPreviewDatafeed_GivenEmptyStream() throws IOException { assertThat(capturedResponse, equalTo("[]")); assertThat(capturedFailure, is(nullValue())); - verify(dataExtractor).cancel(); + verify(dataExtractor).destroy(); } public void testPreviewDatafeed_GivenNonEmptyStream() throws IOException { @@ -108,7 +108,7 @@ public void testPreviewDatafeed_GivenNonEmptyStream() throws IOException { assertThat(capturedResponse, equalTo("[{\"a\":1, \"b\":2},{\"c\":3, \"d\":4},{\"e\":5, \"f\":6}]")); assertThat(capturedFailure, is(nullValue())); - verify(dataExtractor).cancel(); + verify(dataExtractor).destroy(); } public void testPreviewDatafeed_GivenFailure() throws IOException { @@ -118,6 +118,6 @@ public void testPreviewDatafeed_GivenFailure() throws IOException { assertThat(capturedResponse, is(nullValue())); assertThat(capturedFailure.getMessage(), equalTo("failed")); - verify(dataExtractor).cancel(); + verify(dataExtractor).destroy(); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java index afb1d63a2114d..9a76eb5f2b936 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactoryTests.java @@ -86,6 +86,7 @@ private AggregationDataExtractorFactory createFactory(long histogramInterval) { return new AggregationDataExtractorFactory( client, datafeedConfigBuilder.build(), + null, jobBuilder.build(new Date()), xContentRegistry(), timingStatsReporter diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java index 37123e843aeb8..a7260d34a0136 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactoryTests.java @@ -113,6 +113,7 @@ private ChunkedDataExtractorFactory createFactory(long histogramInterval) { return new ChunkedDataExtractorFactory( client, datafeedConfigBuilder.build(), + null, jobBuilder.build(new Date()), xContentRegistry(), dataExtractorFactory, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java index 81d28b461db76..b0b391f92b527 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java @@ -100,7 +100,6 @@ public void setUpTests() { jobId = "test-job"; timeField = "time"; indices = Arrays.asList("index-1", "index-2"); - query = QueryBuilders.matchAllQuery(); scrollSize = 1000; chunkSpan = null; dataExtractorFactory = mock(DataExtractorFactory.class); @@ -605,7 +604,7 @@ private ChunkedDataExtractorContext createContext(long start, long end, boolean jobId, timeField, indices, - query, + QueryBuilders.matchAllQuery(), scrollSize, start, end, @@ -653,6 +652,11 @@ public void cancel() { // do nothing } + @Override + public void destroy() { + // do nothing + } + @Override public long getEndTime() { return 0; From 9d62fd630aa8c1c3213bde069e23e912b808f00c Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 24 Nov 2023 11:12:46 +0000 Subject: [PATCH 26/35] Remove DesiredNodesSettingsValidator from node construction (#102573) It doesn't need to be injected anymore, it can just be used directly --- .../TransportUpdateDesiredNodesAction.java | 31 ++++++++++++++++--- .../DesiredNodesSettingsValidator.java | 6 ++-- .../elasticsearch/node/NodeConstruction.java | 2 -- ...ransportUpdateDesiredNodesActionTests.java | 10 ++---- .../DesiredNodesSettingsValidatorTests.java | 8 ++--- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java index ef04198c7374b..e3373ded94dc7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java @@ -36,7 +36,9 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.List; import java.util.Locale; +import java.util.function.Consumer; import static java.lang.String.format; @@ -44,7 +46,7 @@ public class TransportUpdateDesiredNodesAction extends TransportMasterNodeAction private static final Logger logger = LogManager.getLogger(TransportUpdateDesiredNodesAction.class); private final FeatureService featureService; - private final DesiredNodesSettingsValidator settingsValidator; + private final Consumer> desiredNodesValidator; private final MasterServiceTaskQueue taskQueue; @Inject @@ -55,7 +57,28 @@ public TransportUpdateDesiredNodesAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - DesiredNodesSettingsValidator settingsValidator, + AllocationService allocationService + ) { + this( + transportService, + clusterService, + featureService, + threadPool, + actionFilters, + indexNameExpressionResolver, + new DesiredNodesSettingsValidator(), + allocationService + ); + } + + TransportUpdateDesiredNodesAction( + TransportService transportService, + ClusterService clusterService, + FeatureService featureService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + Consumer> desiredNodesValidator, AllocationService allocationService ) { super( @@ -71,7 +94,7 @@ public TransportUpdateDesiredNodesAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.featureService = featureService; - this.settingsValidator = settingsValidator; + this.desiredNodesValidator = desiredNodesValidator; this.taskQueue = clusterService.createTaskQueue( "update-desired-nodes", Priority.URGENT, @@ -92,7 +115,7 @@ protected void masterOperation( ActionListener responseListener ) throws Exception { ActionListener.run(responseListener, listener -> { - settingsValidator.validate(request.getNodes()); + desiredNodesValidator.accept(request.getNodes()); taskQueue.submitTask("update-desired-nodes", new UpdateDesiredNodesTask(request, listener), request.masterNodeTimeout()); }); } diff --git a/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java b/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java index 51f90b9610805..a2df7c234680a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidator.java @@ -16,14 +16,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.function.Consumer; import java.util.stream.Collectors; import static java.lang.String.format; -public class DesiredNodesSettingsValidator { +public class DesiredNodesSettingsValidator implements Consumer> { private record DesiredNodeValidationError(int position, @Nullable String externalId, RuntimeException exception) {} - public void validate(List nodes) { + @Override + public void accept(List nodes) { final List validationErrors = new ArrayList<>(); for (int i = 0; i < nodes.size(); i++) { final DesiredNode node = nodes.get(i); diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 764d1b1366153..291be2b60ffe6 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.coordination.MasterHistoryService; import org.elasticsearch.cluster.coordination.Reconfigurator; import org.elasticsearch.cluster.coordination.StableMasterHealthIndicatorService; -import org.elasticsearch.cluster.desirednodes.DesiredNodesSettingsValidator; import org.elasticsearch.cluster.metadata.IndexMetadataVerifier; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -1084,7 +1083,6 @@ record PluginServiceInstances( b.bind(FsHealthService.class).toInstance(fsHealthService); b.bind(PluginShutdownService.class).toInstance(pluginShutdownService); b.bind(IndexSettingProviders.class).toInstance(indexSettingProviders); - b.bind(DesiredNodesSettingsValidator.class).toInstance(new DesiredNodesSettingsValidator()); b.bind(Tracer.class).toInstance(tracer); b.bind(FileSettingsService.class).toInstance(fileSettingsService); b.bind(CompatibilityVersions.class).toInstance(compatibilityVersions); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java index b018b6c47ed4d..4e2948eafc1d7 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesActionTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.coordination.NoMasterBlockService; -import org.elasticsearch.cluster.desirednodes.DesiredNodesSettingsValidator; import org.elasticsearch.cluster.desirednodes.VersionConflictException; import org.elasticsearch.cluster.metadata.DesiredNode; import org.elasticsearch.cluster.metadata.DesiredNodeWithStatus; @@ -45,11 +44,6 @@ public class TransportUpdateDesiredNodesActionTests extends DesiredNodesTestCase { - public static final DesiredNodesSettingsValidator NO_OP_SETTINGS_VALIDATOR = new DesiredNodesSettingsValidator() { - @Override - public void validate(List desiredNodes) {} - }; - public void testWriteBlocks() { ThreadPool threadPool = mock(ThreadPool.class); TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); @@ -60,7 +54,7 @@ public void testWriteBlocks() { threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), - NO_OP_SETTINGS_VALIDATOR, + l -> {}, mock(AllocationService.class) ); @@ -88,7 +82,7 @@ public void testNoBlocks() { threadPool, mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), - NO_OP_SETTINGS_VALIDATOR, + l -> {}, mock(AllocationService.class) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidatorTests.java b/server/src/test/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidatorTests.java index 005252994d77e..819ec4b5266ac 100644 --- a/server/src/test/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/desirednodes/DesiredNodesSettingsValidatorTests.java @@ -17,8 +17,8 @@ import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.emptyArray; +import static org.hamcrest.Matchers.not; public class DesiredNodesSettingsValidatorTests extends ESTestCase { public void testNodeVersionValidation() { @@ -26,10 +26,10 @@ public void testNodeVersionValidation() { final DesiredNodesSettingsValidator validator = new DesiredNodesSettingsValidator(); - final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> validator.validate(desiredNodes)); + final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> validator.accept(desiredNodes)); assertThat(exception.getMessage(), containsString("Nodes with ids")); assertThat(exception.getMessage(), containsString("contain invalid settings")); - assertThat(exception.getSuppressed().length > 0, is(equalTo(true))); + assertThat(exception.getSuppressed(), not(emptyArray())); assertThat(exception.getSuppressed()[0].getMessage(), containsString("Illegal node version")); } } From b112822b224480e22f6b3d90be8b84395df63ad8 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 24 Nov 2023 22:13:34 +1100 Subject: [PATCH 27/35] Allow more plugin threadpool for blob operations (#102567) Add one more threadpool from stateless plugin Relates: #102277 --- .../repositories/blobstore/BlobStoreRepository.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index a0f259f95f14c..cd2b8c73fe90b 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -179,6 +179,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp public static final String STATELESS_SHARD_THREAD_NAME = "stateless_shard"; public static final String STATELESS_TRANSLOG_THREAD_NAME = "stateless_translog"; + public static final String STATELESS_UPLOAD_THREAD_NAME = "stateless_upload"; public static final String SNAPSHOT_PREFIX = "snap-"; @@ -1984,7 +1985,8 @@ protected void assertSnapshotOrGenericThread() { ThreadPool.Names.SNAPSHOT_META, ThreadPool.Names.GENERIC, STATELESS_SHARD_THREAD_NAME, - STATELESS_TRANSLOG_THREAD_NAME + STATELESS_TRANSLOG_THREAD_NAME, + STATELESS_UPLOAD_THREAD_NAME ); } From f3d5a65006a549e260936765c02cdfdaed3b524d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:31:00 +0100 Subject: [PATCH 28/35] Expose the invalidation_time field in Get/Query ApiKey APIs (#102472) This PR exposes `invalidation_time` in responses from [GetApiKey](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html) and [QueryApiKey](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-api-key.html) API To address the first half of https://github.com/elastic/elasticsearch/issues/92404. As a follow up to this PR `invalidation_time` needs to be supported as a valid [query value](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-api-key.html#security-api-query-api-key-request-body) for the `QueryApiKey` API. There will also be a separate PR to add it to the API docs for [GetApiKey](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-api-key.html) and [QueryApiKey](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-query-api-key.html). --- docs/changelog/102472.yaml | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../core/security/action/apikey/ApiKey.java | 55 ++++++-- .../apikey/ApiKeySerializationTests.java | 81 ++++++++++++ .../security/action/apikey/ApiKeyTests.java | 124 ++++++++++-------- .../action/apikey/GetApiKeyResponseTests.java | 10 ++ .../apikey/QueryApiKeyResponseTests.java | 1 + .../xpack/security/QueryApiKeyIT.java | 1 + .../security/apikey/GetApiKeysRestIT.java | 39 ++++++ .../xpack/security/authc/ApiKeyService.java | 10 ++ .../security/authc/ApiKeyServiceTests.java | 51 +++---- .../apikey/RestGetApiKeyActionTests.java | 4 + 12 files changed, 295 insertions(+), 87 deletions(-) create mode 100644 docs/changelog/102472.yaml create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeySerializationTests.java diff --git a/docs/changelog/102472.yaml b/docs/changelog/102472.yaml new file mode 100644 index 0000000000000..b0f5bfc714643 --- /dev/null +++ b/docs/changelog/102472.yaml @@ -0,0 +1,5 @@ +pr: 102472 +summary: Expose the `invalidation` field in Get/Query `ApiKey` APIs +area: Security +type: enhancement +issues: [ ] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 13c3ea447aa3f..ed10c2edeea1d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -178,6 +178,7 @@ static TransportVersion def(int id) { public static final TransportVersion GRANT_API_KEY_CLIENT_AUTHENTICATION_ADDED = def(8_545_00_0); public static final TransportVersion PIT_WITH_INDEX_FILTER = def(8_546_00_0); public static final TransportVersion NODE_INFO_VERSION_AS_STRING = def(8_547_00_0); + public static final TransportVersion GET_API_KEY_INVALIDATION_TIME_ADDED = def(8_548_00_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java index 79c8ef10100e9..b06b7728f541f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java @@ -92,6 +92,7 @@ public String value() { private final Instant creation; private final Instant expiration; private final boolean invalidated; + private final Instant invalidation; private final String username; private final String realm; private final Map metadata; @@ -107,6 +108,7 @@ public ApiKey( Instant creation, Instant expiration, boolean invalidated, + @Nullable Instant invalidation, String username, String realm, @Nullable Map metadata, @@ -120,6 +122,7 @@ public ApiKey( creation, expiration, invalidated, + invalidation, username, realm, metadata, @@ -135,6 +138,7 @@ private ApiKey( Instant creation, Instant expiration, boolean invalidated, + Instant invalidation, String username, String realm, @Nullable Map metadata, @@ -150,6 +154,7 @@ private ApiKey( this.creation = Instant.ofEpochMilli(creation.toEpochMilli()); this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()) : null; this.invalidated = invalidated; + this.invalidation = (invalidation != null) ? Instant.ofEpochMilli(invalidation.toEpochMilli()) : null; this.username = username; this.realm = realm; this.metadata = metadata == null ? Map.of() : metadata; @@ -177,6 +182,12 @@ public ApiKey(StreamInput in) throws IOException { this.creation = in.readInstant(); this.expiration = in.readOptionalInstant(); this.invalidated = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.GET_API_KEY_INVALIDATION_TIME_ADDED)) { + this.invalidation = in.readOptionalInstant(); + } else { + this.invalidation = null; + } + this.username = in.readString(); this.realm = in.readString(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) { @@ -218,6 +229,10 @@ public boolean isInvalidated() { return invalidated; } + public Instant getInvalidation() { + return invalidation; + } + public String getUsername() { return username; } @@ -252,10 +267,11 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t if (expiration != null) { builder.field("expiration", expiration.toEpochMilli()); } - builder.field("invalidated", invalidated) - .field("username", username) - .field("realm", realm) - .field("metadata", (metadata == null ? Map.of() : metadata)); + builder.field("invalidated", invalidated); + if (invalidation != null) { + builder.field("invalidation", invalidation.toEpochMilli()); + } + builder.field("username", username).field("realm", realm).field("metadata", (metadata == null ? Map.of() : metadata)); if (roleDescriptors != null) { builder.startObject("role_descriptors"); for (var roleDescriptor : roleDescriptors) { @@ -321,6 +337,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInstant(creation); out.writeOptionalInstant(expiration); out.writeBoolean(invalidated); + if (out.getTransportVersion().onOrAfter(TransportVersions.GET_API_KEY_INVALIDATION_TIME_ADDED)) { + out.writeOptionalInstant(invalidation); + } out.writeString(username); out.writeString(realm); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) { @@ -334,7 +353,20 @@ public void writeTo(StreamOutput out) throws IOException { @Override public int hashCode() { - return Objects.hash(name, id, type, creation, expiration, invalidated, username, realm, metadata, roleDescriptors, limitedBy); + return Objects.hash( + name, + id, + type, + creation, + expiration, + invalidated, + invalidation, + username, + realm, + metadata, + roleDescriptors, + limitedBy + ); } @Override @@ -355,6 +387,7 @@ public boolean equals(Object obj) { && Objects.equals(creation, other.creation) && Objects.equals(expiration, other.expiration) && Objects.equals(invalidated, other.invalidated) + && Objects.equals(invalidation, other.invalidation) && Objects.equals(username, other.username) && Objects.equals(realm, other.realm) && Objects.equals(metadata, other.metadata) @@ -371,11 +404,12 @@ public boolean equals(Object obj) { Instant.ofEpochMilli((Long) args[3]), (args[4] == null) ? null : Instant.ofEpochMilli((Long) args[4]), (Boolean) args[5], - (String) args[6], + (args[6] == null) ? null : Instant.ofEpochMilli((Long) args[6]), (String) args[7], - (args[8] == null) ? null : (Map) args[8], - (List) args[9], - (RoleDescriptorsIntersection) args[10] + (String) args[8], + (args[9] == null) ? null : (Map) args[9], + (List) args[10], + (RoleDescriptorsIntersection) args[11] ); }); static { @@ -385,6 +419,7 @@ public boolean equals(Object obj) { PARSER.declareLong(constructorArg(), new ParseField("creation")); PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("invalidation")); PARSER.declareString(constructorArg(), new ParseField("username")); PARSER.declareString(constructorArg(), new ParseField("realm")); PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.map(), new ParseField("metadata")); @@ -418,6 +453,8 @@ public String toString() { + expiration + ", invalidated=" + invalidated + + ", invalidation=" + + invalidation + ", username=" + username + ", realm=" diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeySerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeySerializationTests.java new file mode 100644 index 0000000000000..d2a02cd053ca1 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeySerializationTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.action.apikey; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.XPackClientPlugin; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.core.security.action.apikey.ApiKeyTests.randomApiKeyInstance; +import static org.hamcrest.Matchers.nullValue; + +public class ApiKeySerializationTests extends AbstractWireSerializingTestCase { + + public void testSerializationBackwardsCompatibility() throws IOException { + ApiKey testInstance = createTestInstance(); + ApiKey deserializedInstance = copyInstance(testInstance, TransportVersions.V_8_500_064); + try { + // Transport is on a version before invalidation was introduced, so should always be null + assertThat(deserializedInstance.getInvalidation(), nullValue()); + } finally { + dispose(deserializedInstance); + } + } + + @Override + protected ApiKey createTestInstance() { + return randomApiKeyInstance(); + } + + @Override + protected ApiKey mutateInstance(ApiKey instance) throws IOException { + ApiKey copyOfInstance = copyInstance(instance); + // Metadata in test instance is mutable, so mutate it instead of the copy (immutable metadata) to make sure they differ + Object metadataNumberValue = instance.getMetadata().getOrDefault("number", Integer.toString(randomInt())); + instance.getMetadata().put("number", Integer.parseInt(metadataNumberValue.toString()) + randomInt()); + return copyOfInstance; + } + + @Override + protected Writeable.Reader instanceReader() { + return ApiKey::new; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new XPackClientPlugin().getNamedWriteables()); + } + + public static Map randomMetadata() { + Map randomMetadata = randomFrom( + Map.of( + "application", + randomAlphaOfLength(5), + "number", + 1, + "numbers", + List.of(1, 3, 5), + "environment", + Map.of("os", "linux", "level", 42, "category", "trusted") + ), + Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)), + Map.of(), + null + ); + + // Make metadata mutable for testing purposes + return randomMetadata == null ? new HashMap<>() : new HashMap<>(randomMetadata); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeyTests.java index d6c218c47bc39..9e357915186a5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKeyTests.java @@ -22,9 +22,11 @@ import java.io.IOException; import java.time.Instant; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomCrossClusterAccessRoleDescriptor; import static org.elasticsearch.xpack.core.security.authz.RoleDescriptorTests.randomUniquelyNamedRoleDescriptors; @@ -40,38 +42,7 @@ public class ApiKeyTests extends ESTestCase { @SuppressWarnings("unchecked") public void testXContent() throws IOException { - final String name = randomAlphaOfLengthBetween(4, 10); - final String id = randomAlphaOfLength(20); - final ApiKey.Type type = randomFrom(ApiKey.Type.values()); - // between 1970 and 2065 - final Instant creation = Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); - final Instant expiration = randomBoolean() - ? null - : Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); - final boolean invalidated = randomBoolean(); - final String username = randomAlphaOfLengthBetween(4, 10); - final String realmName = randomAlphaOfLengthBetween(3, 8); - final Map metadata = randomMetadata(); - final List roleDescriptors = type == ApiKey.Type.CROSS_CLUSTER - ? List.of(randomCrossClusterAccessRoleDescriptor()) - : randomFrom(randomUniquelyNamedRoleDescriptors(0, 3), null); - final List limitedByRoleDescriptors = type == ApiKey.Type.CROSS_CLUSTER - ? null - : randomUniquelyNamedRoleDescriptors(0, 3); - - final ApiKey apiKey = new ApiKey( - name, - id, - type, - creation, - expiration, - invalidated, - username, - realmName, - metadata, - roleDescriptors, - limitedByRoleDescriptors - ); + final ApiKey apiKey = randomApiKeyInstance(); // The metadata will never be null because the constructor convert it to empty map if a null is passed in assertThat(apiKey.getMetadata(), notNullValue()); @@ -84,51 +55,56 @@ public void testXContent() throws IOException { assertThat(ApiKey.fromXContent(parser), equalTo(apiKey)); } - assertThat(map.get("name"), equalTo(name)); - assertThat(map.get("id"), equalTo(id)); - assertThat(map.get("type"), equalTo(type.value())); - assertThat(Long.valueOf(map.get("creation").toString()), equalTo(creation.toEpochMilli())); - if (expiration != null) { - assertThat(Long.valueOf(map.get("expiration").toString()), equalTo(expiration.toEpochMilli())); + assertThat(map.get("name"), equalTo(apiKey.getName())); + assertThat(map.get("id"), equalTo(apiKey.getId())); + assertThat(map.get("type"), equalTo(apiKey.getType().value())); + assertThat(Long.valueOf(map.get("creation").toString()), equalTo(apiKey.getCreation().toEpochMilli())); + if (apiKey.getExpiration() != null) { + assertThat(Long.valueOf(map.get("expiration").toString()), equalTo(apiKey.getExpiration().toEpochMilli())); } else { assertThat(map.containsKey("expiration"), is(false)); } - assertThat(map.get("invalidated"), is(invalidated)); - assertThat(map.get("username"), equalTo(username)); - assertThat(map.get("realm"), equalTo(realmName)); - assertThat(map.get("metadata"), equalTo(Objects.requireNonNullElseGet(metadata, Map::of))); + assertThat(map.get("invalidated"), is(apiKey.isInvalidated())); + assertThat(map.get("username"), equalTo(apiKey.getUsername())); + assertThat(map.get("realm"), equalTo(apiKey.getRealm())); + assertThat(map.get("metadata"), equalTo(Objects.requireNonNullElseGet(apiKey.getMetadata(), Map::of))); - if (roleDescriptors == null) { + if (apiKey.getRoleDescriptors() == null) { assertThat(map, not(hasKey("role_descriptors"))); assertThat(map, not(hasKey("access"))); } else { final var rdMap = (Map) map.get("role_descriptors"); - assertThat(rdMap.size(), equalTo(roleDescriptors.size())); - for (var roleDescriptor : roleDescriptors) { + assertThat(rdMap.size(), equalTo(apiKey.getRoleDescriptors().size())); + for (var roleDescriptor : apiKey.getRoleDescriptors()) { assertThat(rdMap, hasKey(roleDescriptor.getName())); assertThat(XContentTestUtils.convertToMap(roleDescriptor), equalTo(rdMap.get(roleDescriptor.getName()))); } - if (type == ApiKey.Type.CROSS_CLUSTER) { + if (apiKey.getType() == ApiKey.Type.CROSS_CLUSTER) { final var accessMap = (Map) map.get("access"); final CrossClusterApiKeyRoleDescriptorBuilder roleDescriptorBuilder = CrossClusterApiKeyRoleDescriptorBuilder.parse( XContentTestUtils.convertToXContent(accessMap, XContentType.JSON).utf8ToString() ); - assertThat(roleDescriptorBuilder.build(), equalTo(roleDescriptors.get(0))); + assertThat(roleDescriptorBuilder.build(), equalTo(apiKey.getRoleDescriptors().get(0))); } else { assertThat(map, not(hasKey("access"))); } } final var limitedByList = (List>) map.get("limited_by"); - if (type != ApiKey.Type.CROSS_CLUSTER) { + if (apiKey.getType() != ApiKey.Type.CROSS_CLUSTER) { assertThat(limitedByList.size(), equalTo(1)); final Map limitedByMap = limitedByList.get(0); - assertThat(limitedByMap.size(), equalTo(limitedByRoleDescriptors.size())); - for (RoleDescriptor roleDescriptor : limitedByRoleDescriptors) { - assertThat(limitedByMap, hasKey(roleDescriptor.getName())); - assertThat(XContentTestUtils.convertToMap(roleDescriptor), equalTo(limitedByMap.get(roleDescriptor.getName()))); + + int roleDescriptorCount = 0; + for (Set roleDescriptors : apiKey.getLimitedBy().roleDescriptorsList()) { + for (RoleDescriptor roleDescriptor : roleDescriptors) { + assertThat(limitedByMap, hasKey(roleDescriptor.getName())); + assertThat(XContentTestUtils.convertToMap(roleDescriptor), equalTo(limitedByMap.get(roleDescriptor.getName()))); + roleDescriptorCount++; + } } + assertThat(limitedByMap.size(), equalTo(roleDescriptorCount)); } else { assertThat(limitedByList, nullValue()); } @@ -156,7 +132,7 @@ private ApiKey.Type parseTypeString(String typeString) throws IOException { } public static Map randomMetadata() { - return randomFrom( + Map randomMetadata = randomFrom( Map.of( "application", randomAlphaOfLength(5), @@ -171,5 +147,47 @@ public static Map randomMetadata() { Map.of(), null ); + + // Make metadata mutable for testing purposes + return randomMetadata == null ? new HashMap<>() : new HashMap<>(randomMetadata); + } + + public static ApiKey randomApiKeyInstance() { + final String name = randomAlphaOfLengthBetween(4, 10); + final String id = randomAlphaOfLength(20); + final ApiKey.Type type = randomFrom(ApiKey.Type.values()); + // between 1970 and 2065 + final Instant creation = Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); + final Instant expiration = randomBoolean() + ? null + : Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)); + final boolean invalidated = randomBoolean(); + final Instant invalidation = invalidated + ? Instant.ofEpochSecond(randomLongBetween(0, 3000000000L), randomLongBetween(0, 999999999)) + : null; + final String username = randomAlphaOfLengthBetween(4, 10); + final String realmName = randomAlphaOfLengthBetween(3, 8); + final Map metadata = randomMetadata(); + final List roleDescriptors = type == ApiKey.Type.CROSS_CLUSTER + ? List.of(randomCrossClusterAccessRoleDescriptor()) + : randomFrom(randomUniquelyNamedRoleDescriptors(0, 3), null); + final List limitedByRoleDescriptors = type == ApiKey.Type.CROSS_CLUSTER + ? null + : randomUniquelyNamedRoleDescriptors(0, 3); + + return new ApiKey( + name, + id, + type, + creation, + expiration, + invalidated, + invalidation, + username, + realmName, + metadata, + roleDescriptors, + limitedByRoleDescriptors + ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponseTests.java index 3fcf8ea5a0deb..a32ed8f53f5b2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/GetApiKeyResponseTests.java @@ -49,6 +49,7 @@ public void testSerialization() throws IOException { Instant.now(), (withExpiration) ? Instant.now() : null, false, + null, randomAlphaOfLength(4), randomAlphaOfLength(5), randomBoolean() ? null : Map.of(randomAlphaOfLengthBetween(3, 8), randomAlphaOfLengthBetween(3, 8)), @@ -110,6 +111,7 @@ public void testToXContent() throws IOException { Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + null, "user-a", "realm-x", null, @@ -123,6 +125,7 @@ public void testToXContent() throws IOException { Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + Instant.ofEpochMilli(100000000L), "user-b", "realm-y", Map.of(), @@ -136,6 +139,7 @@ public void testToXContent() throws IOException { Instant.ofEpochMilli(100000L), null, true, + Instant.ofEpochMilli(100000000L), "user-c", "realm-z", Map.of("foo", "bar"), @@ -159,6 +163,7 @@ public void testToXContent() throws IOException { Instant.ofEpochMilli(100000L), null, true, + Instant.ofEpochMilli(100000000L), "user-c", "realm-z", Map.of("foo", "bar"), @@ -192,6 +197,7 @@ public void testToXContent() throws IOException { "creation": 100000, "expiration": 10000000, "invalidated": true, + "invalidation": 100000000, "username": "user-b", "realm": "realm-y", "metadata": {}, @@ -231,6 +237,7 @@ public void testToXContent() throws IOException { %s "creation": 100000, "invalidated": true, + "invalidation": 100000000, "username": "user-c", "realm": "realm-z", "metadata": { @@ -297,6 +304,7 @@ public void testToXContent() throws IOException { %s "creation": 100000, "invalidated": true, + "invalidation": 100000000, "username": "user-c", "realm": "realm-z", "metadata": { @@ -365,6 +373,7 @@ private ApiKey createApiKeyInfo( Instant creation, Instant expiration, boolean invalidated, + Instant invalidation, String username, String realm, Map metadata, @@ -378,6 +387,7 @@ private ApiKey createApiKeyInfo( creation, expiration, invalidated, + invalidation, username, realm, metadata, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java index 9ee71415a91ea..677d2201fe1e1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/QueryApiKeyResponseTests.java @@ -105,6 +105,7 @@ private ApiKey randomApiKeyInfo() { creation, expiration, false, + null, username, realm_name, metadata, diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java index 456fcbd80cf12..f67ce3fc7f2d7 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java @@ -176,6 +176,7 @@ public void testQuery() throws IOException { assertThat(apiKeys.get(0).get("name"), equalTo("temporary-key-1")); assertThat(apiKeys.get(0).get("id"), equalTo(invalidatedApiKeyId1)); assertThat(apiKeys.get(0).get("invalidated"), is(true)); + assertThat(apiKeys.get(0).get("invalidation"), notNullValue()); } apiKeys.forEach(k -> assertThat(k, not(hasKey("_sort")))); }); diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/GetApiKeysRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/GetApiKeysRestIT.java index 0996463e9adb9..e9dc00acf3211 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/GetApiKeysRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/GetApiKeysRestIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.apikey; +import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; @@ -20,6 +21,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; import org.elasticsearch.xpack.core.security.action.apikey.GetApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.apikey.InvalidateApiKeyResponse; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.SecurityOnTrialLicenseRestTestCase; import org.junit.Before; @@ -37,6 +39,8 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class GetApiKeysRestIT extends SecurityOnTrialLicenseRestTestCase { private static final SecureString END_USER_PASSWORD = new SecureString("end-user-password".toCharArray()); @@ -194,6 +198,41 @@ public void testGetApiKeysWithActiveOnlyFlagAndMultipleUsers() throws Exception ); } + public void testInvalidateApiKey() throws Exception { + final String apiKeyId0 = createApiKey(MANAGE_SECURITY_USER, "key-2"); + + Request request = new Request(HttpGet.METHOD_NAME, "/_security/api_key/"); + setUserForRequest(request, MANAGE_SECURITY_USER); + GetApiKeyResponse getApiKeyResponse = GetApiKeyResponse.fromXContent(getParser(client().performRequest(request))); + + assertThat(getApiKeyResponse.getApiKeyInfos().length, equalTo(1)); + ApiKey apiKey = getApiKeyResponse.getApiKeyInfos()[0]; + assertThat(apiKey.isInvalidated(), equalTo(false)); + assertThat(apiKey.getInvalidation(), nullValue()); + assertThat(apiKey.getId(), equalTo(apiKeyId0)); + + request = new Request(HttpDelete.METHOD_NAME, "/_security/api_key/"); + setUserForRequest(request, MANAGE_SECURITY_USER); + request.setJsonEntity(XContentTestUtils.convertToXContent(Map.of("ids", List.of(apiKeyId0)), XContentType.JSON).utf8ToString()); + + InvalidateApiKeyResponse invalidateApiKeyResponse = InvalidateApiKeyResponse.fromXContent( + getParser(client().performRequest(request)) + ); + + assertThat(invalidateApiKeyResponse.getInvalidatedApiKeys().size(), equalTo(1)); + assertThat(invalidateApiKeyResponse.getInvalidatedApiKeys().get(0), equalTo(apiKey.getId())); + + request = new Request(HttpGet.METHOD_NAME, "/_security/api_key/"); + setUserForRequest(request, MANAGE_SECURITY_USER); + getApiKeyResponse = GetApiKeyResponse.fromXContent(getParser(client().performRequest(request))); + + assertThat(getApiKeyResponse.getApiKeyInfos().length, equalTo(1)); + apiKey = getApiKeyResponse.getApiKeyInfos()[0]; + assertThat(apiKey.isInvalidated(), equalTo(true)); + assertThat(apiKey.getInvalidation(), notNullValue()); + assertThat(apiKey.getId(), equalTo(apiKeyId0)); + } + private GetApiKeyResponse getApiKeysWithRequestParams(Map requestParams) throws IOException { return getApiKeysWithRequestParams(MANAGE_SECURITY_USER, requestParams); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 7a3e0474eec51..fc8cc79e419be 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -1996,6 +1996,7 @@ private ApiKey convertSearchHitToApiKeyInfo(SearchHit hit, boolean withLimitedBy Instant.ofEpochMilli(apiKeyDoc.creationTime), apiKeyDoc.expirationTime != -1 ? Instant.ofEpochMilli(apiKeyDoc.expirationTime) : null, apiKeyDoc.invalidated, + apiKeyDoc.invalidation != -1 ? Instant.ofEpochMilli(apiKeyDoc.invalidation) : null, (String) apiKeyDoc.creator.get("principal"), (String) apiKeyDoc.creator.get("realm"), metadata, @@ -2183,6 +2184,7 @@ public static final class ApiKeyDoc { builder.declareLong(constructorArg(), new ParseField("creation_time")); builder.declareLongOrNull(constructorArg(), -1, new ParseField("expiration_time")); builder.declareBoolean(constructorArg(), new ParseField("api_key_invalidated")); + builder.declareLong(optionalConstructorArg(), new ParseField("invalidation_time")); builder.declareString(constructorArg(), new ParseField("api_key_hash")); builder.declareStringOrNull(optionalConstructorArg(), new ParseField("name")); builder.declareInt(constructorArg(), new ParseField("version")); @@ -2198,6 +2200,7 @@ public static final class ApiKeyDoc { final long creationTime; final long expirationTime; final Boolean invalidated; + final long invalidation; final String hash; @Nullable final String name; @@ -2214,6 +2217,7 @@ public ApiKeyDoc( long creationTime, long expirationTime, Boolean invalidated, + @Nullable Long invalidation, String hash, @Nullable String name, int version, @@ -2232,6 +2236,7 @@ public ApiKeyDoc( this.creationTime = creationTime; this.expirationTime = expirationTime; this.invalidated = invalidated; + this.invalidation = (invalidation == null) ? -1 : invalidation; this.hash = hash; this.name = name; this.version = version; @@ -2253,6 +2258,7 @@ public CachedApiKeyDoc toCachedApiKeyDoc() { creationTime, expirationTime, invalidated, + invalidation, hash, name, version, @@ -2278,6 +2284,7 @@ public static final class CachedApiKeyDoc { final long creationTime; final long expirationTime; final Boolean invalidated; + final long invalidation; final String hash; final String name; final int version; @@ -2292,6 +2299,7 @@ public CachedApiKeyDoc( long creationTime, long expirationTime, Boolean invalidated, + long invalidation, String hash, String name, int version, @@ -2304,6 +2312,7 @@ public CachedApiKeyDoc( this.creationTime = creationTime; this.expirationTime = expirationTime; this.invalidated = invalidated; + this.invalidation = invalidation; this.hash = hash; this.name = name; this.version = version; @@ -2320,6 +2329,7 @@ public ApiKeyDoc toApiKeyDoc(BytesReference roleDescriptorsBytes, BytesReference creationTime, expirationTime, invalidated, + invalidation, hash, name, version, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 3be60eb48b42f..90f2520b63473 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -478,8 +478,8 @@ public void testInvalidateApiKeysWillSetInvalidatedFlagAndRecordTimestamp() { return null; }).when(client).execute(eq(ClearSecurityCacheAction.INSTANCE), any(ClearSecurityCacheRequest.class), anyActionListener()); - final long invalidationTime = randomMillisUpToYear9999(); - when(clock.instant()).thenReturn(Instant.ofEpochMilli(invalidationTime)); + final long invalidation = randomMillisUpToYear9999(); + when(clock.instant()).thenReturn(Instant.ofEpochMilli(invalidation)); final ApiKeyService service = createApiKeyService(); PlainActionFuture future = new PlainActionFuture<>(); service.invalidateApiKeys(null, null, null, new String[] { apiKeyId }, future); @@ -488,12 +488,8 @@ public void testInvalidateApiKeysWillSetInvalidatedFlagAndRecordTimestamp() { assertThat(invalidateApiKeyResponse.getInvalidatedApiKeys(), equalTo(List.of(apiKeyId))); verify(updateRequestBuilder).setDoc( argThat( - (ArgumentMatcher>) argument -> Map.of( - "api_key_invalidated", - true, - "invalidation_time", - invalidationTime - ).equals(argument) + (ArgumentMatcher>) argument -> Map.of("api_key_invalidated", true, "invalidation_time", invalidation) + .equals(argument) ) ); } @@ -958,7 +954,7 @@ public void testValidateApiKey() throws Exception { Hasher hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, -1); ApiKeyService service = createApiKeyService(Settings.EMPTY); PlainActionFuture> future = new PlainActionFuture<>(); @@ -985,7 +981,7 @@ public void testValidateApiKey() throws Exception { assertThat(result.getMetadata().get(AuthenticationField.API_KEY_CREATOR_REALM_NAME), is("realm1")); assertThat(result.getMetadata().get(API_KEY_TYPE_KEY), is(apiKeyDoc.type.value())); - apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli(), false); + apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().plus(1L, ChronoUnit.HOURS).toEpochMilli(), false, -1); future = new PlainActionFuture<>(); service.validateApiKeyCredentials( apiKeyId, @@ -1010,7 +1006,7 @@ public void testValidateApiKey() throws Exception { assertThat(result.getMetadata().get(AuthenticationField.API_KEY_CREATOR_REALM_NAME), is("realm1")); assertThat(result.getMetadata().get(API_KEY_TYPE_KEY), is(apiKeyDoc.type.value())); - apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli(), false); + apiKeyDoc = buildApiKeyDoc(hash, Clock.systemUTC().instant().minus(1L, ChronoUnit.HOURS).toEpochMilli(), false, -1); future = new PlainActionFuture<>(); service.validateApiKeyCredentials( apiKeyId, @@ -1024,7 +1020,7 @@ public void testValidateApiKey() throws Exception { assertFalse(result.isAuthenticated()); // key is invalidated - apiKeyDoc = buildApiKeyDoc(hash, -1, true); + apiKeyDoc = buildApiKeyDoc(hash, -1, true, randomLongBetween(0, 3000000000L)); service.getApiKeyAuthCache().put(apiKeyId, new ListenableFuture<>()); assertNotNull(service.getApiKeyAuthCache().get(apiKeyId)); future = new PlainActionFuture<>(); @@ -1215,7 +1211,7 @@ public void testApiKeyCache() throws IOException { Hasher hasher = getFastStoredHashAlgoForTests(); final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); - ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, -1); ApiKeyService service = createApiKeyService(Settings.EMPTY); ApiKeyCredentials creds = getApiKeyCredentials(apiKeyId, apiKey, apiKeyDoc.type); @@ -1236,7 +1232,7 @@ public void testApiKeyCache() throws IOException { assertNotNull(shouldBeSame); assertThat(shouldBeSame, sameInstance(cachedApiKeyHashResult)); - apiKeyDoc = buildApiKeyDoc(hasher.hash(new SecureString("somelongenoughrandomstring".toCharArray())), -1, false); + apiKeyDoc = buildApiKeyDoc(hasher.hash(new SecureString("somelongenoughrandomstring".toCharArray())), -1, false, -1); creds = getApiKeyCredentials(randomAlphaOfLength(12), "otherlongenoughrandomstring", apiKeyDoc.type); future = new PlainActionFuture<>(); service.validateApiKeyCredentials(creds.getId(), apiKeyDoc, creds, Clock.systemUTC(), future); @@ -1514,7 +1510,7 @@ public void testApiKeyCacheDisabled() throws IOException { final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); final Settings settings = Settings.builder().put(ApiKeyService.CACHE_TTL_SETTING.getKey(), "0s").build(); - ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, -1); ApiKeyService service = createApiKeyService(settings); ApiKeyCredentials creds = getApiKeyCredentials(randomAlphaOfLength(12), apiKey, apiKeyDoc.type); @@ -1534,7 +1530,7 @@ public void testApiKeyDocCacheCanBeDisabledSeparately() throws IOException { final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); final Settings settings = Settings.builder().put(ApiKeyService.DOC_CACHE_TTL_SETTING.getKey(), "0s").build(); - ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false); + ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, -1); ApiKeyService service = createApiKeyService(settings); @@ -2078,7 +2074,7 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { final char[] hash = hasher.hash(new SecureString(apiKey.toCharArray())); final var apiKeyService = createApiKeyService(); - final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, null, Version.V_8_2_0.id); + final var apiKeyDocWithNullName = buildApiKeyDoc(hash, -1, false, -1, null, Version.V_8_2_0.id); final var auth = Authentication.newRealmAuthentication( new User("test_user", "role"), new Authentication.RealmRef("realm1", "realm_type1", "node") @@ -2090,14 +2086,14 @@ public void testValidateApiKeyDocBeforeUpdate() throws IOException { ); assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); - final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, "", Version.V_8_2_0.id); + final var apiKeyDocWithEmptyName = buildApiKeyDoc(hash, -1, false, -1, "", Version.V_8_2_0.id); ex = expectThrows( IllegalArgumentException.class, () -> apiKeyService.validateForUpdate(apiKeyId, apiKeyDocWithEmptyName.type, auth, apiKeyDocWithEmptyName) ); assertThat(ex.getMessage(), containsString("cannot update legacy API key [" + apiKeyId + "] without name")); - final ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id); + final ApiKeyDoc apiKeyDoc = buildApiKeyDoc(hash, -1, false, -1, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id); final ApiKey.Type expectedType = randomValueOtherThan(apiKeyDoc.type, () -> randomFrom(ApiKey.Type.values())); ex = expectThrows(IllegalArgumentException.class, () -> apiKeyService.validateForUpdate(apiKeyId, expectedType, auth, apiKeyDoc)); assertThat( @@ -2511,6 +2507,7 @@ public void testValidateApiKeyTypeAndExpiration() throws IOException { hash, randomFrom(-1L, futureTime), false, + -1, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id ); @@ -2535,7 +2532,7 @@ public void testValidateApiKeyTypeAndExpiration() throws IOException { ); // Expired API key - final var apiKeyDoc2 = buildApiKeyDoc(hash, pastTime, false, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id); + final var apiKeyDoc2 = buildApiKeyDoc(hash, pastTime, false, -1, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id); final ApiKeyCredentials apiKeyCredentials2 = getApiKeyCredentials(apiKeyId, apiKey, apiKeyDoc2.type); final PlainActionFuture> future2 = new PlainActionFuture<>(); ApiKeyService.validateApiKeyTypeAndExpiration(apiKeyDoc2, apiKeyCredentials2, clock, future2); @@ -2549,6 +2546,7 @@ public void testValidateApiKeyTypeAndExpiration() throws IOException { hash, randomFrom(-1L, futureTime), false, + -1, randomAlphaOfLengthBetween(3, 8), Version.CURRENT.id ); @@ -2861,15 +2859,17 @@ private void mockSourceDocument(String id, Map sourceMap) throws } } - private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated) throws IOException { - return buildApiKeyDoc(hash, expirationTime, invalidated, randomAlphaOfLength(12)); + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, long invalidation) throws IOException { + return buildApiKeyDoc(hash, expirationTime, invalidated, invalidation, randomAlphaOfLength(12)); } - private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, String name) throws IOException { - return buildApiKeyDoc(hash, expirationTime, invalidated, name, 0); + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, long invalidation, String name) + throws IOException { + return buildApiKeyDoc(hash, expirationTime, invalidated, invalidation, name, 0); } - private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, String name, int version) throws IOException { + private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean invalidated, long invalidation, String name, int version) + throws IOException { final BytesReference metadataBytes = XContentTestUtils.convertToXContent(ApiKeyTests.randomMetadata(), XContentType.JSON); return new ApiKeyDoc( "api_key", @@ -2877,6 +2877,7 @@ private ApiKeyDoc buildApiKeyDoc(char[] hash, long expirationTime, boolean inval Clock.systemUTC().instant().toEpochMilli(), expirationTime, invalidated, + invalidation, new String(hash), name, version, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java index e842dd8588fa9..a1f696cc5dddd 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGetApiKeyActionTests.java @@ -113,6 +113,7 @@ public void sendResponse(RestResponse restResponse) { creation, expiration, false, + null, "user-x", "realm-1", metadata, @@ -171,6 +172,7 @@ public void doE creation, expiration, false, + null, "user-x", "realm-1", metadata, @@ -220,6 +222,7 @@ public void sendResponse(RestResponse restResponse) { creation, expiration, false, + null, "user-x", "realm-1", ApiKeyTests.randomMetadata(), @@ -235,6 +238,7 @@ public void sendResponse(RestResponse restResponse) { creation, expiration, false, + null, "user-y", "realm-1", ApiKeyTests.randomMetadata(), From d13fb4f6fcc6764b3357c65f009551fc3f5875d6 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 24 Nov 2023 12:43:46 +0000 Subject: [PATCH 29/35] Log exceptions thrown during chunked encoding (#102585) Today on exception we `assert false` in the `finally` block, but that means we don't get to see the exception that caused the problem. This commit adds a little extra logging to help there. --- .../org/elasticsearch/http/DefaultRestChannel.java | 3 +-- .../elasticsearch/rest/ChunkedRestResponseBody.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java index 930b20b927bd8..24df7875f7e3d 100644 --- a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java @@ -23,7 +23,6 @@ import org.elasticsearch.rest.AbstractRestChannel; import org.elasticsearch.rest.ChunkedRestResponseBody; import org.elasticsearch.rest.LoggingChunkedRestResponseBody; -import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; @@ -40,7 +39,7 @@ * The default rest channel for incoming requests. This class implements the basic logic for sending a rest * response. It will set necessary headers nad ensure that bytes are released after the response is sent. */ -public class DefaultRestChannel extends AbstractRestChannel implements RestChannel { +public class DefaultRestChannel extends AbstractRestChannel { static final String CLOSE = "close"; static final String CONNECTION = "connection"; diff --git a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java index 9cfe7b84577db..ae267573b4cab 100644 --- a/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java +++ b/server/src/main/java/org/elasticsearch/rest/ChunkedRestResponseBody.java @@ -20,6 +20,8 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Streams; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -36,6 +38,8 @@ */ public interface ChunkedRestResponseBody extends Releasable { + Logger logger = LogManager.getLogger(ChunkedRestResponseBody.class); + /** * @return true once this response has been written fully. */ @@ -126,6 +130,9 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec ); target = null; return result; + } catch (Exception e) { + logger.error("failure encoding chunk", e); + throw e; } finally { if (target != null) { assert false : "failure encoding chunk"; @@ -212,6 +219,9 @@ public ReleasableBytesReference encodeChunk(int sizeHint, Recycler rec ); currentOutput = null; return result; + } catch (Exception e) { + logger.error("failure encoding text chunk", e); + throw e; } finally { if (currentOutput != null) { assert false : "failure encoding text chunk"; From 1319ad08c30e8dcb88848fab6cc4fddc33e5461b Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 24 Nov 2023 13:53:12 +0100 Subject: [PATCH 30/35] Added beat.stats.libbeat.pipeline.queue.max_events (#102570) --- docs/changelog/102570.yaml | 5 +++++ .../src/main/resources/monitoring-beats-mb.json | 7 +++++++ .../src/main/resources/monitoring-beats.json | 3 +++ .../xpack/monitoring/MonitoringTemplateRegistry.java | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/102570.yaml diff --git a/docs/changelog/102570.yaml b/docs/changelog/102570.yaml new file mode 100644 index 0000000000000..2d3f878dbbb27 --- /dev/null +++ b/docs/changelog/102570.yaml @@ -0,0 +1,5 @@ +pr: 102570 +summary: Added `beat.stats.libbeat.pipeline.queue.max_events` +area: Monitoring +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json index ebe8f4dfbcce1..fab8ca451358f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats-mb.json @@ -838,6 +838,9 @@ "properties": { "acked": { "type": "long" + }, + "max_events": { + "type": "long" } } } @@ -1928,6 +1931,10 @@ "acked": { "type": "alias", "path": "beat.stats.libbeat.pipeline.queue.acked" + }, + "max_events": { + "type": "alias", + "path": "beat.stats.libbeat.pipeline.queue.max_events" } } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json index 14174c6b86dcb..6dee05564cc10 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/monitoring-beats.json @@ -854,6 +854,9 @@ "properties": { "acked": { "type": "long" + }, + "max_events": { + "type": "long" } } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java index ec8bef6e14c24..442cd2479f87c 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @@ -77,7 +77,7 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { * writes monitoring data in ECS format as of 8.0. These templates define the ECS schema as well as alias fields for the old monitoring * mappings that point to the corresponding ECS fields. */ - public static final int STACK_MONITORING_REGISTRY_VERSION = 8_00_00_99 + 10; + public static final int STACK_MONITORING_REGISTRY_VERSION = 8_00_00_99 + 11; private static final String STACK_MONITORING_REGISTRY_VERSION_VARIABLE = "xpack.stack.monitoring.template.release.version"; private static final String STACK_TEMPLATE_VERSION = "8"; private static final String STACK_TEMPLATE_VERSION_VARIABLE = "xpack.stack.monitoring.template.version"; From bba233a00aef4bb2e7c236b737b31709dbf461dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 24 Nov 2023 14:13:24 +0100 Subject: [PATCH 31/35] REST tests: GET features from currently running cluster (#102546) --- .../test/rest/ESRestTestCase.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index cb9e430aaf66a..c1c770c7f99df 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -36,6 +36,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.cluster.ClusterFeatures; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -291,8 +292,7 @@ public void initClient() throws IOException { // TODO (ES-7313): add new ESRestTestCaseHistoricalFeatures() too List.of(new RestTestLegacyFeatures()), semanticNodeVersions, - // TODO (ES-7316): GET and pass cluster state - Set.of() + ClusterFeatures.calculateAllNodeFeatures(getClusterStateFeatures().values()) ); } @@ -2054,6 +2054,25 @@ public void ensurePeerRecoveryRetentionLeasesRenewedAndSynced(String index) thro }, 60, TimeUnit.SECONDS); } + private static Map> getClusterStateFeatures() throws IOException { + final Request request = new Request("GET", "_cluster/state"); + request.addParameter("filter_path", "nodes_features"); + + final Response response = adminClient().performRequest(request); + + var responseData = responseAsMap(response); + if (responseData.get("nodes_features") instanceof List nodesFeatures) { + return nodesFeatures.stream() + .map(Map.class::cast) + .collect(Collectors.toUnmodifiableMap(nodeFeatureMap -> nodeFeatureMap.get("node_id").toString(), nodeFeatureMap -> { + @SuppressWarnings("unchecked") + var nodeFeatures = (List) nodeFeatureMap.get("features"); + return new HashSet<>(nodeFeatures); + })); + } + return Map.of(); + } + /** * Returns the minimum index version among all nodes of the cluster */ From 6f49da29b740f4fd634035108f26928738056523 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 24 Nov 2023 13:38:06 +0000 Subject: [PATCH 32/35] [ML] Add cluster settings to override ML autoscaling (#102413) Add the concept of an autoscaling "dummy" entity, comprised of the xpack.ml.dummy_entity_memory and xpack.ml.dummy_entity_processors settings. These are intended to be used in extreme cases to trigger autoscaling events. --- .../xpack/ml/MachineLearning.java | 21 +- .../MlAutoscalingResourceTracker.java | 112 ++++- .../MlAutoscalingResourceTrackerTests.java | 434 +++++++++++++++++- 3 files changed, 551 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 538f02b3f9092..1afdbe392a742 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -653,6 +653,23 @@ public void loadExtensions(ExtensionLoader loader) { Property.NodeScope ); + // The next two settings currently only have an effect in serverless. They can be set as overrides to + // trigger a scale up of the ML tier so that it could accommodate the dummy entity in addition to + // whatever the standard autoscaling formula thinks is necessary. + public static final Setting DUMMY_ENTITY_MEMORY = Setting.memorySizeSetting( + "xpack.ml.dummy_entity_memory", + ByteSizeValue.ZERO, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public static final Setting DUMMY_ENTITY_PROCESSORS = Setting.intSetting( + "xpack.ml.dummy_entity_processors", + 0, + 0, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + public static final Setting PROCESS_CONNECT_TIMEOUT = Setting.timeSetting( "xpack.ml.process_connect_timeout", TimeValue.timeValueSeconds(10), @@ -782,7 +799,9 @@ public List> getSettings() { NIGHTLY_MAINTENANCE_REQUESTS_PER_SECOND, MachineLearningField.USE_AUTO_MACHINE_MEMORY_PERCENT, MAX_ML_NODE_SIZE, - DELAYED_DATA_CHECK_FREQ + DELAYED_DATA_CHECK_FREQ, + DUMMY_ENTITY_MEMORY, + DUMMY_ENTITY_PROCESSORS ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java index ac6f3914b8f40..e2980e11c2643 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTracker.java @@ -41,6 +41,8 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.core.Tuple.tuple; +import static org.elasticsearch.xpack.ml.MachineLearning.DUMMY_ENTITY_MEMORY; +import static org.elasticsearch.xpack.ml.MachineLearning.DUMMY_ENTITY_PROCESSORS; import static org.elasticsearch.xpack.ml.MachineLearning.MAX_OPEN_JOBS_PER_NODE; import static org.elasticsearch.xpack.ml.job.JobNodeSelector.AWAITING_LAZY_ASSIGNMENT; @@ -58,7 +60,13 @@ static MlJobRequirements of(long memory, int processors, int jobs) { static MlJobRequirements of(long memory, int processors) { return new MlJobRequirements(memory, processors, 1); } - }; + } + + record MlDummyAutoscalingEntity(long memory, int processors) { + static MlDummyAutoscalingEntity of(long memory, int processors) { + return new MlDummyAutoscalingEntity(memory, processors); + } + } private MlAutoscalingResourceTracker() {} @@ -85,6 +93,12 @@ public static void getMlAutoscalingStats( .roundDown() : 0; + MlDummyAutoscalingEntity mlDummyAutoscalingEntity = new MlDummyAutoscalingEntity( + // Treat a ByteSizeValue of -1 as 0, since 0 is the default dummy entity size + Math.max(0L, DUMMY_ENTITY_MEMORY.get(settings).getBytes()), + DUMMY_ENTITY_PROCESSORS.get(settings) + ); + // Todo: MAX_LOW_PRIORITY_MODELS_PER_NODE not checked yet int maxOpenJobsPerNode = MAX_OPEN_JOBS_PER_NODE.get(settings); @@ -95,6 +109,7 @@ public static void getMlAutoscalingStats( modelMemoryAvailableFirstNode, processorsAvailableFirstNode, maxOpenJobsPerNode, + mlDummyAutoscalingEntity, listener ); } @@ -106,6 +121,7 @@ static void getMemoryAndProcessors( long perNodeAvailableModelMemoryInBytes, int perNodeAvailableProcessors, int maxOpenJobsPerNode, + MlDummyAutoscalingEntity dummyAutoscalingEntity, ActionListener listener ) { Map> perNodeModelMemoryInBytes = new HashMap<>(); @@ -262,6 +278,23 @@ static void getMemoryAndProcessors( minNodes = Math.min(3, Math.max(minNodes, numberOfAllocations)); } + // dummy autoscaling entity + if (dummyEntityFitsOnLeastLoadedNode( + perNodeModelMemoryInBytes, + perNodeAvailableModelMemoryInBytes, + perNodeAvailableProcessors, + dummyAutoscalingEntity + ) == false) { + logger.info( + "Scaling up due to dummy entity: dummyEntityMemory: [{}], dummyEntityProcessors: [{}]", + dummyAutoscalingEntity.memory, + dummyAutoscalingEntity.processors + ); + + modelMemoryBytesSum += dummyAutoscalingEntity.memory; + processorsSum += dummyAutoscalingEntity.processors; + } + // check for downscaling long removeNodeMemoryInBytes = 0; @@ -282,7 +315,8 @@ static void getMemoryAndProcessors( perNodeModelMemoryInBytes, perNodeAvailableModelMemoryInBytes, perNodeAvailableProcessors, - maxOpenJobsPerNode + maxOpenJobsPerNode, + dummyAutoscalingEntity ))) { removeNodeMemoryInBytes = perNodeMemoryInBytes; } @@ -304,6 +338,73 @@ static void getMemoryAndProcessors( ); } + /** + * Check if the dummy autoscaling entity task can be added by placing + * the task on the least loaded node. + * + * If there exists a node that can accommodate the dummy entity then return true (nothing to do), + * else return false and increment the memory and processor counts accordingly. + * + * We perform the calculation by identifying the least loaded node in terms of memory + * and determining if the addition of the dummy entity's memory and processor requirements could + * be accommodated on it. + * + * If the calculation returns false then treat the case as for a single trained model job + * that is already assigned, i.e. increment modelMemoryBytesSum and processorsSum appropriately. + * + * @param perNodeJobRequirements per Node lists of requirements + * @param perNodeMemoryInBytes total model memory available on every node + * @param perNodeProcessors total processors on every node + * @param dummyAutoscalingEntity "dummy" entity requirements used to potentially trigger a scaling event + * @return true if the dummy entity can be accommodated, false if not + */ + static boolean dummyEntityFitsOnLeastLoadedNode( + Map> perNodeJobRequirements, // total up requirements... + long perNodeMemoryInBytes, + int perNodeProcessors, + MlDummyAutoscalingEntity dummyAutoscalingEntity + ) { + + if (dummyAutoscalingEntity.processors == 0 && dummyAutoscalingEntity.memory == 0L) { + return true; + } + + if (perNodeJobRequirements.size() < 1) { + return false; + } + + // Note: we check least loaded based _only_ on memory... + Optional leastLoadedNodeRequirements = perNodeJobRequirements.values() + .stream() + .map( + value -> value.stream() + .reduce( + MlJobRequirements.of(0L, 0, 0), + (subtotal, element) -> MlJobRequirements.of( + subtotal.memory + element.memory, + subtotal.processors + element.processors, + subtotal.jobs + element.jobs + ) + ) + ) + .min(Comparator.comparingLong(value -> value.memory)); + + assert (leastLoadedNodeRequirements.isPresent()); + assert leastLoadedNodeRequirements.get().memory >= 0L; + assert leastLoadedNodeRequirements.get().processors >= 0; + + // Check if the dummy entity could be accommodated + if (leastLoadedNodeRequirements.get().memory + dummyAutoscalingEntity.memory > perNodeMemoryInBytes) { + return false; + } + + if (leastLoadedNodeRequirements.get().processors + dummyAutoscalingEntity.processors > perNodeProcessors) { + return false; + } + + return true; + } + /** * Return some autoscaling stats that tell the autoscaler not to change anything, but without making it think an error has occurred. */ @@ -340,7 +441,8 @@ static boolean checkIfOneNodeCouldBeRemoved( Map> perNodeJobRequirements, long perNodeMemoryInBytes, int perNodeProcessors, - int maxOpenJobsPerNode + int maxOpenJobsPerNode, + MlDummyAutoscalingEntity dummyAutoscalingEntity ) { if (perNodeJobRequirements.size() <= 1) { return false; @@ -378,6 +480,10 @@ static boolean checkIfOneNodeCouldBeRemoved( String candidateNode = leastLoadedNodeAndMemoryUsage.get().getKey(); List candidateJobRequirements = perNodeJobRequirements.get(candidateNode); + if (dummyAutoscalingEntity.memory > 0L || dummyAutoscalingEntity.processors > 0) { + candidateJobRequirements = new ArrayList<>(candidateJobRequirements); + candidateJobRequirements.add(MlJobRequirements.of(dummyAutoscalingEntity.memory, dummyAutoscalingEntity.processors)); + } perNodeMlJobRequirementSum.remove(candidateNode); // if all jobs fit on other nodes, we can scale down one node diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java index 42deca32811b2..0d91ce45c46ba 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/autoscaling/MlAutoscalingResourceTrackerTests.java @@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingResourceTracker.MlDummyAutoscalingEntity; import static org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingResourceTracker.MlJobRequirements; import static org.elasticsearch.xpack.ml.job.JobNodeSelector.AWAITING_LAZY_ASSIGNMENT; import static org.mockito.Mockito.mock; @@ -62,6 +63,7 @@ public void testGetMemoryAndProcessors() throws InterruptedException { memory / 2, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { @@ -82,6 +84,7 @@ public void testGetMemoryAndProcessors() throws InterruptedException { memory / 2, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { @@ -92,6 +95,30 @@ public void testGetMemoryAndProcessors() throws InterruptedException { assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); } ); + + // Simulate 1 node & 1 "dummy" task requiring 1 processor and the same memory as the other node + // We don't expect any extra memory or processor usage in this situation. + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-1", randomLongBetween(0, memory), "ml-2", randomLongBetween(0, memory)), + memory / 2, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(memory / 2, 1), + listener + ), + stats -> { + assertEquals(0, stats.perNodeMemoryInBytes()); + assertEquals(2, stats.nodes()); + assertEquals(0, stats.minNodes()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(0, stats.extraModelMemoryInBytes()); + assertEquals(0, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); } public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignment() throws InterruptedException { @@ -144,12 +171,68 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignment() throw memory / 2, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { assertEquals(memory, stats.perNodeMemoryInBytes()); assertEquals(2, stats.nodes()); assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraProcessors()); + assertEquals(0, stats.modelMemoryInBytesSum()); + assertEquals(0, stats.processorsSum()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(memory / 4, stats.extraModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + + // As above but allocate an equal amount of memory to a dummy task + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-1", memory, "ml-2", memory), + memory / 2, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(memory / 4, 0), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(2, stats.nodes()); + assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraProcessors()); + assertEquals(memory / 4, stats.modelMemoryInBytesSum()); + assertEquals(0, stats.processorsSum()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(memory / 4, stats.extraModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + + // As above but also allocate a processor to the dummy task + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-1", memory, "ml-2", memory), + memory / 2, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(memory / 4, 1), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(2, stats.nodes()); + assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraProcessors()); + assertEquals(memory / 4, stats.modelMemoryInBytesSum()); + assertEquals(1, stats.processorsSum()); assertEquals(0, stats.extraSingleNodeProcessors()); assertEquals(memory / 4, stats.extraSingleNodeModelMemoryInBytes()); assertEquals(memory / 4, stats.extraModelMemoryInBytes()); @@ -211,10 +294,12 @@ public void testGetMemoryAndProcessorsScaleUpGivenAwaitingLazyAssignmentButFaile memory / 2, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(memory, stats.removeNodeMemoryInBytes()); assertEquals(2, stats.nodes()); assertEquals(0, stats.minNodes()); assertEquals(0, stats.extraSingleNodeProcessors()); @@ -681,7 +766,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 600L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -704,7 +790,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 600L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -723,7 +810,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 600L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -733,7 +821,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { Collections.emptyMap(), 999L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -758,7 +847,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 1000L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -787,7 +877,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 1000L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -815,7 +906,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 1000L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -843,7 +935,8 @@ public void testCheckIfOneNodeCouldBeRemovedMemoryOnly() { ), 1000L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); } @@ -869,7 +962,8 @@ public void testCheckIfOneNodeCouldBeRemovedProcessorAndMemory() { ), 600L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -893,7 +987,8 @@ public void testCheckIfOneNodeCouldBeRemovedProcessorAndMemory() { ), 600L, 2, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -917,7 +1012,8 @@ public void testCheckIfOneNodeCouldBeRemovedProcessorAndMemory() { ), 600L, 10, - 5 + 5, + MlDummyAutoscalingEntity.of(0L, 0) ) ); @@ -937,7 +1033,8 @@ public void testCheckIfOneNodeCouldBeRemovedProcessorAndMemory() { ), 600L, 10, - MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0) ) ); } @@ -958,6 +1055,29 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept perNodeAvailableModelMemoryInBytes, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(1, stats.nodes()); + assertEquals(0, stats.minNodes()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(memory, stats.removeNodeMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + + // Dummy task should not affect results + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-1", memory), + perNodeAvailableModelMemoryInBytes, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 1), listener ), stats -> { @@ -965,6 +1085,7 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept assertEquals(1, stats.nodes()); assertEquals(0, stats.minNodes()); assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(0, stats.extraProcessors()); assertEquals(memory, stats.removeNodeMemoryInBytes()); assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); } @@ -979,6 +1100,7 @@ public void testGetMemoryAndProcessorsScaleDownToZero() throws InterruptedExcept perNodeAvailableModelMemoryInBytes, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { @@ -1068,6 +1190,7 @@ public void testGetMemoryAndProcessorsScaleDown() throws InterruptedException { perNodeAvailableModelMemoryInBytes, 10, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { @@ -1167,6 +1290,7 @@ public void testGetMemoryAndProcessorsScaleDownPreventedByMinNodes() throws Inte NativeMemoryCalculator.allowedBytesForMl(firstNode, settings).getAsLong(), 4, MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 0), listener ), stats -> { @@ -1180,6 +1304,292 @@ public void testGetMemoryAndProcessorsScaleDownPreventedByMinNodes() throws Inte ); } + // scenario: 3 ml nodes, but only 2 have assigned models. This situation would normally result in a scale down but that is prevented + // by a "dummy" entity having sufficient memory to do so. + public void testGetMemoryAndProcessorsScaleDownPreventedByDummyEntityMemory() throws InterruptedException { + Map nodeAttr = Map.of( + MachineLearning.MACHINE_MEMORY_NODE_ATTR, + "1000000000", + MachineLearning.MAX_JVM_SIZE_NODE_ATTR, + "400000000", + MachineLearning.ML_CONFIG_VERSION_NODE_ATTR, + "7.2.0" + ); + + MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext( + List.of(), + List.of(), + List.of(), + Map.of( + "model-1", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-1", + "model-1-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build(), + "model-2", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-2", + "model-2-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-3", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build() + ), + List.of( + DiscoveryNodeUtils.builder("ml-node-1") + .name("ml-node-name-1") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build(), + DiscoveryNodeUtils.builder("ml-node-3") + .name("ml-node-name-3") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build() + ), + PersistentTasksCustomMetadata.builder().build() + ); + MlMemoryTracker mockTracker = mock(MlMemoryTracker.class); + + long memory = 1000000000; + long perNodeAvailableModelMemoryInBytes = 600000000; + + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-node-1", memory, "ml-node-2", memory, "ml-node-3", memory), + perNodeAvailableModelMemoryInBytes, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(perNodeAvailableModelMemoryInBytes, 1), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(perNodeAvailableModelMemoryInBytes + 503318080, stats.modelMemoryInBytesSum()); // total model memory is that + // configured in the dummy + // entity plus that used by the + // trained models. + assertEquals(5, stats.processorsSum()); // account for the extra processor from the dummy entity + assertEquals(3, stats.nodes()); + assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(0, stats.extraProcessors()); + assertEquals(0, stats.extraModelMemoryInBytes()); + assertEquals(0, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + } + + // scenario: 3 ml nodes, but only 2 have assigned models. This situation does result in a scale down since dummy + // processors alone are not sufficient to prevent it. + public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityProcessors() throws InterruptedException { + Map nodeAttr = Map.of( + MachineLearning.MACHINE_MEMORY_NODE_ATTR, + "1000000000", + MachineLearning.MAX_JVM_SIZE_NODE_ATTR, + "400000000", + MachineLearning.ML_CONFIG_VERSION_NODE_ATTR, + "7.2.0" + ); + + MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext( + List.of(), + List.of(), + List.of(), + Map.of( + "model-1", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-1", + "model-1-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build(), + "model-2", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-2", + "model-2-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-3", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build() + ), + List.of( + DiscoveryNodeUtils.builder("ml-node-1") + .name("ml-node-name-1") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build(), + DiscoveryNodeUtils.builder("ml-node-3") + .name("ml-node-name-3") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build() + ), + PersistentTasksCustomMetadata.builder().build() + ); + MlMemoryTracker mockTracker = mock(MlMemoryTracker.class); + + long memory = 1000000000; + long perNodeAvailableModelMemoryInBytes = 600000000; + + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-node-1", memory, "ml-node-2", memory, "ml-node-3", memory), + perNodeAvailableModelMemoryInBytes, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(0L, 9), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(503318080, stats.modelMemoryInBytesSum()); + assertEquals(13, stats.processorsSum()); // account for the extra processors from the dummy entity + assertEquals(3, stats.nodes()); + assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(0, stats.extraProcessors()); + assertEquals(0, stats.extraModelMemoryInBytes()); + assertEquals(0, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + } + + public void testGetMemoryAndProcessorsScaleDownNotPreventedByDummyEntityAsMemoryTooLow() throws InterruptedException { + Map nodeAttr = Map.of( + MachineLearning.MACHINE_MEMORY_NODE_ATTR, + "1000000000", + MachineLearning.MAX_JVM_SIZE_NODE_ATTR, + "400000000", + MachineLearning.ML_CONFIG_VERSION_NODE_ATTR, + "7.2.0" + ); + + MlAutoscalingContext mlAutoscalingContext = new MlAutoscalingContext( + List.of(), + List.of(), + List.of(), + Map.of( + "model-1", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-1", + "model-1-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-1", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build(), + "model-2", + TrainedModelAssignment.Builder.empty( + new StartTrainedModelDeploymentAction.TaskParams( + "model-2", + "model-2-deployment", + 400, + 1, + 2, + 100, + null, + Priority.NORMAL, + 0L, + 0L + ) + ).addRoutingEntry("ml-node-3", new RoutingInfo(1, 1, RoutingState.STARTED, "")).build() + ), + List.of( + DiscoveryNodeUtils.builder("ml-node-1") + .name("ml-node-name-1") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build(), + DiscoveryNodeUtils.builder("ml-node-3") + .name("ml-node-name-3") + .address(new TransportAddress(InetAddress.getLoopbackAddress(), 9300)) + .attributes(nodeAttr) + .roles(Set.of(DiscoveryNodeRole.ML_ROLE)) + .build() + ), + PersistentTasksCustomMetadata.builder().build() + ); + MlMemoryTracker mockTracker = mock(MlMemoryTracker.class); + + long memory = 1000000000; + long perNodeAvailableModelMemoryInBytes = 600000000; + + this.assertAsync( + listener -> MlAutoscalingResourceTracker.getMemoryAndProcessors( + mlAutoscalingContext, + mockTracker, + Map.of("ml-node-1", memory, "ml-node-2", memory, "ml-node-3", memory), + perNodeAvailableModelMemoryInBytes, + 10, + MachineLearning.DEFAULT_MAX_OPEN_JOBS_PER_NODE, + MlDummyAutoscalingEntity.of(1024, 0), + listener + ), + stats -> { + assertEquals(memory, stats.perNodeMemoryInBytes()); + assertEquals(503318080, stats.modelMemoryInBytesSum()); + assertEquals(4, stats.processorsSum()); + assertEquals(3, stats.nodes()); + assertEquals(1, stats.minNodes()); + assertEquals(0, stats.extraSingleNodeProcessors()); + assertEquals(0, stats.extraProcessors()); + assertEquals(0, stats.extraModelMemoryInBytes()); + assertEquals(0, stats.extraSingleNodeModelMemoryInBytes()); + assertEquals(MachineLearning.NATIVE_EXECUTABLE_CODE_OVERHEAD.getBytes(), stats.perNodeMemoryOverheadInBytes()); + } + ); + } + private void assertAsync(Consumer> function, Consumer furtherTests) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); AtomicBoolean listenerCalled = new AtomicBoolean(false); From 0e304e273aa04739495bfac97134b2c45fa42f31 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 24 Nov 2023 15:02:28 +0100 Subject: [PATCH 33/35] Additional minor cleanup of org.elasticsearch.search.aggregations package (#102293) * Convert Collections.sort() to List.sort() * Use Collections.addAll() instead of manually invoking add() * Use Map.merge() / Map.computIfAbsent() * Use primitive double over Double * Replace some lambdas with method references. * Replaces for loops with index variable to use foreach iteration style for loops. --- .../search/aggregations/AggregatorBase.java | 4 ++-- .../search/aggregations/InternalOrder.java | 2 +- .../aggregations/bucket/BucketsAggregator.java | 10 +++++----- .../composite/CompositeValuesCollectorQueue.java | 16 ++++++++-------- .../bucket/composite/ParsedComposite.java | 2 +- .../geogrid/GeoGridAggregationBuilder.java | 2 +- .../bucket/range/RangeAggregator.java | 3 +-- .../bucket/terms/AbstractInternalTerms.java | 2 +- .../bucket/terms/InternalSignificantTerms.java | 6 +----- .../terms/StringTermsAggregatorFromFilters.java | 4 ++-- .../metrics/AbstractHyperLogLogPlusPlus.java | 7 +------ .../metrics/AbstractInternalHDRPercentiles.java | 14 +++++++------- .../AbstractInternalTDigestPercentiles.java | 14 +++++++------- .../aggregations/metrics/ParsedGeoBounds.java | 6 +----- .../metrics/ParsedHDRPercentiles.java | 2 +- .../metrics/ParsedPercentileRanks.java | 2 +- .../metrics/ParsedTDigestPercentiles.java | 2 +- .../metrics/TopHitsAggregationBuilder.java | 4 +--- .../pipeline/InternalPercentilesBucket.java | 2 +- .../aggregations/pipeline/MovingFunctions.java | 2 +- .../pipeline/ParsedPercentilesBucket.java | 2 +- 21 files changed, 46 insertions(+), 62 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java index 88c893c1d1f19..795f51a729ed6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java @@ -268,8 +268,8 @@ public Aggregator[] subAggregators() { public Aggregator subAggregator(String aggName) { if (subAggregatorbyName == null) { subAggregatorbyName = Maps.newMapWithExpectedSize(subAggregators.length); - for (int i = 0; i < subAggregators.length; i++) { - subAggregatorbyName.put(subAggregators[i].name(), subAggregators[i]); + for (Aggregator subAggregator : subAggregators) { + subAggregatorbyName.put(subAggregator.name(), subAggregator); } } return subAggregatorbyName.get(aggName); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java index ecfeff45140fb..15d4a03be81f2 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java @@ -289,7 +289,7 @@ byte id() { @Override public Comparator partiallyBuiltBucketComparator(ToLongFunction ordinalReader, Aggregator aggregator) { Comparator comparator = comparator(); - return (lhs, rhs) -> comparator.compare(lhs, rhs); + return comparator::compare; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java index 06456b2396522..ec8117cf03135 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketsAggregator.java @@ -223,9 +223,9 @@ protected final void buildSubAggsForAllBuckets( } InternalAggregations[] results = buildSubAggsForBuckets(bucketOrdsToCollect); s = 0; - for (int r = 0; r < buckets.length; r++) { - for (int b = 0; b < buckets[r].length; b++) { - setAggs.accept(buckets[r][b], results[s++]); + for (B[] bucket : buckets) { + for (int b = 0; b < bucket.length; b++) { + setAggs.accept(bucket[b], results[s++]); } } } @@ -330,8 +330,8 @@ protected final InternalAggregation[] buildAggregationsForVariableBuckets( } long[] bucketOrdsToCollect = new long[(int) totalOrdsToCollect]; int b = 0; - for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { - LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]); + for (long owningBucketOrd : owningBucketOrds) { + LongKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrd); while (ordsEnum.next()) { bucketOrdsToCollect[b++] = ordsEnum.ord(); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java index a6aab3c52a2a3..0a7f6a26f580b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java @@ -83,7 +83,7 @@ private interface CompetitiveBoundsChangedListener { // tracking the highest competitive value. if (arrays[0] instanceof GlobalOrdinalValuesSource globalOrdinalValuesSource) { if (shouldApplyGlobalOrdinalDynamicPruningForLeadingSource(sources, size, indexReader)) { - competitiveBoundsChangedListener = topSlot -> globalOrdinalValuesSource.updateHighestCompetitiveValue(topSlot); + competitiveBoundsChangedListener = globalOrdinalValuesSource::updateHighestCompetitiveValue; } else { competitiveBoundsChangedListener = null; } @@ -207,8 +207,8 @@ long getDocCount(int slot) { * Copies the current value in slot. */ private void copyCurrent(int slot, long value) { - for (int i = 0; i < arrays.length; i++) { - arrays[i].copyCurrent(slot); + for (SingleDimensionValuesSource array : arrays) { + array.copyCurrent(slot); } docCounts = bigArrays.grow(docCounts, slot + 1); docCounts.set(slot, value); @@ -238,12 +238,12 @@ int compare(int slot1, int slot2) { */ boolean equals(int slot1, int slot2) { assert slot2 != CANDIDATE_SLOT; - for (int i = 0; i < arrays.length; i++) { + for (SingleDimensionValuesSource array : arrays) { final int cmp; if (slot1 == CANDIDATE_SLOT) { - cmp = arrays[i].compareCurrent(slot2); + cmp = array.compareCurrent(slot2); } else { - cmp = arrays[i].compare(slot1, slot2); + cmp = array.compare(slot1, slot2); } if (cmp != 0) { return false; @@ -257,8 +257,8 @@ boolean equals(int slot1, int slot2) { */ int hashCode(int slot) { int result = 1; - for (int i = 0; i < arrays.length; i++) { - result = 31 * result + (slot == CANDIDATE_SLOT ? arrays[i].hashCodeCurrent() : arrays[i].hashCode(slot)); + for (SingleDimensionValuesSource array : arrays) { + result = 31 * result + (slot == CANDIDATE_SLOT ? array.hashCodeCurrent() : array.hashCode(slot)); } return result; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/ParsedComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/ParsedComposite.java index 0c3ec9a521b8e..847e35cf0d4ea 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/ParsedComposite.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/ParsedComposite.java @@ -32,7 +32,7 @@ public class ParsedComposite extends ParsedMultiBucketAggregation ParsedComposite.ParsedBucket.fromXContent(parser), parser -> null); + declareMultiBucketAggregationFields(PARSER, ParsedBucket::fromXContent, parser -> null); } private Map afterKey; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index ff9495ca4d825..8bc1e3d17642a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -64,7 +64,7 @@ public static ObjectParser crea parser.declareInt(GeoGridAggregationBuilder::size, FIELD_SIZE); parser.declareInt(GeoGridAggregationBuilder::shardSize, FIELD_SHARD_SIZE); parser.declareField( - (p, builder, context) -> { builder.setGeoBoundingBox(GeoBoundingBox.parseBoundingBox(p)); }, + (p, builder, context) -> builder.setGeoBoundingBox(GeoBoundingBox.parseBoundingBox(p)), GeoBoundingBox.BOUNDS_FIELD, org.elasticsearch.xcontent.ObjectParser.ValueType.OBJECT ); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java index b47259360f263..7d7e1a1a03bc4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java @@ -555,8 +555,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I public InternalAggregation buildEmptyAggregation() { InternalAggregations subAggs = buildEmptySubAggregations(); List buckets = new ArrayList<>(ranges.length); - for (int i = 0; i < ranges.length; i++) { - Range range = ranges[i]; + for (Range range : ranges) { org.elasticsearch.search.aggregations.bucket.range.Range.Bucket bucket = rangeFactory.createBucket( range.key, range.originalFrom, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java index 3f4ceda326140..aed2119dec483 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java @@ -300,7 +300,7 @@ public InternalAggregation reduce(List aggregations, Aggreg TopBucketBuilder top = TopBucketBuilder.build( getRequiredSize(), getOrder(), - removed -> { otherDocCount[0] += removed.getDocCount(); } + removed -> otherDocCount[0] += removed.getDocCount() ); thisReduceOrder = reduceBuckets(aggregations, reduceContext, bucket -> { if (bucket.getDocCount() >= getMinDocCount()) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java index cffe11c5729eb..07aa318e9c487 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalSignificantTerms.java @@ -213,11 +213,7 @@ public InternalAggregation reduce(List aggregations, Aggreg @SuppressWarnings("unchecked") InternalSignificantTerms terms = (InternalSignificantTerms) aggregation; for (B bucket : terms.getBuckets()) { - List existingBuckets = buckets.get(bucket.getKeyAsString()); - if (existingBuckets == null) { - existingBuckets = new ArrayList<>(aggregations.size()); - buckets.put(bucket.getKeyAsString(), existingBuckets); - } + List existingBuckets = buckets.computeIfAbsent(bucket.getKeyAsString(), k -> new ArrayList<>(aggregations.size())); // Adjust the buckets with the global stats representing the // total size of the pots from which the stats are drawn existingBuckets.add( diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java index 949685ec29487..d1458b04f17a1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregatorFromFilters.java @@ -186,7 +186,7 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) { for (OrdBucket b : queue) { buckets.add(buildBucket(b, terms)); } - Collections.sort(buckets, reduceOrder.comparator()); + buckets.sort(reduceOrder.comparator()); } else { /* * Note for the curious: you can just use a for loop to iterate @@ -208,7 +208,7 @@ protected boolean lessThan(OrdBucket a, OrdBucket b) { } buckets.add(buildBucket(b, terms)); } - Collections.sort(buckets, reduceOrder.comparator()); + buckets.sort(reduceOrder.comparator()); } return new StringTerms( filters.getName(), diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java index edd1b42668697..abe4987573cb9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractHyperLogLogPlusPlus.java @@ -81,12 +81,7 @@ private Object getComparableData(long bucketOrd) { AbstractHyperLogLog.RunLenIterator iterator = getHyperLogLog(bucketOrd); while (iterator.next()) { byte runLength = iterator.value(); - Integer numOccurances = values.get(runLength); - if (numOccurances == null) { - values.put(runLength, 1); - } else { - values.put(runLength, numOccurances + 1); - } + values.merge(runLength, 1, Integer::sum); } return values; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java index 9b4656ee7cf7e..a77cd495f87db 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java @@ -121,7 +121,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return Arrays.stream(getKeys()).mapToObj(d -> String.valueOf(d)).toList(); + return Arrays.stream(getKeys()).mapToObj(String::valueOf).toList(); } public DocValueFormat formatter() { @@ -210,9 +210,9 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th DoubleHistogram state = getState(); if (keyed) { builder.startObject(CommonFields.VALUES.getPreferredName()); - for (int i = 0; i < keys.length; ++i) { - String key = String.valueOf(keys[i]); - double value = value(keys[i]); + for (double v : keys) { + String key = String.valueOf(v); + double value = value(v); builder.field(key, state.getTotalCount() == 0 ? null : value); if (format != DocValueFormat.RAW && state.getTotalCount() > 0) { builder.field(key + "_as_string", format.format(value).toString()); @@ -221,10 +221,10 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.endObject(); } else { builder.startArray(CommonFields.VALUES.getPreferredName()); - for (int i = 0; i < keys.length; i++) { - double value = value(keys[i]); + for (double key : keys) { + double value = value(key); builder.startObject(); - builder.field(CommonFields.KEY.getPreferredName(), keys[i]); + builder.field(CommonFields.KEY.getPreferredName(), key); builder.field(CommonFields.VALUE.getPreferredName(), state.getTotalCount() == 0 ? null : value); if (format != DocValueFormat.RAW && state.getTotalCount() > 0) { builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java index 08588473c61d1..3ae609689ed7a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java @@ -101,7 +101,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return Arrays.stream(getKeys()).mapToObj(d -> String.valueOf(d)).toList(); + return Arrays.stream(getKeys()).mapToObj(String::valueOf).toList(); } public abstract double value(double key); @@ -188,9 +188,9 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th TDigestState state = getState(); if (keyed) { builder.startObject(CommonFields.VALUES.getPreferredName()); - for (int i = 0; i < keys.length; ++i) { - String key = String.valueOf(keys[i]); - double value = value(keys[i]); + for (double v : keys) { + String key = String.valueOf(v); + double value = value(v); builder.field(key, state.size() == 0 ? null : value); if (format != DocValueFormat.RAW && state.size() > 0) { builder.field(key + "_as_string", format.format(value).toString()); @@ -199,10 +199,10 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th builder.endObject(); } else { builder.startArray(CommonFields.VALUES.getPreferredName()); - for (int i = 0; i < keys.length; i++) { - double value = value(keys[i]); + for (double key : keys) { + double value = value(key); builder.startObject(); - builder.field(CommonFields.KEY.getPreferredName(), keys[i]); + builder.field(CommonFields.KEY.getPreferredName(), key); builder.field(CommonFields.VALUE.getPreferredName(), state.size() == 0 ? null : value); if (format != DocValueFormat.RAW && state.size() > 0) { builder.field(CommonFields.VALUE_AS_STRING.getPreferredName(), format.format(value).toString()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedGeoBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedGeoBounds.java index fed48ec7640e3..24f68d87802bf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedGeoBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedGeoBounds.java @@ -80,11 +80,7 @@ public GeoPoint bottomRight() { static { declareAggregationFields(PARSER); - PARSER.declareObject( - (agg, bbox) -> { agg.geoBoundingBox = new GeoBoundingBox(bbox.v1(), bbox.v2()); }, - BOUNDS_PARSER, - BOUNDS_FIELD - ); + PARSER.declareObject((agg, bbox) -> agg.geoBoundingBox = new GeoBoundingBox(bbox.v1(), bbox.v2()), BOUNDS_PARSER, BOUNDS_FIELD); BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, TOP_LEFT_FIELD); BOUNDS_PARSER.declareObject(constructorArg(), GEO_POINT_PARSER, BOTTOM_RIGHT_FIELD); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedHDRPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedHDRPercentiles.java index 59deb90c7e5a2..2ee56bf648dcc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedHDRPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedHDRPercentiles.java @@ -52,6 +52,6 @@ public double value(String name) { @Override public Iterable valueNames() { - return percentiles.keySet().stream().map(d -> d.toString()).toList(); + return percentiles.keySet().stream().map(Object::toString).toList(); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedPercentileRanks.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedPercentileRanks.java index 0bf317c36be16..44ecf5cf69b4c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedPercentileRanks.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedPercentileRanks.java @@ -31,7 +31,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return percentiles.keySet().stream().map(d -> d.toString()).toList(); + return percentiles.keySet().stream().map(Object::toString).toList(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedTDigestPercentiles.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedTDigestPercentiles.java index 78001e3c65534..b5ab17ba335c3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedTDigestPercentiles.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ParsedTDigestPercentiles.java @@ -37,7 +37,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return percentiles.keySet().stream().map(d -> d.toString()).toList(); + return percentiles.keySet().stream().map(Object::toString).toList(); } private static final ObjectParser PARSER = new ObjectParser<>( diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index d7113fc6ec798..00db45e2d06b4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -272,9 +272,7 @@ public TopHitsAggregationBuilder sorts(List> sorts) { if (this.sorts == null) { this.sorts = new ArrayList<>(); } - for (SortBuilder sort : sorts) { - this.sorts.add(sort); - } + this.sorts.addAll(sorts); return this; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java index bef9b64c6e95b..c763ea5cf2bd3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalPercentilesBucket.java @@ -126,7 +126,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return Arrays.stream(percents).mapToObj(d -> String.valueOf(d)).toList(); + return Arrays.stream(percents).mapToObj(String::valueOf).toList(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java index 0b982f8f2e586..53bf09329c57b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java @@ -174,7 +174,7 @@ public static double holt(double[] values, double alpha, double beta) { int counter = 0; - Double last; + double last; for (double v : values) { if (Double.isNaN(v) == false) { last = v; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedPercentilesBucket.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedPercentilesBucket.java index f9e037247bf2c..7da76d2d4c2eb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedPercentilesBucket.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/ParsedPercentilesBucket.java @@ -57,7 +57,7 @@ public double value(String name) { @Override public Iterable valueNames() { - return percentiles.keySet().stream().map(d -> d.toString()).toList(); + return percentiles.keySet().stream().map(Object::toString).toList(); } @Override From fa416c4b655782dcc723a4b6e580facb10d1922c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 24 Nov 2023 15:27:03 +0100 Subject: [PATCH 34/35] Unmute DownsampleActionIT#testTsdbDataStreams test (#102334) The likely cause is that the downsample step wasn't executed, because the index.time_series.end_time index setting was after current time. Two tweaks to avoid this failure: * Lower index.look_ahead_time * Increase wait time in waitAndGetRollupIndexName() Closes #100271 --- .../xpack/ilm/actions/DownsampleActionIT.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index 4f1efbbca387c..2e61b6e978b61 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -80,6 +80,7 @@ public class DownsampleActionIT extends ESRestTestCase { }, "routing_path": ["metricset"], "mode": "time_series", + "look_ahead_time": "1m", "lifecycle.name": "%s" } }, @@ -301,7 +302,6 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { }); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/100271") public void testTsdbDataStreams() throws Exception { // Create the ILM policy DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); @@ -337,7 +337,10 @@ public void testTsdbDataStreams() throws Exception { rolloverMaxOneDocCondition(client(), dataStream); String rollupIndex = waitAndGetRollupIndexName(client(), backingIndexName, fixedInterval); - assertNotNull("Cannot retrieve rollup index name", rollupIndex); + if (rollupIndex == null) { + logger.warn("explain:" + explainIndex(client(), backingIndexName)); + } + assertNotNull(String.format(Locale.ROOT, "Cannot retrieve rollup index [%s]", rollupIndex), rollupIndex); assertBusy(() -> assertTrue("Rollup index does not exist", indexExists(rollupIndex)), 30, TimeUnit.SECONDS); assertBusy(() -> assertFalse("Source index should have been deleted", indexExists(backingIndexName)), 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -611,7 +614,7 @@ public String waitAndGetRollupIndexName(RestClient client, String originalIndexN } catch (IOException e) { return false; } - }, 60, TimeUnit.SECONDS); + }, 120, TimeUnit.SECONDS); // High timeout in case we're unlucky and end_time has been increased. logger.info("--> original index name is [{}], rollup index name is [{}]", originalIndexName, rollupIndexName[0]); return rollupIndexName[0]; } From d44e05d2c24c77932f8895cadc715c6afdbaa6ee Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Fri, 24 Nov 2023 15:27:41 +0100 Subject: [PATCH 35/35] [DOCS] Add ES quickstart (#102226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [DOCS] TEST restore quickstart * Use up to date Docker instructions, minor user-friendly modifications * Use books dataset, update verbiage, add examples * Update verbiage * Updated Elasticsearch 'Getting Started' docs: added SSL, Docker setup, Python resources, and expanded next steps * minor formatting * Collapse responses, TODO comment tests * Add request tests * Edit superfluities * Apply suggestions Co-authored-by: István Zoltán Szabó * Update docs/reference/tab-widgets/quick-start-install.asciidoc Co-authored-by: István Zoltán Szabó --------- Co-authored-by: István Zoltán Szabó --- docs/reference/getting-started.asciidoc | 285 ++++++++++++++++++ docs/reference/index.asciidoc | 2 + docs/reference/redirects.asciidoc | 5 - .../tab-widgets/api-call-widget.asciidoc | 40 +++ docs/reference/tab-widgets/api-call.asciidoc | 57 ++++ docs/reference/tab-widgets/code.asciidoc | 163 ++++++++++ .../quick-start-install-widget.asciidoc | 40 +++ .../tab-widgets/quick-start-install.asciidoc | 71 +++++ 8 files changed, 658 insertions(+), 5 deletions(-) create mode 100644 docs/reference/getting-started.asciidoc create mode 100644 docs/reference/tab-widgets/api-call-widget.asciidoc create mode 100644 docs/reference/tab-widgets/api-call.asciidoc create mode 100644 docs/reference/tab-widgets/code.asciidoc create mode 100644 docs/reference/tab-widgets/quick-start-install-widget.asciidoc create mode 100644 docs/reference/tab-widgets/quick-start-install.asciidoc diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc new file mode 100644 index 0000000000000..3e474953a72f9 --- /dev/null +++ b/docs/reference/getting-started.asciidoc @@ -0,0 +1,285 @@ +[chapter] +[[getting-started]] += Quick start + +This guide helps you learn how to: + +* install and run {es} and {kib} (using {ecloud} or Docker), +* add simple (non-timestamped) dataset to {es}, +* run basic searches. + +[TIP] +==== +If you're interested in using {es} with Python, check out Elastic Search Labs. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG). + +* https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: this walks you through building a complete search solution with {es}, from the ground up. +* https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: it contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. +==== + +[discrete] +[[run-elasticsearch]] +=== Run {es} + +The simplest way to set up {es} is to create a managed deployment with {ess} on +{ecloud}. If you prefer to manage your own test environment, install and +run {es} using Docker. + +include::{es-repo-dir}/tab-widgets/code.asciidoc[] +include::{es-repo-dir}/tab-widgets/quick-start-install-widget.asciidoc[] + +[discrete] +[[send-requests-to-elasticsearch]] +=== Send requests to {es} + +You send data and other requests to {es} using REST APIs. This lets you interact +with {es} using any client that sends HTTP requests, such as +https://curl.se[curl]. You can also use {kib}'s Console to send requests to +{es}. + +include::{es-repo-dir}/tab-widgets/api-call-widget.asciidoc[] + +[discrete] +[[add-data]] +=== Add data + +You add data to {es} as JSON objects called documents. {es} stores these +documents in searchable indices. + +[discrete] +[[add-single-document]] +==== Add a single document + +Submit the following indexing request to add a single document to the +`books` index. +The request automatically creates the index. + +//// +[source,console] +---- +PUT books +---- +// TESTSETUP +//// + +[source,console] +---- +POST books/_doc +{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} +---- +// TEST[s/_doc/_doc?refresh=wait_for/] + +The response includes metadata that {es} generates for the document including a unique `_id` for the document within the index. + +.Expand to see example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "_index": "books", + "_id": "O0lG2IsBaSa7VYx_rEia", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 0, + "_primary_term": 1 +} +---- +// TEST[skip:TODO] +=============== + +[discrete] +[[add-multiple-documents]] +==== Add multiple documents + +Use the `_bulk` endpoint to add multiple documents in one request. Bulk data +must be newline-delimited JSON (NDJSON). Each line must end in a newline +character (`\n`), including the last line. + +[source,console] +---- +POST /_bulk +{ "index" : { "_index" : "books" } } +{"name": "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} +{ "index" : { "_index" : "books" } } +{"name": "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} +{ "index" : { "_index" : "books" } } +{"name": "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} +{ "index" : { "_index" : "books" } } +{"name": "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} +{ "index" : { "_index" : "books" } } +{"name": "The Handmaids Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} +---- +// TEST[continued] + +You should receive a response indicating there were no errors. + +.Expand to see example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "errors": false, + "took": 29, + "items": [ + { + "index": { + "_index": "books", + "_id": "QklI2IsBaSa7VYx_Qkh-", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "books", + "_id": "Q0lI2IsBaSa7VYx_Qkh-", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "books", + "_id": "RElI2IsBaSa7VYx_Qkh-", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 3, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "books", + "_id": "RUlI2IsBaSa7VYx_Qkh-", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 4, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "books", + "_id": "RklI2IsBaSa7VYx_Qkh-", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 5, + "_primary_term": 1, + "status": 201 + } + } + ] +} +---- +// TEST[skip:TODO] +=============== + +[discrete] +[[qs-search-data]] +=== Search data + +Indexed documents are available for search in near real-time. + +[discrete] +[[search-all-documents]] +==== Search all documents + +Run the following command to search the `books` index for all documents: +[source,console] +---- +GET books/_search +---- +// TEST[continued] + +The `_source` of each hit contains the original +JSON object submitted during indexing. + +[discrete] +[[qs-match-query]] +==== `match` query + +You can use the `match` query to search for documents that contain a specific value in a specific field. +This is the standard query for performing full-text search, including fuzzy matching and phrase searches. + +Run the following command to search the `books` index for documents containing `brave` in the `name` field: +[source,console] +---- +GET books/_search +{ + "query": { + "match": { + "name": "brave" + } + } +} +---- +// TEST[continued] + +[discrete] +[[whats-next]] +=== Next steps + +Now that {es} is up and running and you've learned the basics, you'll probably want to test out larger datasets, or index your own data. + +[discrete] +[[whats-next-search-learn-more]] +==== Learn more about search queries + +* <>. Jump here to learn about exact value search, full-text search, vector search, and more, using the <>. + +[discrete] +[[whats-next-more-data]] +==== Add more data + +* Learn how to {kibana-ref}/sample-data.html[install sample data] using {kib}. This is a quick way to test out {es} on larger workloads. +* Learn how to use the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[upload data UI] in {kib} to add your own CSV, TSV, or JSON files. +* Use the https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[bulk API] to ingest your own datasets to {es}. + +[discrete] +[[whats-next-client-libraries]] +==== {es} programming language clients + +* Check out our https://www.elastic.co/guide/en/elasticsearch/client/index.html[client library] to work with your {es} instance in your preferred programming language. +* If you're using Python, check out https://www.elastic.co/search-labs[Elastic Search Labs] for a range of examples that use the {es} Python client. This is the best place to explore AI-powered search use cases, such as working with embeddings, vector search, and retrieval augmented generation (RAG). +** This extensive, hands-on https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[tutorial] +walks you through building a complete search solution with {es}, from the ground up. +** https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs`] contains a range of executable Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. \ No newline at end of file diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 828a3e4d1d01d..b09d67e990636 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -17,6 +17,8 @@ include::intro.asciidoc[] include::release-notes/highlights.asciidoc[] +include::getting-started.asciidoc[] + include::setup.asciidoc[] include::upgrade.asciidoc[] diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index f065c2deeae72..e0568f500f268 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -1720,11 +1720,6 @@ See <>. See <>. -[role="exclude",id="getting-started"] -=== Quick start - -See {estc-welcome}/getting-started-general-purpose.html[Set up a general purpose Elastic deployment]. - [role="exclude",id="getting-started-index"] === Index some documents diff --git a/docs/reference/tab-widgets/api-call-widget.asciidoc b/docs/reference/tab-widgets/api-call-widget.asciidoc new file mode 100644 index 0000000000000..adc2aa86f1c0e --- /dev/null +++ b/docs/reference/tab-widgets/api-call-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::api-call.asciidoc[tag=cloud] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/api-call.asciidoc b/docs/reference/tab-widgets/api-call.asciidoc new file mode 100644 index 0000000000000..ecbd49eae7f8f --- /dev/null +++ b/docs/reference/tab-widgets/api-call.asciidoc @@ -0,0 +1,57 @@ +// tag::cloud[] +**Use {kib}** + +//tag::kibana-api-ex[] +. Open {kib}'s main menu ("*☰*" near Elastic logo) and go to **Dev Tools > Console**. ++ +[role="screenshot"] +image::images/kibana-console.png[{kib} Console,align="center"] + +. Run the following test API request in Console: ++ +[source,console] +---- +GET / +---- + +//end::kibana-api-ex[] + +**Use curl** + +To communicate with {es} using curl or another client, you need your cluster's +endpoint. + +. Open {kib}'s main menu and click **Manage this deployment**. + +. From your deployment menu, go to the **Elasticsearch** page. Click **Copy +endpoint**. + +. To submit an example API request, run the following curl command in a new +terminal session. Replace `` with the password for the `elastic` user. +Replace `` with your endpoint. ++ +[source,sh] +---- +curl -u elastic: / +---- +// NOTCONSOLE + +// end::cloud[] + +// tag::self-managed[] +**Use {kib}** + +include::api-call.asciidoc[tag=kibana-api-ex] + +**Use curl** + +To submit an example API request, run the following curl command in a new +terminal session. + +[source,sh] +---- +curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 +---- +// NOTCONSOLE + +// end::self-managed[] \ No newline at end of file diff --git a/docs/reference/tab-widgets/code.asciidoc b/docs/reference/tab-widgets/code.asciidoc new file mode 100644 index 0000000000000..a6949b681edc6 --- /dev/null +++ b/docs/reference/tab-widgets/code.asciidoc @@ -0,0 +1,163 @@ +// Defining styles and script here for simplicity. +++++ + + +++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/quick-start-install-widget.asciidoc b/docs/reference/tab-widgets/quick-start-install-widget.asciidoc new file mode 100644 index 0000000000000..f3ff804ade255 --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-install-widget.asciidoc @@ -0,0 +1,40 @@ +++++ +
+
+ + +
+
+++++ + +include::quick-start-install.asciidoc[tag=cloud] + +++++ +
+ +
+++++ \ No newline at end of file diff --git a/docs/reference/tab-widgets/quick-start-install.asciidoc b/docs/reference/tab-widgets/quick-start-install.asciidoc new file mode 100644 index 0000000000000..b8daf62dad63b --- /dev/null +++ b/docs/reference/tab-widgets/quick-start-install.asciidoc @@ -0,0 +1,71 @@ + +// tag::cloud[] +include::{docs-root}/shared/cloud/ess-getting-started.asciidoc[tag=generic] + +. Click **Continue** to open {kib}, the user interface for {ecloud}. + +. Click **Explore on my own**. +// end::cloud[] + +// tag::self-managed[] +*Start a single-node cluster* + +We'll use a single-node {es} cluster in this quick start, which makes sense for testing and development. +Refer to <> for advanced Docker documentation. + +. Run the following Docker commands: ++ +[source,sh,subs="attributes"] +---- +docker network create elastic +docker pull {docker-image} +docker run --name es01 --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -t {docker-image} +---- + +. Copy the generated `elastic` password and enrollment token, which are output to your terminal. +You'll use these to enroll {kib} with your {es} cluster and log in. +These credentials are only shown when you start {es} for the first time. ++ +We recommend storing the `elastic` password as an environment variable in your shell. Example: ++ +[source,sh] +---- +export ELASTIC_PASSWORD="your_password" +---- ++ +. Copy the `http_ca.crt` SSL certificate from the container to your local machine. ++ +[source,sh] +---- +docker cp es01:/usr/share/elasticsearch/config/certs/http_ca.crt . +---- ++ +. Make a REST API call to {es} to ensure the {es} container is running. ++ +[source,sh] +---- +curl --cacert http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 +---- +// NOTCONSOLE + +*Run {kib}* + +{kib} is the user interface for Elastic. +It's great for getting started with {es} and exploring your data. +We'll be using the Dev Tools *Console* in {kib} to make REST API calls to {es}. + +In a new terminal session, start {kib} and connect it to your {es} container: + +[source,sh,subs="attributes"] +---- +docker pull {kib-docker-image} +docker run --name kibana --net elastic -p 5601:5601 {kib-docker-image} +---- + +When you start {kib}, a unique URL is output to your terminal. +To access {kib}: + +. Open the generated URL in your browser. +. Paste the enrollment token that you copied earlier, to connect your {kib} instance with {es}. +. Log in to {kib} as the `elastic` user with the password that was generated when you started {es}. +// end::self-managed[] \ No newline at end of file