From 2e7b7a8f1e989c1d6eea044e104261b336200156 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 5 Feb 2019 06:10:34 -0500 Subject: [PATCH 01/25] Deprecate types in rollover index API (#38389) Backport for #38039 Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 67 ++++++-- .../client/IndicesRequestConverters.java | 24 ++- .../elasticsearch/client/TimedRequest.java | 8 + .../client/indices/CreateIndexRequest.java | 8 +- .../indices/rollover/RolloverRequest.java | 148 ++++++++++++++++++ .../indices/rollover/RolloverResponse.java | 129 +++++++++++++++ .../elasticsearch/client/IndicesClientIT.java | 39 ++++- .../client/IndicesRequestConvertersTests.java | 51 +++++- .../IndicesClientDocumentationIT.java | 15 +- .../indices/RandomCreateIndexGenerator.java | 13 ++ .../rollover/RolloverRequestTests.java | 65 ++++++++ .../rollover/RolloverResponseTests.java | 100 ++++++++++++ .../high-level/indices/rollover.asciidoc | 15 +- .../rest-api-spec/api/indices.rollover.json | 4 + .../test/indices.rollover/40_mapping.yml | 47 ++++++ .../41_mapping_with_types.yml | 47 ++++++ .../admin/indices/rollover/Condition.java | 4 + .../indices/rollover/RolloverRequest.java | 22 ++- .../indices/rollover/RolloverResponse.java | 7 + .../indices/RestRolloverIndexAction.java | 13 +- .../rollover/RolloverRequestTests.java | 9 +- .../rollover/RolloverResponseTests.java | 3 +- .../index/RandomCreateIndexGenerator.java | 2 +- 23 files changed, 784 insertions(+), 56 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 17d803a713a7b..31833b204c81c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -42,8 +42,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -65,6 +63,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -1234,17 +1234,54 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, RequestOptions RolloverResponse::fromXContent, emptySet()); } + /** + * Asynchronously rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + RolloverResponse::fromXContent, listener, emptySet()); + } + + /** + * Rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet()); + } + /** * Rolls over an index using the Rollover Index API. *

* See * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rollover(RolloverRequest, RequestOptions)} + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. */ @Deprecated - public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... headers) throws IOException { + public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - RolloverResponse::fromXContent, emptySet(), headers); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet(), headers); } /** @@ -1254,10 +1291,16 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... head * @param rolloverRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which + * accepts a new request object. */ - public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { + @Deprecated + public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, - RolloverResponse::fromXContent, listener, emptySet()); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet()); } /** @@ -1265,12 +1308,16 @@ public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions option *

* See * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which + * accepts a new request object. */ @Deprecated - public void rolloverAsync(RolloverRequest rolloverRequest, ActionListener listener, Header... headers) { + public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - RolloverResponse::fromXContent, listener, emptySet(), headers); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet(), headers); } /** diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index a600b4a2eba3a..7c08bffa99e02 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -37,7 +37,6 @@ import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -53,6 +52,7 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; @@ -345,7 +345,7 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { static Request rollover(RolloverRequest rolloverRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") - .addPathPart(rolloverRequest.getNewIndexName()).build(); + .addPathPart(rolloverRequest.getNewIndexName()).build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); @@ -355,11 +355,31 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { if (rolloverRequest.isDryRun()) { params.putParam("dry_run", Boolean.TRUE.toString()); } + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "false"); request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); return request; } + @Deprecated + static Request rollover(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") + .addPathPart(rolloverRequest.getNewIndexName()).build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withTimeout(rolloverRequest.timeout()); + params.withMasterTimeout(rolloverRequest.masterNodeTimeout()); + params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards()); + if (rolloverRequest.isDryRun()) { + params.putParam("dry_run", Boolean.TRUE.toString()); + } + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + + return request; + } + static Request getSettings(GetSettingsRequest getSettingsRequest) { String[] indices = getSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.indices(); String[] names = getSettingsRequest.names() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.names(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java index 60ecea39ae093..d03e183dc1029 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java @@ -36,11 +36,19 @@ public abstract class TimedRequest implements Validatable { private TimeValue timeout = DEFAULT_ACK_TIMEOUT; private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT; + /** + * Sets the timeout to wait for the all the nodes to acknowledge + * @param timeout timeout as a {@link TimeValue} + */ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } + /** + * Sets the timeout to connect to the master node + * @param masterTimeout timeout as a {@link TimeValue} + */ public void setMasterTimeout(TimeValue masterTimeout) { this.masterTimeout = masterTimeout; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java index f0bff6e6f4307..1a018591dc770 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java @@ -338,10 +338,14 @@ public CreateIndexRequest waitForActiveShards(ActiveShardCount waitForActiveShar return this; } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + innerToXContent(builder, params); + builder.endObject(); + return builder; + } + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(SETTINGS.getPreferredName()); settings.toXContent(builder, params); builder.endObject(); @@ -356,8 +360,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (Alias alias : aliases) { alias.toXContent(builder, params); } - builder.endObject(); - builder.endObject(); return builder; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java new file mode 100644 index 0000000000000..ef78fb7353067 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.client.TimedRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Request class to swap index under an alias upon satisfying conditions + */ +public class RolloverRequest extends TimedRequest implements ToXContentObject { + + private final String alias; + private final String newIndexName; + private boolean dryRun; + private final Map> conditions = new HashMap<>(2); + //the index name "_na_" is never read back, what matters are settings, mappings and aliases + private final CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); + + public RolloverRequest(String alias, String newIndexName) { + if (alias == null) { + throw new IllegalArgumentException("The index alias cannot be null!"); + } + this.alias = alias; + this.newIndexName = newIndexName; + } + + /** + * Returns the alias of the rollover operation + */ + public String getAlias() { + return alias; + } + + /** + * Returns the new index name for the rollover + */ + public String getNewIndexName() { + return newIndexName; + } + + + /** + * Sets if the rollover should not be executed when conditions are met + */ + public RolloverRequest dryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + /** + * Returns if the rollover should not be executed when conditions are met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Adds condition to check if the index is at least age old + */ + public RolloverRequest addMaxIndexAgeCondition(TimeValue age) { + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(age); + if (this.conditions.containsKey(maxAgeCondition.name())) { + throw new IllegalArgumentException(maxAgeCondition.name() + " condition is already set"); + } + this.conditions.put(maxAgeCondition.name(), maxAgeCondition); + return this; + } + + /** + * Adds condition to check if the index has at least numDocs + */ + public RolloverRequest addMaxIndexDocsCondition(long numDocs) { + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(numDocs); + if (this.conditions.containsKey(maxDocsCondition.name())) { + throw new IllegalArgumentException(maxDocsCondition.name() + " condition is already set"); + } + this.conditions.put(maxDocsCondition.name(), maxDocsCondition); + return this; + } + /** + * Adds a size-based condition to check if the index size is at least size. + */ + public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(size); + if (this.conditions.containsKey(maxSizeCondition.name())) { + throw new IllegalArgumentException(maxSizeCondition + " condition is already set"); + } + this.conditions.put(maxSizeCondition.name(), maxSizeCondition); + return this; + } + /** + * Returns all set conditions + */ + public Map> getConditions() { + return conditions; + } + + /** + * Returns the inner {@link CreateIndexRequest}. Allows to configure mappings, settings and aliases for the new index. + */ + public CreateIndexRequest getCreateIndexRequest() { + return createIndexRequest; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + createIndexRequest.innerToXContent(builder, params); + + builder.startObject("conditions"); + for (Condition condition : conditions.values()) { + condition.toXContent(builder, params); + } + builder.endObject(); + + builder.endObject(); + return builder; + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java new file mode 100644 index 0000000000000..2bcd683d7b1f6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + */ +public final class RolloverResponse extends ShardsAcknowledgedResponse { + + private static final ParseField NEW_INDEX = new ParseField("new_index"); + private static final ParseField OLD_INDEX = new ParseField("old_index"); + private static final ParseField DRY_RUN = new ParseField("dry_run"); + private static final ParseField ROLLED_OVER = new ParseField("rolled_over"); + private static final ParseField CONDITIONS = new ParseField("conditions"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rollover", + true, args -> new RolloverResponse((String) args[0], (String) args[1], (Map) args[2], + (Boolean)args[3], (Boolean)args[4], (Boolean) args[5], (Boolean) args[6])); + + static { + PARSER.declareString(constructorArg(), OLD_INDEX); + PARSER.declareString(constructorArg(), NEW_INDEX); + PARSER.declareObject(constructorArg(), (parser, context) -> parser.map(), CONDITIONS); + PARSER.declareBoolean(constructorArg(), DRY_RUN); + PARSER.declareBoolean(constructorArg(), ROLLED_OVER); + declareAcknowledgedAndShardsAcknowledgedFields(PARSER); + } + + private final String oldIndex; + private final String newIndex; + private final Map conditionStatus; + private final boolean dryRun; + private final boolean rolledOver; + + public RolloverResponse(String oldIndex, String newIndex, Map conditionResults, + boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) { + super(acknowledged, shardsAcknowledged); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + this.dryRun = dryRun; + this.rolledOver = rolledOver; + this.conditionStatus = conditionResults; + } + + /** + * Returns the name of the index that the request alias was pointing to + */ + public String getOldIndex() { + return oldIndex; + } + + /** + * Returns the name of the index that the request alias currently points to + */ + public String getNewIndex() { + return newIndex; + } + + /** + * Returns the statuses of all the request conditions + */ + public Map getConditionStatus() { + return conditionStatus; + } + + /** + * Returns if the rollover execution was skipped even when conditions were met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Returns true if the rollover was not simulated and the conditions were met + */ + public boolean isRolledOver() { + return rolledOver; + } + + public static RolloverResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (super.equals(o)) { + RolloverResponse that = (RolloverResponse) o; + return dryRun == that.dryRun && + rolledOver == that.rolledOver && + Objects.equals(oldIndex, that.oldIndex) && + Objects.equals(newIndex, that.newIndex) && + Objects.equals(conditionStatus, that.conditionStatus); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), oldIndex, newIndex, conditionStatus, dryRun, rolledOver); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 59c834fba0fee..134679e87e12a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -47,8 +47,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -77,6 +75,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ValidationException; @@ -90,6 +90,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryBuilder; @@ -99,6 +100,7 @@ import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; +import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; @@ -1109,7 +1111,6 @@ public void testRollover() throws IOException { { RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1129,7 +1130,6 @@ public void testRollover() throws IOException { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1)); rolloverRequest.dryRun(true); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertTrue(rolloverResponse.isDryRun()); @@ -1141,10 +1141,11 @@ public void testRollover() throws IOException { assertEquals("test_new", rolloverResponse.getNewIndex()); } { + String mappings = "{\"properties\":{\"field2\":{\"type\":\"keyword\"}}}"; + rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1158,6 +1159,34 @@ public void testRollover() throws IOException { } } + public void testRolloverWithTypes() throws IOException { + highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test", "_doc", "1").source("field", "value"), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test", "_doc", "2").source("field", "value") + .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT); + + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest("alias", "test_new"); + rolloverRequest.addMaxIndexDocsCondition(1); + + Settings.Builder settings = Settings.builder().put("number_of_shards", 1); + rolloverRequest.getCreateIndexRequest().mapping("_doc", "field2", "type=keyword") + .settings(settings); + + org.elasticsearch.action.admin.indices.rollover.RolloverResponse rolloverResponse = execute( + rolloverRequest, + highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, + expectWarnings(RestRolloverIndexAction.TYPES_DEPRECATION_MESSAGE) + ); + assertTrue(rolloverResponse.isRolledOver()); + assertFalse(rolloverResponse.isDryRun()); + Map conditionStatus = rolloverResponse.getConditionStatus(); + assertTrue(conditionStatus.get("[max_docs: 1]")); + assertEquals("test", rolloverResponse.getOldIndex()); + assertEquals("test_new", rolloverResponse.getNewIndex()); + } + public void testGetAlias() throws IOException { { createIndex("index1", Settings.EMPTY); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index ba471baaa42c8..e1d3d9b4221c8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -39,7 +39,6 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -56,6 +55,7 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -76,7 +76,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomMapping; import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; @@ -813,7 +814,7 @@ private void resizeTest(ResizeType resizeType, CheckedFunction expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomTimeout(rolloverRequest, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); + if (ESTestCase.randomBoolean()) { + rolloverRequest.dryRun(ESTestCase.randomBoolean()); + if (rolloverRequest.isDryRun()) { + expectedParams.put("dry_run", "true"); + } + } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "false"); + if (ESTestCase.randomBoolean()) { + rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().mapping(randomMapping()); + } + if (ESTestCase.randomBoolean()) { + randomAliases(rolloverRequest.getCreateIndexRequest()); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().settings( + org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings()); + } + RequestConvertersTests.setRandomWaitForActiveShards(rolloverRequest.getCreateIndexRequest()::waitForActiveShards, expectedParams); + + Request request = IndicesRequestConverters.rollover(rolloverRequest); + if (rolloverRequest.getNewIndexName() == null) { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover", request.getEndpoint()); + } else { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover/" + rolloverRequest.getNewIndexName(), + request.getEndpoint()); + } + Assert.assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + RequestConvertersTests.assertToXContentBody(rolloverRequest, request.getEntity()); + Assert.assertEquals(expectedParams, request.getParameters()); + } + + public void testRolloverWithTypes() throws IOException { + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest(ESTestCase.randomAlphaOfLengthBetween(3, 10), + ESTestCase.randomBoolean() ? null : ESTestCase.randomAlphaOfLengthBetween(3, 10)); + Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomTimeout(rolloverRequest::timeout, rolloverRequest.timeout(), expectedParams); RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); if (ESTestCase.randomBoolean()) { @@ -840,6 +882,7 @@ public void testRollover() throws IOException { expectedParams.put("dry_run", "true"); } } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); } @@ -849,7 +892,7 @@ public void testRollover() throws IOException { org.elasticsearch.index.RandomCreateIndexGenerator.randomMapping(type)); } if (ESTestCase.randomBoolean()) { - randomAliases(rolloverRequest.getCreateIndexRequest()); + org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases(rolloverRequest.getCreateIndexRequest()); } if (ESTestCase.randomBoolean()) { rolloverRequest.getCreateIndexRequest().settings( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index fc4f69e281da9..8c20ad2dd32da 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -46,8 +46,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -80,6 +78,8 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -1823,18 +1823,16 @@ public void testRolloverIndex() throws Exception { // end::rollover-index-request // tag::rollover-index-request-timeout - request.timeout(TimeValue.timeValueMinutes(2)); // <1> - request.timeout("2m"); // <2> + request.setTimeout(TimeValue.timeValueMinutes(2)); // <1> // end::rollover-index-request-timeout // tag::rollover-index-request-masterTimeout - request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> - request.masterNodeTimeout("1m"); // <2> + request.setMasterTimeout(TimeValue.timeValueMinutes(1)); // <1> // end::rollover-index-request-masterTimeout // tag::rollover-index-request-dryRun request.dryRun(true); // <1> // end::rollover-index-request-dryRun // tag::rollover-index-request-waitForActiveShards - request.getCreateIndexRequest().waitForActiveShards(2); // <1> + request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.from(2)); // <1> request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.DEFAULT); // <2> // end::rollover-index-request-waitForActiveShards // tag::rollover-index-request-settings @@ -1842,7 +1840,8 @@ public void testRolloverIndex() throws Exception { .put("index.number_of_shards", 4)); // <1> // end::rollover-index-request-settings // tag::rollover-index-request-mapping - request.getCreateIndexRequest().mapping("type", "field", "type=keyword"); // <1> + String mappings = "{\"properties\":{\"field-1\":{\"type\":\"keyword\"}}}"; + request.getCreateIndexRequest().mapping(mappings, XContentType.JSON); // <1> // end::rollover-index-request-mapping // tag::rollover-index-request-alias request.getCreateIndexRequest().alias(new Alias("another_alias")); // <1> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java index 179b7e728b620..610cc54678ae0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java @@ -24,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAlias; +import static org.elasticsearch.test.ESTestCase.randomIntBetween; + public class RandomCreateIndexGenerator { /** @@ -58,4 +61,14 @@ public static XContentBuilder randomMapping() throws IOException { builder.endObject(); return builder; } + + /** + * Sets random aliases to the provided {@link CreateIndexRequest} + */ + public static void randomAliases(CreateIndexRequest request) { + int aliasesNo = randomIntBetween(0, 2); + for (int i = 0; i < aliasesNo; i++) { + request.alias(randomAlias()); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java new file mode 100644 index 0000000000000..57798c393db8f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; + + +public class RolloverRequestTests extends ESTestCase { + public void testConstructorAndFieldAssignments() { + // test constructor + String alias = randomAlphaOfLength(5); + String newIndexName = null; + if (randomBoolean()) { + newIndexName = randomAlphaOfLength(8); + } + RolloverRequest rolloverRequest = new RolloverRequest(alias, newIndexName); + assertEquals(alias, rolloverRequest.getAlias()); + assertEquals(newIndexName, rolloverRequest.getNewIndexName()); + + // test assignment of conditions + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); + Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; + rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); + rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); + rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); + List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); + assertThat(requestConditions, containsInAnyOrder(expectedConditions)); + } + + public void testValidation() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> + new RolloverRequest(null, null)); + assertEquals("The index alias cannot be null!", exception.getMessage()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java new file mode 100644 index 0000000000000..53fe3bb279e3f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.common.xcontent.ToXContent.Params; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.Collections; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class RolloverResponseTests extends ESTestCase { + + private static final List>> conditionSuppliers = new ArrayList<>(); + static { + conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + } + + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + RolloverResponseTests::createTestInstance, + RolloverResponseTests::toXContent, + RolloverResponse::fromXContent) + .supportsUnknownFields(true) + .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter()) + .test(); + } + + private static RolloverResponse createTestInstance() { + final String oldIndex = randomAlphaOfLength(8); + final String newIndex = randomAlphaOfLength(8); + final boolean dryRun = randomBoolean(); + final boolean rolledOver = randomBoolean(); + final boolean acknowledged = randomBoolean(); + final boolean shardsAcknowledged = acknowledged && randomBoolean(); + + Map results = new HashMap<>(); + int numResults = randomIntBetween(0, 3); + List>> conditions = randomSubsetOf(numResults, conditionSuppliers); + conditions.forEach(condition -> results.put(condition.get().name(), randomBoolean())); + + return new RolloverResponse(oldIndex, newIndex, results, dryRun, rolledOver, acknowledged, shardsAcknowledged); + } + + private Predicate getRandomFieldsExcludeFilter() { + return field -> field.startsWith("conditions"); + } + + private static void toXContent(RolloverResponse response, XContentBuilder builder) throws IOException { + Params params = new ToXContent.MapParams( + Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse serverResponse = + new org.elasticsearch.action.admin.indices.rollover.RolloverResponse( + response.getOldIndex(), + response.getNewIndex(), + response.getConditionStatus(), + response.isDryRun(), + response.isRolledOver(), + response.isAcknowledged(), + response.isShardsAcknowledged() + ); + serverResponse.toXContent(builder, params); + } +} diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index c6134cd5579df..6b7a82a11ae2b 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -19,7 +19,8 @@ one or more conditions that determine when the index has to be rolled over: include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> The alias (first argument) that points to the index to rollover, and -optionally the name of the new index in case the rollover operation is performed +the name of the new index in case the rollover operation is performed. +The new index argument is optional, and can be set to null <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index @@ -39,24 +40,20 @@ include-tagged::{doc-tests-file}[{api}-request-timeout] -------------------------------------------------- <1> Timeout to wait for the all the nodes to acknowledge the index is opened as a `TimeValue` -<2> Timeout to wait for the all the nodes to acknowledge the index is opened -as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- <1> Timeout to connect to the master node as a `TimeValue` -<2> Timeout to connect to the master node as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> The number of active shard copies to wait for before the rollover index API -returns a response, as an `int` -<2> The number of active shard copies to wait for before the rollover index API -returns a response, as an `ActiveShardCount` +<1> Sets the number of active shard copies to wait for before the rollover +index API returns a response +<2> Resets the number of active shard copies to wait for to the default value ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -98,5 +95,3 @@ each shard in the index before timing out <5> Whether the index has been rolled over <6> Whether the operation was performed or it was a dry run <7> The different conditions and whether they were matched or not - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 5e5ba1367ad3e..7bf1513969fb3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -18,6 +18,10 @@ } }, "params": { + "include_type_name": { + "type" : "boolean", + "description" : "Whether a type should be included in the body of the mappings." + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml new file mode 100644 index 0000000000000..7ed78c6e3159a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml @@ -0,0 +1,47 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.6.99" + reason: include_type_name was introduced in 6.7.0 + + - do: + indices.create: + include_type_name: false + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + include_type_name: false + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml new file mode 100644 index 0000000000000..c5d0bd518937e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml @@ -0,0 +1,47 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.6.99" + reason: include_type_name was introduced in 6.7.0 + + - do: + indices.create: + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + include_type_name: true + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + _doc: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index 1b385ed9d0dbc..a2cad7a35a398 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -75,6 +75,10 @@ public T value() { return value; } + public String name() { + return name; + } + /** * Holder for index stats used to evaluate conditions */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index f36636594a4d6..49db34aa5386e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -41,10 +42,13 @@ /** * Request class to swap index under an alias upon satisfying conditions + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -66,9 +70,14 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, context) -> { - for (Map.Entry mappingsEntry : parser.map().entrySet()) { - request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + PARSER.declareField((parser, request, isTypeIncluded) -> { + if (isTypeIncluded) { + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + } + } else { + // a type is not included, add a dummy _doc type + request.createIndexRequest.mapping(MapperService.SINGLE_MAPPING_NAME, parser.map()); } }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), @@ -230,7 +239,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public void fromXContent(XContentParser parser) throws IOException { - PARSER.parse(parser, this, null); + // param isTypeIncluded decides how mappings should be parsed from XContent + public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { + PARSER.parse(parser, this, isTypeIncluded); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java index 356f805c24bd4..78bdde7b7bd18 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java @@ -37,6 +37,13 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. + */ public final class RolloverResponse extends ShardsAcknowledgedResponse implements ToXContentObject { private static final ParseField NEW_INDEX = new ParseField("new_index"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 489001bf2a14f..a7e0b3b22d6f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -19,9 +19,11 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -31,6 +33,11 @@ import java.io.IOException; public class RestRolloverIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestRolloverIndexAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of rollover " + + "index requests will change in 7.0. Please start using the include_type_name parameter set to false " + + "to move to the new, typeless response format that will become the default."; public RestRolloverIndexAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this); @@ -44,8 +51,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (includeTypeName) { + deprecationLogger.deprecatedAndMaybeLog("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); + } RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(rolloverIndexRequest::fromXContent); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 1e8d8e2a2932c..e4ccb70f4862a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -52,7 +52,6 @@ import static org.hamcrest.Matchers.equalTo; public class RolloverRequestTests extends ESTestCase { - private NamedWriteableRegistry writeableRegistry; @Override @@ -72,7 +71,7 @@ public void testConditionsParsing() throws Exception { .field("max_size", "45gb") .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(3)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -108,7 +107,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(true, createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1)); @@ -147,7 +146,7 @@ public void testToAndFromXContent() throws IOException { BytesReference originalBytes = toShuffledXContent(rolloverRequest, xContentType, EMPTY_PARAMS, humanReadable); RolloverRequest parsedRolloverRequest = new RolloverRequest(); - parsedRolloverRequest.fromXContent(createParser(xContentType.xContent(), originalBytes)); + parsedRolloverRequest.fromXContent(true, createParser(xContentType.xContent(), originalBytes)); CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest(); CreateIndexRequest parsedCreateIndexRequest = parsedRolloverRequest.getCreateIndexRequest(); @@ -172,7 +171,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index e0f58ecd92daf..05eaefffaebf0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.rollover; import org.elasticsearch.Version; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractStreamableXContentTestCase; @@ -57,7 +58,7 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java index 345ef1f58bcac..9732504cac6d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java @@ -126,7 +126,7 @@ public static void randomAliases(CreateIndexRequest request) { } } - private static Alias randomAlias() { + public static Alias randomAlias() { Alias alias = new Alias(randomAlphaOfLength(5)); if (randomBoolean()) { From a12358232e222e227d8dbc6b379db0f3d613e4df Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Tue, 5 Feb 2019 13:37:17 +0200 Subject: [PATCH 02/25] SecuritySettingsSource license.self_generated: trial (#38233) (#38397) Authn is enabled only if `license_type` is non `basic`, but `basic` is what the `LicenseService` generates implicitly. This commit explicitly sets license type to `trial`, which allows for authn, in the `SecuritySettingsSource` which is the settings configuration parameter for `InternalTestCluster`s. The real problem, that had created tests failures like #31028 and #32685, is that the check `licenseState.isAuthAllowed()` can change sporadically. If it were to return `true` or `false` during the whole test there would be no problem. The problem manifests when it turns from `true` to `false` right before `Realms.asList()`. There are other license checks before this one (request filter, token service, etc) that would not cause a problem if they would suddenly see the check as `false`. But switching to `false` before `Realms.asList()` makes it appear that no installed realms could have handled the authn token which is an authentication error, as can be seen in the failing tests. Closes #31028 #32685 --- .../java/org/elasticsearch/test/SecuritySettingsSource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 883cf23bec6b6..024272bda8e25 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -136,7 +136,8 @@ public Settings nodeSettings(int nodeOrdinal) { .put("xpack.security.authc.realms.file.type", FileRealmSettings.TYPE) .put("xpack.security.authc.realms.file.order", 0) .put("xpack.security.authc.realms.index.type", NativeRealmSettings.TYPE) - .put("xpack.security.authc.realms.index.order", "1"); + .put("xpack.security.authc.realms.index.order", "1") + .put("xpack.license.self_generated.type", "trial"); addNodeSSLSettings(builder); return builder.build(); } From 766f7b5bf3a896d3379999143770f548fd8c4a6a Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 5 Feb 2019 15:41:17 +0200 Subject: [PATCH 03/25] Revert "Deprecate types in rollover index API (#38389)" (#38419) This reverts commit 2e7b7a8f1e989c1d6eea044e104261b336200156. --- .../elasticsearch/client/IndicesClient.java | 67 ++------ .../client/IndicesRequestConverters.java | 24 +-- .../elasticsearch/client/TimedRequest.java | 8 - .../client/indices/CreateIndexRequest.java | 8 +- .../indices/rollover/RolloverRequest.java | 148 ------------------ .../indices/rollover/RolloverResponse.java | 129 --------------- .../elasticsearch/client/IndicesClientIT.java | 39 +---- .../client/IndicesRequestConvertersTests.java | 51 +----- .../IndicesClientDocumentationIT.java | 15 +- .../indices/RandomCreateIndexGenerator.java | 13 -- .../rollover/RolloverRequestTests.java | 65 -------- .../rollover/RolloverResponseTests.java | 100 ------------ .../high-level/indices/rollover.asciidoc | 15 +- .../rest-api-spec/api/indices.rollover.json | 4 - .../test/indices.rollover/40_mapping.yml | 47 ------ .../41_mapping_with_types.yml | 47 ------ .../admin/indices/rollover/Condition.java | 4 - .../indices/rollover/RolloverRequest.java | 22 +-- .../indices/rollover/RolloverResponse.java | 7 - .../indices/RestRolloverIndexAction.java | 13 +- .../rollover/RolloverRequestTests.java | 9 +- .../rollover/RolloverResponseTests.java | 3 +- .../index/RandomCreateIndexGenerator.java | 2 +- 23 files changed, 56 insertions(+), 784 deletions(-) delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 31833b204c81c..17d803a713a7b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -42,6 +42,8 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -63,8 +65,6 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; -import org.elasticsearch.client.indices.rollover.RolloverRequest; -import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -1234,54 +1234,17 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, RequestOptions RolloverResponse::fromXContent, emptySet()); } - /** - * Asynchronously rolls over an index using the Rollover Index API. - * See - * Rollover Index API on elastic.co - * @param rolloverRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - */ - public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, - RolloverResponse::fromXContent, listener, emptySet()); - } - - /** - * Rolls over an index using the Rollover Index API. - * See - * Rollover Index API on elastic.co - * @param rolloverRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - * @throws IOException in case there is a problem sending the request or parsing back the response - * - * @deprecated This method uses deprecated request and response objects. - * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. - */ - @Deprecated - public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( - org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, - RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, - org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet()); - } - /** * Rolls over an index using the Rollover Index API. *

* See * Rollover Index API on elastic.co - * - * @deprecated This method uses deprecated request and response objects. - * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. + * @deprecated Prefer {@link #rollover(RolloverRequest, RequestOptions)} */ @Deprecated - public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( - org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, - Header... headers) throws IOException { + public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet(), headers); + RolloverResponse::fromXContent, emptySet(), headers); } /** @@ -1291,16 +1254,10 @@ public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover * @param rolloverRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion - * - * @deprecated This method uses deprecated request and response objects. - * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which - * accepts a new request object. */ - @Deprecated - public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, - RequestOptions options, ActionListener listener) { + public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, - org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet()); + RolloverResponse::fromXContent, listener, emptySet()); } /** @@ -1308,16 +1265,12 @@ public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.Rollov *

* See * Rollover Index API on elastic.co - * - * @deprecated This method uses deprecated request and response objects. - * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which - * accepts a new request object. + * @deprecated Prefer {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} */ @Deprecated - public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, - ActionListener listener, Header... headers) { + public void rolloverAsync(RolloverRequest rolloverRequest, ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet(), headers); + RolloverResponse::fromXContent, listener, emptySet(), headers); } /** diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 7c08bffa99e02..a600b4a2eba3a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -37,6 +37,7 @@ import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -52,7 +53,6 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; -import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; @@ -345,7 +345,7 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { static Request rollover(RolloverRequest rolloverRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") - .addPathPart(rolloverRequest.getNewIndexName()).build(); + .addPathPart(rolloverRequest.getNewIndexName()).build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); @@ -355,31 +355,11 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { if (rolloverRequest.isDryRun()) { params.putParam("dry_run", Boolean.TRUE.toString()); } - params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "false"); request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); return request; } - @Deprecated - static Request rollover(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest) throws IOException { - String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") - .addPathPart(rolloverRequest.getNewIndexName()).build(); - Request request = new Request(HttpPost.METHOD_NAME, endpoint); - - RequestConverters.Params params = new RequestConverters.Params(request); - params.withTimeout(rolloverRequest.timeout()); - params.withMasterTimeout(rolloverRequest.masterNodeTimeout()); - params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards()); - if (rolloverRequest.isDryRun()) { - params.putParam("dry_run", Boolean.TRUE.toString()); - } - params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); - request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); - - return request; - } - static Request getSettings(GetSettingsRequest getSettingsRequest) { String[] indices = getSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.indices(); String[] names = getSettingsRequest.names() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.names(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java index d03e183dc1029..60ecea39ae093 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java @@ -36,19 +36,11 @@ public abstract class TimedRequest implements Validatable { private TimeValue timeout = DEFAULT_ACK_TIMEOUT; private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT; - /** - * Sets the timeout to wait for the all the nodes to acknowledge - * @param timeout timeout as a {@link TimeValue} - */ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } - /** - * Sets the timeout to connect to the master node - * @param masterTimeout timeout as a {@link TimeValue} - */ public void setMasterTimeout(TimeValue masterTimeout) { this.masterTimeout = masterTimeout; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java index 1a018591dc770..f0bff6e6f4307 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java @@ -338,14 +338,10 @@ public CreateIndexRequest waitForActiveShards(ActiveShardCount waitForActiveShar return this; } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - innerToXContent(builder, params); - builder.endObject(); - return builder; - } - public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(SETTINGS.getPreferredName()); settings.toXContent(builder, params); builder.endObject(); @@ -360,6 +356,8 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t for (Alias alias : aliases) { alias.toXContent(builder, params); } + builder.endObject(); + builder.endObject(); return builder; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java deleted file mode 100644 index ef78fb7353067..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.client.indices.rollover; - -import org.elasticsearch.action.admin.indices.rollover.Condition; -import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; -import org.elasticsearch.client.TimedRequest; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Request class to swap index under an alias upon satisfying conditions - */ -public class RolloverRequest extends TimedRequest implements ToXContentObject { - - private final String alias; - private final String newIndexName; - private boolean dryRun; - private final Map> conditions = new HashMap<>(2); - //the index name "_na_" is never read back, what matters are settings, mappings and aliases - private final CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); - - public RolloverRequest(String alias, String newIndexName) { - if (alias == null) { - throw new IllegalArgumentException("The index alias cannot be null!"); - } - this.alias = alias; - this.newIndexName = newIndexName; - } - - /** - * Returns the alias of the rollover operation - */ - public String getAlias() { - return alias; - } - - /** - * Returns the new index name for the rollover - */ - public String getNewIndexName() { - return newIndexName; - } - - - /** - * Sets if the rollover should not be executed when conditions are met - */ - public RolloverRequest dryRun(boolean dryRun) { - this.dryRun = dryRun; - return this; - } - /** - * Returns if the rollover should not be executed when conditions are met - */ - public boolean isDryRun() { - return dryRun; - } - - /** - * Adds condition to check if the index is at least age old - */ - public RolloverRequest addMaxIndexAgeCondition(TimeValue age) { - MaxAgeCondition maxAgeCondition = new MaxAgeCondition(age); - if (this.conditions.containsKey(maxAgeCondition.name())) { - throw new IllegalArgumentException(maxAgeCondition.name() + " condition is already set"); - } - this.conditions.put(maxAgeCondition.name(), maxAgeCondition); - return this; - } - - /** - * Adds condition to check if the index has at least numDocs - */ - public RolloverRequest addMaxIndexDocsCondition(long numDocs) { - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(numDocs); - if (this.conditions.containsKey(maxDocsCondition.name())) { - throw new IllegalArgumentException(maxDocsCondition.name() + " condition is already set"); - } - this.conditions.put(maxDocsCondition.name(), maxDocsCondition); - return this; - } - /** - * Adds a size-based condition to check if the index size is at least size. - */ - public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(size); - if (this.conditions.containsKey(maxSizeCondition.name())) { - throw new IllegalArgumentException(maxSizeCondition + " condition is already set"); - } - this.conditions.put(maxSizeCondition.name(), maxSizeCondition); - return this; - } - /** - * Returns all set conditions - */ - public Map> getConditions() { - return conditions; - } - - /** - * Returns the inner {@link CreateIndexRequest}. Allows to configure mappings, settings and aliases for the new index. - */ - public CreateIndexRequest getCreateIndexRequest() { - return createIndexRequest; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - createIndexRequest.innerToXContent(builder, params); - - builder.startObject("conditions"); - for (Condition condition : conditions.values()) { - condition.toXContent(builder, params); - } - builder.endObject(); - - builder.endObject(); - return builder; - } - -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java deleted file mode 100644 index 2bcd683d7b1f6..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.indices.rollover; - -import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; - -/** - * Response object for {@link RolloverRequest} API - */ -public final class RolloverResponse extends ShardsAcknowledgedResponse { - - private static final ParseField NEW_INDEX = new ParseField("new_index"); - private static final ParseField OLD_INDEX = new ParseField("old_index"); - private static final ParseField DRY_RUN = new ParseField("dry_run"); - private static final ParseField ROLLED_OVER = new ParseField("rolled_over"); - private static final ParseField CONDITIONS = new ParseField("conditions"); - - @SuppressWarnings("unchecked") - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rollover", - true, args -> new RolloverResponse((String) args[0], (String) args[1], (Map) args[2], - (Boolean)args[3], (Boolean)args[4], (Boolean) args[5], (Boolean) args[6])); - - static { - PARSER.declareString(constructorArg(), OLD_INDEX); - PARSER.declareString(constructorArg(), NEW_INDEX); - PARSER.declareObject(constructorArg(), (parser, context) -> parser.map(), CONDITIONS); - PARSER.declareBoolean(constructorArg(), DRY_RUN); - PARSER.declareBoolean(constructorArg(), ROLLED_OVER); - declareAcknowledgedAndShardsAcknowledgedFields(PARSER); - } - - private final String oldIndex; - private final String newIndex; - private final Map conditionStatus; - private final boolean dryRun; - private final boolean rolledOver; - - public RolloverResponse(String oldIndex, String newIndex, Map conditionResults, - boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) { - super(acknowledged, shardsAcknowledged); - this.oldIndex = oldIndex; - this.newIndex = newIndex; - this.dryRun = dryRun; - this.rolledOver = rolledOver; - this.conditionStatus = conditionResults; - } - - /** - * Returns the name of the index that the request alias was pointing to - */ - public String getOldIndex() { - return oldIndex; - } - - /** - * Returns the name of the index that the request alias currently points to - */ - public String getNewIndex() { - return newIndex; - } - - /** - * Returns the statuses of all the request conditions - */ - public Map getConditionStatus() { - return conditionStatus; - } - - /** - * Returns if the rollover execution was skipped even when conditions were met - */ - public boolean isDryRun() { - return dryRun; - } - - /** - * Returns true if the rollover was not simulated and the conditions were met - */ - public boolean isRolledOver() { - return rolledOver; - } - - public static RolloverResponse fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - - @Override - public boolean equals(Object o) { - if (super.equals(o)) { - RolloverResponse that = (RolloverResponse) o; - return dryRun == that.dryRun && - rolledOver == that.rolledOver && - Objects.equals(oldIndex, that.oldIndex) && - Objects.equals(newIndex, that.newIndex) && - Objects.equals(conditionStatus, that.conditionStatus); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), oldIndex, newIndex, conditionStatus, dryRun, rolledOver); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 134679e87e12a..59c834fba0fee 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -47,6 +47,8 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -75,8 +77,6 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; -import org.elasticsearch.client.indices.rollover.RolloverRequest; -import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ValidationException; @@ -90,7 +90,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryBuilder; @@ -100,7 +99,6 @@ import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; -import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; @@ -1111,6 +1109,7 @@ public void testRollover() throws IOException { { RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1130,6 +1129,7 @@ public void testRollover() throws IOException { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1)); rolloverRequest.dryRun(true); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertTrue(rolloverResponse.isDryRun()); @@ -1141,11 +1141,10 @@ public void testRollover() throws IOException { assertEquals("test_new", rolloverResponse.getNewIndex()); } { - String mappings = "{\"properties\":{\"field2\":{\"type\":\"keyword\"}}}"; - rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1159,34 +1158,6 @@ public void testRollover() throws IOException { } } - public void testRolloverWithTypes() throws IOException { - highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT); - highLevelClient().index(new IndexRequest("test", "_doc", "1").source("field", "value"), RequestOptions.DEFAULT); - highLevelClient().index(new IndexRequest("test", "_doc", "2").source("field", "value") - .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT); - - org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = - new org.elasticsearch.action.admin.indices.rollover.RolloverRequest("alias", "test_new"); - rolloverRequest.addMaxIndexDocsCondition(1); - - Settings.Builder settings = Settings.builder().put("number_of_shards", 1); - rolloverRequest.getCreateIndexRequest().mapping("_doc", "field2", "type=keyword") - .settings(settings); - - org.elasticsearch.action.admin.indices.rollover.RolloverResponse rolloverResponse = execute( - rolloverRequest, - highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, - expectWarnings(RestRolloverIndexAction.TYPES_DEPRECATION_MESSAGE) - ); - assertTrue(rolloverResponse.isRolledOver()); - assertFalse(rolloverResponse.isDryRun()); - Map conditionStatus = rolloverResponse.getConditionStatus(); - assertTrue(conditionStatus.get("[max_docs: 1]")); - assertEquals("test", rolloverResponse.getOldIndex()); - assertEquals("test_new", rolloverResponse.getNewIndex()); - } - public void testGetAlias() throws IOException { { createIndex("index1", Settings.EMPTY); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index e1d3d9b4221c8..ba471baaa42c8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -55,7 +56,6 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; -import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -76,8 +76,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomAliases; -import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomMapping; +import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases; import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; @@ -814,7 +813,7 @@ private void resizeTest(ResizeType resizeType, CheckedFunction expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomTimeout(rolloverRequest, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); - RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); - if (ESTestCase.randomBoolean()) { - rolloverRequest.dryRun(ESTestCase.randomBoolean()); - if (rolloverRequest.isDryRun()) { - expectedParams.put("dry_run", "true"); - } - } - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "false"); - if (ESTestCase.randomBoolean()) { - rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); - } - if (ESTestCase.randomBoolean()) { - rolloverRequest.getCreateIndexRequest().mapping(randomMapping()); - } - if (ESTestCase.randomBoolean()) { - randomAliases(rolloverRequest.getCreateIndexRequest()); - } - if (ESTestCase.randomBoolean()) { - rolloverRequest.getCreateIndexRequest().settings( - org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings()); - } - RequestConvertersTests.setRandomWaitForActiveShards(rolloverRequest.getCreateIndexRequest()::waitForActiveShards, expectedParams); - - Request request = IndicesRequestConverters.rollover(rolloverRequest); - if (rolloverRequest.getNewIndexName() == null) { - Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover", request.getEndpoint()); - } else { - Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover/" + rolloverRequest.getNewIndexName(), - request.getEndpoint()); - } - Assert.assertEquals(HttpPost.METHOD_NAME, request.getMethod()); - RequestConvertersTests.assertToXContentBody(rolloverRequest, request.getEntity()); - Assert.assertEquals(expectedParams, request.getParameters()); - } - - public void testRolloverWithTypes() throws IOException { - org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = - new org.elasticsearch.action.admin.indices.rollover.RolloverRequest(ESTestCase.randomAlphaOfLengthBetween(3, 10), - ESTestCase.randomBoolean() ? null : ESTestCase.randomAlphaOfLengthBetween(3, 10)); - Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomTimeout(rolloverRequest::timeout, rolloverRequest.timeout(), expectedParams); RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); if (ESTestCase.randomBoolean()) { @@ -882,7 +840,6 @@ public void testRolloverWithTypes() throws IOException { expectedParams.put("dry_run", "true"); } } - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); } @@ -892,7 +849,7 @@ public void testRolloverWithTypes() throws IOException { org.elasticsearch.index.RandomCreateIndexGenerator.randomMapping(type)); } if (ESTestCase.randomBoolean()) { - org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases(rolloverRequest.getCreateIndexRequest()); + randomAliases(rolloverRequest.getCreateIndexRequest()); } if (ESTestCase.randomBoolean()) { rolloverRequest.getCreateIndexRequest().settings( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 8c20ad2dd32da..fc4f69e281da9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -46,6 +46,8 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -78,8 +80,6 @@ import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; -import org.elasticsearch.client.indices.rollover.RolloverRequest; -import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -1823,16 +1823,18 @@ public void testRolloverIndex() throws Exception { // end::rollover-index-request // tag::rollover-index-request-timeout - request.setTimeout(TimeValue.timeValueMinutes(2)); // <1> + request.timeout(TimeValue.timeValueMinutes(2)); // <1> + request.timeout("2m"); // <2> // end::rollover-index-request-timeout // tag::rollover-index-request-masterTimeout - request.setMasterTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> // end::rollover-index-request-masterTimeout // tag::rollover-index-request-dryRun request.dryRun(true); // <1> // end::rollover-index-request-dryRun // tag::rollover-index-request-waitForActiveShards - request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.from(2)); // <1> + request.getCreateIndexRequest().waitForActiveShards(2); // <1> request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.DEFAULT); // <2> // end::rollover-index-request-waitForActiveShards // tag::rollover-index-request-settings @@ -1840,8 +1842,7 @@ public void testRolloverIndex() throws Exception { .put("index.number_of_shards", 4)); // <1> // end::rollover-index-request-settings // tag::rollover-index-request-mapping - String mappings = "{\"properties\":{\"field-1\":{\"type\":\"keyword\"}}}"; - request.getCreateIndexRequest().mapping(mappings, XContentType.JSON); // <1> + request.getCreateIndexRequest().mapping("type", "field", "type=keyword"); // <1> // end::rollover-index-request-mapping // tag::rollover-index-request-alias request.getCreateIndexRequest().alias(new Alias("another_alias")); // <1> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java index 610cc54678ae0..179b7e728b620 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java @@ -24,9 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAlias; -import static org.elasticsearch.test.ESTestCase.randomIntBetween; - public class RandomCreateIndexGenerator { /** @@ -61,14 +58,4 @@ public static XContentBuilder randomMapping() throws IOException { builder.endObject(); return builder; } - - /** - * Sets random aliases to the provided {@link CreateIndexRequest} - */ - public static void randomAliases(CreateIndexRequest request) { - int aliasesNo = randomIntBetween(0, 2); - for (int i = 0; i < aliasesNo; i++) { - request.alias(randomAlias()); - } - } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java deleted file mode 100644 index 57798c393db8f..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.indices.rollover; - -import org.elasticsearch.action.admin.indices.rollover.Condition; -import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.test.ESTestCase; - -import java.util.ArrayList; -import java.util.List; - -import static org.hamcrest.Matchers.containsInAnyOrder; - - -public class RolloverRequestTests extends ESTestCase { - public void testConstructorAndFieldAssignments() { - // test constructor - String alias = randomAlphaOfLength(5); - String newIndexName = null; - if (randomBoolean()) { - newIndexName = randomAlphaOfLength(8); - } - RolloverRequest rolloverRequest = new RolloverRequest(alias, newIndexName); - assertEquals(alias, rolloverRequest.getAlias()); - assertEquals(newIndexName, rolloverRequest.getNewIndexName()); - - // test assignment of conditions - MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); - MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); - MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); - Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; - rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); - rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); - rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); - List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); - assertThat(requestConditions, containsInAnyOrder(expectedConditions)); - } - - public void testValidation() { - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> - new RolloverRequest(null, null)); - assertEquals("The index alias cannot be null!", exception.getMessage()); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java deleted file mode 100644 index 53fe3bb279e3f..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.indices.rollover; - -import org.elasticsearch.action.admin.indices.rollover.Condition; -import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; -import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.common.xcontent.ToXContent.Params; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.Collections; - -import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; - -public class RolloverResponseTests extends ESTestCase { - - private static final List>> conditionSuppliers = new ArrayList<>(); - static { - conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); - } - - public void testFromXContent() throws IOException { - xContentTester( - this::createParser, - RolloverResponseTests::createTestInstance, - RolloverResponseTests::toXContent, - RolloverResponse::fromXContent) - .supportsUnknownFields(true) - .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter()) - .test(); - } - - private static RolloverResponse createTestInstance() { - final String oldIndex = randomAlphaOfLength(8); - final String newIndex = randomAlphaOfLength(8); - final boolean dryRun = randomBoolean(); - final boolean rolledOver = randomBoolean(); - final boolean acknowledged = randomBoolean(); - final boolean shardsAcknowledged = acknowledged && randomBoolean(); - - Map results = new HashMap<>(); - int numResults = randomIntBetween(0, 3); - List>> conditions = randomSubsetOf(numResults, conditionSuppliers); - conditions.forEach(condition -> results.put(condition.get().name(), randomBoolean())); - - return new RolloverResponse(oldIndex, newIndex, results, dryRun, rolledOver, acknowledged, shardsAcknowledged); - } - - private Predicate getRandomFieldsExcludeFilter() { - return field -> field.startsWith("conditions"); - } - - private static void toXContent(RolloverResponse response, XContentBuilder builder) throws IOException { - Params params = new ToXContent.MapParams( - Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); - org.elasticsearch.action.admin.indices.rollover.RolloverResponse serverResponse = - new org.elasticsearch.action.admin.indices.rollover.RolloverResponse( - response.getOldIndex(), - response.getNewIndex(), - response.getConditionStatus(), - response.isDryRun(), - response.isRolledOver(), - response.isAcknowledged(), - response.isShardsAcknowledged() - ); - serverResponse.toXContent(builder, params); - } -} diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index 6b7a82a11ae2b..c6134cd5579df 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -19,8 +19,7 @@ one or more conditions that determine when the index has to be rolled over: include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> The alias (first argument) that points to the index to rollover, and -the name of the new index in case the rollover operation is performed. -The new index argument is optional, and can be set to null +optionally the name of the new index in case the rollover operation is performed <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index @@ -40,20 +39,24 @@ include-tagged::{doc-tests-file}[{api}-request-timeout] -------------------------------------------------- <1> Timeout to wait for the all the nodes to acknowledge the index is opened as a `TimeValue` +<2> Timeout to wait for the all the nodes to acknowledge the index is opened +as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- <1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> Sets the number of active shard copies to wait for before the rollover -index API returns a response -<2> Resets the number of active shard copies to wait for to the default value +<1> The number of active shard copies to wait for before the rollover index API +returns a response, as an `int` +<2> The number of active shard copies to wait for before the rollover index API +returns a response, as an `ActiveShardCount` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -95,3 +98,5 @@ each shard in the index before timing out <5> Whether the index has been rolled over <6> Whether the operation was performed or it was a dry run <7> The different conditions and whether they were matched or not + + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 7bf1513969fb3..5e5ba1367ad3e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -18,10 +18,6 @@ } }, "params": { - "include_type_name": { - "type" : "boolean", - "description" : "Whether a type should be included in the body of the mappings." - }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml deleted file mode 100644 index 7ed78c6e3159a..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -"Typeless mapping": - - skip: - version: " - 6.6.99" - reason: include_type_name was introduced in 6.7.0 - - - do: - indices.create: - include_type_name: false - index: logs-1 - body: - aliases: - logs_search: {} - - # index first document and wait for refresh - - do: - index: - index: logs-1 - type: _doc - id: "1" - body: { "foo": "hello world" } - refresh: true - - # index second document and wait for refresh - - do: - index: - index: logs-1 - type: _doc - id: "2" - body: { "foo": "hello world" } - refresh: true - - # perform alias rollover with new typeless mapping - - do: - indices.rollover: - include_type_name: false - alias: "logs_search" - body: - conditions: - max_docs: 2 - mappings: - properties: - foo2: - type: keyword - - - match: { conditions: { "[max_docs: 2]": true } } - - match: { rolled_over: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml deleted file mode 100644 index c5d0bd518937e..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -"Typeless mapping": - - skip: - version: " - 6.6.99" - reason: include_type_name was introduced in 6.7.0 - - - do: - indices.create: - index: logs-1 - body: - aliases: - logs_search: {} - - # index first document and wait for refresh - - do: - index: - index: logs-1 - type: _doc - id: "1" - body: { "foo": "hello world" } - refresh: true - - # index second document and wait for refresh - - do: - index: - index: logs-1 - type: _doc - id: "2" - body: { "foo": "hello world" } - refresh: true - - # perform alias rollover with new typeless mapping - - do: - indices.rollover: - include_type_name: true - alias: "logs_search" - body: - conditions: - max_docs: 2 - mappings: - _doc: - properties: - foo2: - type: keyword - - - match: { conditions: { "[max_docs: 2]": true } } - - match: { rolled_over: true } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index a2cad7a35a398..1b385ed9d0dbc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -75,10 +75,6 @@ public T value() { return value; } - public String name() { - return name; - } - /** * Holder for index stats used to evaluate conditions */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index 49db34aa5386e..f36636594a4d6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -42,13 +41,10 @@ /** * Request class to swap index under an alias upon satisfying conditions - * - * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. - * Any changes done to this class should also go to that client class. */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -70,14 +66,9 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, isTypeIncluded) -> { - if (isTypeIncluded) { - for (Map.Entry mappingsEntry : parser.map().entrySet()) { - request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); - } - } else { - // a type is not included, add a dummy _doc type - request.createIndexRequest.mapping(MapperService.SINGLE_MAPPING_NAME, parser.map()); + PARSER.declareField((parser, request, context) -> { + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); } }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), @@ -239,8 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - // param isTypeIncluded decides how mappings should be parsed from XContent - public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { - PARSER.parse(parser, this, isTypeIncluded); + public void fromXContent(XContentParser parser) throws IOException { + PARSER.parse(parser, this, null); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java index 78bdde7b7bd18..356f805c24bd4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java @@ -37,13 +37,6 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; - -/** - * Response object for {@link RolloverRequest} API - * - * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. - * Any changes done to this class should also go to that client class. - */ public final class RolloverResponse extends ShardsAcknowledgedResponse implements ToXContentObject { private static final ParseField NEW_INDEX = new ParseField("new_index"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index a7e0b3b22d6f2..489001bf2a14f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -19,11 +19,9 @@ package org.elasticsearch.rest.action.admin.indices; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -33,11 +31,6 @@ import java.io.IOException; public class RestRolloverIndexAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger( - LogManager.getLogger(RestRolloverIndexAction.class)); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of rollover " + - "index requests will change in 7.0. Please start using the include_type_name parameter set to false " + - "to move to the new, typeless response format that will become the default."; public RestRolloverIndexAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this); @@ -51,12 +44,8 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); - if (includeTypeName) { - deprecationLogger.deprecatedAndMaybeLog("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); - } RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); + request.applyContentParser(rolloverIndexRequest::fromXContent); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index e4ccb70f4862a..1e8d8e2a2932c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -52,6 +52,7 @@ import static org.hamcrest.Matchers.equalTo; public class RolloverRequestTests extends ESTestCase { + private NamedWriteableRegistry writeableRegistry; @Override @@ -71,7 +72,7 @@ public void testConditionsParsing() throws Exception { .field("max_size", "45gb") .endObject() .endObject(); - request.fromXContent(false, createParser(builder)); + request.fromXContent(createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(3)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -107,7 +108,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(true, createParser(builder)); + request.fromXContent(createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1)); @@ -146,7 +147,7 @@ public void testToAndFromXContent() throws IOException { BytesReference originalBytes = toShuffledXContent(rolloverRequest, xContentType, EMPTY_PARAMS, humanReadable); RolloverRequest parsedRolloverRequest = new RolloverRequest(); - parsedRolloverRequest.fromXContent(true, createParser(xContentType.xContent(), originalBytes)); + parsedRolloverRequest.fromXContent(createParser(xContentType.xContent(), originalBytes)); CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest(); CreateIndexRequest parsedCreateIndexRequest = parsedRolloverRequest.getCreateIndexRequest(); @@ -171,7 +172,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index 05eaefffaebf0..e0f58ecd92daf 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.action.admin.indices.rollover; import org.elasticsearch.Version; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractStreamableXContentTestCase; @@ -58,7 +57,7 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java index 9732504cac6d4..345ef1f58bcac 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java @@ -126,7 +126,7 @@ public static void randomAliases(CreateIndexRequest request) { } } - public static Alias randomAlias() { + private static Alias randomAlias() { Alias alias = new Alias(randomAlphaOfLength(5)); if (randomBoolean()) { From 65b0493e868cedb0a9163b4d416e1883e5fd8f04 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 5 Feb 2019 14:14:38 +0000 Subject: [PATCH 04/25] Deprecate DiscoveryPlugin#getDiscoveryTypes (#38410) Marks `DiscoveryPlugin#getDiscoveryTypes` as deprecated since we intend to remove the ability for plugins to provide their own discovery type in a future version. Relates #38414 --- .../main/java/org/elasticsearch/plugins/DiscoveryPlugin.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java b/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java index 1d5ee13f50dcd..c49f615cfbb29 100644 --- a/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java @@ -66,7 +66,10 @@ public interface DiscoveryPlugin { * @param clusterApplier Use to locally apply cluster state updates * @param clusterSettings Use to get cluster settings * @param hostsProvider Use to find configured hosts which should be pinged for initial discovery + * + * @deprecated Future versions will not support pluggable discovery implementations. */ + @Deprecated default Map> getDiscoveryTypes(ThreadPool threadPool, TransportService transportService, NamedWriteableRegistry namedWriteableRegistry, MasterService masterService, From 5e9688b4db94dbf69056fd83cb5373a55093e501 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Tue, 5 Feb 2019 17:01:12 +0200 Subject: [PATCH 05/25] Revert "Backport of types removal for Put/Get index templates (#38022)" (#38427) This reverts commit ca21cb2838e16864bb741f85f4e6eef7249ea932. --- .../elasticsearch/client/IndicesClient.java | 109 +---- .../client/IndicesRequestConverters.java | 43 +- .../client/RequestConverters.java | 9 + .../indices/GetIndexTemplatesResponse.java | 90 ---- .../client/indices/IndexTemplateMetaData.java | 300 ------------ .../indices/PutIndexTemplateRequest.java | 453 ------------------ .../elasticsearch/client/IndicesClientIT.java | 242 +--------- .../client/IndicesRequestConvertersTests.java | 61 +-- .../client/RestHighLevelClientTests.java | 13 +- .../IndicesClientDocumentationIT.java | 77 +-- .../GetIndexTemplatesResponseTests.java | 140 ------ .../indices/PutIndexTemplateRequestTests.java | 116 ----- .../high-level/indices/put_template.asciidoc | 10 +- .../get/GetIndexTemplatesResponse.java | 2 +- .../indices/RestGetIndexTemplateAction.java | 11 +- .../indices/RestPutIndexTemplateAction.java | 8 +- .../RestPutIndexTemplateActionTests.java | 64 +-- 17 files changed, 118 insertions(+), 1630 deletions(-) delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 17d803a713a7b..f484d7375ba6f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -50,6 +50,8 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -60,9 +62,7 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.rest.RestStatus; @@ -1341,7 +1341,6 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques AcknowledgedResponse::fromXContent, listener, emptySet()); } - /** * Asynchronously updates specific index level settings using the Update Indices Settings API. *

@@ -1364,48 +1363,9 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Action * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated This old form of request allows types in mappings. Use {@link #putTemplate(PutIndexTemplateRequest, RequestOptions)} - * instead which introduces a new request object without types. */ - @Deprecated - public AcknowledgedResponse putTemplate( - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, - AcknowledgedResponse::fromXContent, emptySet()); - } - - /** - * Asynchronously puts an index template using the Index Templates API. - * See Index Templates API - * on elastic.co - * @param putIndexTemplateRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - * @deprecated This old form of request allows types in mappings. - * Use {@link #putTemplateAsync(PutIndexTemplateRequest, RequestOptions, ActionListener)} - * instead which introduces a new request object without types. - */ - @Deprecated - public void putTemplateAsync(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, - AcknowledgedResponse::fromXContent, listener, emptySet()); - } - - - /** - * Puts an index template using the Index Templates API. - * See Index Templates API - * on elastic.co - * @param putIndexTemplateRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - * @throws IOException in case there is a problem sending the request or parsing back the response - */ - public AcknowledgedResponse putTemplate( - PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options) throws IOException { + public AcknowledgedResponse putTemplate(PutIndexTemplateRequest putIndexTemplateRequest, + RequestOptions options) throws IOException { return restHighLevelClient.performRequestAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, emptySet()); } @@ -1418,8 +1378,8 @@ public AcknowledgedResponse putTemplate( * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ - public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, - RequestOptions options, ActionListener listener) { + public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, RequestOptions options, + ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(putIndexTemplateRequest, IndicesRequestConverters::putTemplate, options, AcknowledgedResponse::fromXContent, listener, emptySet()); } @@ -1454,61 +1414,21 @@ public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, Reques ValidateQueryResponse::fromXContent, listener, emptySet()); } - /** - * Gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, where the - * mapping definition is nested under the type name. - * See Index Templates API - * on elastic.co - * @param getIndexTemplatesRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - * @throws IOException in case there is a problem sending the request or parsing back the response - * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use - * {@link #getIndexTemplate(GetIndexTemplatesRequest, RequestOptions)} instead which returns a new response object - */ - @Deprecated - public org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate( - GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, - IndicesRequestConverters::getTemplatesWithDocumentTypes, - options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, emptySet()); - } - /** * Gets index templates using the Index Templates API * See Index Templates API * on elastic.co - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param getIndexTemplatesRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @return the response * @throws IOException in case there is a problem sending the request or parsing back the response */ - public GetIndexTemplatesResponse getIndexTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options) - throws IOException { - return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, - IndicesRequestConverters::getTemplates, + public GetIndexTemplatesResponse getTemplate(GetIndexTemplatesRequest getIndexTemplatesRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, emptySet()); - } - - /** - * Asynchronously gets index templates using the Index Templates API. The mappings will be returned in a legacy deprecated format, - * where the mapping definition is nested under the type name. - * See Index Templates API - * on elastic.co - * @param getIndexTemplatesRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - * @deprecated This method uses an old response object which still refers to types, a deprecated feature. Use - * {@link #getIndexTemplateAsync(GetIndexTemplatesRequest, RequestOptions, ActionListener)} instead which returns a new response object - */ - @Deprecated - public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, - ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, - IndicesRequestConverters::getTemplatesWithDocumentTypes, - options, org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse::fromXContent, listener, emptySet()); } - + /** * Asynchronously gets index templates using the Index Templates API * See Index Templates API @@ -1517,12 +1437,11 @@ public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion */ - public void getIndexTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, + public void getTemplateAsync(GetIndexTemplatesRequest getIndexTemplatesRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, - IndicesRequestConverters::getTemplates, + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexTemplatesRequest, IndicesRequestConverters::getTemplates, options, GetIndexTemplatesResponse::fromXContent, listener, emptySet()); - } + } /** * Uses the Index Templates API to determine if index templates exist diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index a600b4a2eba3a..d26f9babed889 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -43,6 +43,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.indices.CreateIndexRequest; @@ -50,11 +51,9 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.rest.BaseRestHandler; import java.io.IOException; import java.util.Locale; @@ -389,7 +388,7 @@ static Request getIndex(GetIndexRequest getIndexRequest) { params.withHuman(getIndexRequest.humanReadable()); params.withMasterTimeout(getIndexRequest.masterNodeTimeout()); // Force "include_type_name" parameter since responses need to be compatible when coming from 7.0 nodes - params.putParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); + params.withIncludeTypeName(true); return request; } @@ -424,38 +423,12 @@ static Request indexPutSettings(UpdateSettingsRequest updateSettingsRequest) thr return request; } - /** - * @deprecated This uses the old form of PutIndexTemplateRequest which uses types. - * Use (@link {@link #putTemplate(PutIndexTemplateRequest)} instead - */ - @Deprecated - static Request putTemplate(org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putIndexTemplateRequest) - throws IOException { - String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") - .addPathPart(putIndexTemplateRequest.name()).build(); - Request request = new Request(HttpPut.METHOD_NAME, endpoint); - RequestConverters.Params params = new RequestConverters.Params(request); - params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); - params.putParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); - if (putIndexTemplateRequest.create()) { - params.putParam("create", Boolean.TRUE.toString()); - } - if (Strings.hasText(putIndexTemplateRequest.cause())) { - params.putParam("cause", putIndexTemplateRequest.cause()); - } - request.setEntity(RequestConverters.createEntity(putIndexTemplateRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); - return request; - } - static Request putTemplate(PutIndexTemplateRequest putIndexTemplateRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_template") .addPathPart(putIndexTemplateRequest.name()).build(); Request request = new Request(HttpPut.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); params.withMasterTimeout(putIndexTemplateRequest.masterNodeTimeout()); - if (putIndexTemplateRequest.mappings() != null) { - params.putParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, Boolean.FALSE.toString()); - } if (putIndexTemplateRequest.create()) { params.putParam("create", Boolean.TRUE.toString()); } @@ -491,16 +464,7 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) { return request; } - @Deprecated - static Request getTemplatesWithDocumentTypes(GetIndexTemplatesRequest getIndexTemplatesRequest) { - return getTemplates(getIndexTemplatesRequest, true); - } - static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) { - return getTemplates(getIndexTemplatesRequest, false); - } - - private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest, boolean includeTypeName) { final String endpoint = new RequestConverters.EndpointBuilder() .addPathPartAsIs("_template") .addCommaSeparatedPathParts(getIndexTemplatesRequest.names()) @@ -509,9 +473,8 @@ private static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRe final RequestConverters.Params params = new RequestConverters.Params(request); params.withLocal(getIndexTemplatesRequest.isLocal()); params.withMasterTimeout(getIndexTemplatesRequest.getMasterNodeTimeout()); - params.putParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, Boolean.toString(includeTypeName)); return request; - } + } static Request templatesExist(IndexTemplatesExistRequest indexTemplatesExistRequest) { final String endpoint = new RequestConverters.EndpointBuilder() diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 3ad1ecfc106f8..5af5e9ed9d825 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -76,6 +76,7 @@ import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; @@ -949,6 +950,14 @@ Params withIncludeDefaults(boolean includeDefaults) { return this; } + Params withIncludeTypeName(boolean includeTypeName) { + if (includeTypeName) { + return putParam(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, + Boolean.toString(BaseRestHandler.DEFAULT_INCLUDE_TYPE_NAME_POLICY)); + } + return this; + } + Params withPreserveExisting(boolean preserveExisting) { if (preserveExisting) { return putParam("preserve_existing", Boolean.TRUE.toString()); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java deleted file mode 100644 index ef283b31c0634..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexTemplatesResponse.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.client.indices; - -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; - - -public class GetIndexTemplatesResponse { - - @Override - public String toString() { - List thisList = new ArrayList<>(this.indexTemplates); - thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); - return "GetIndexTemplatesResponse [indexTemplates=" + thisList + "]"; - } - - private final List indexTemplates; - - GetIndexTemplatesResponse() { - indexTemplates = new ArrayList<>(); - } - - GetIndexTemplatesResponse(List indexTemplates) { - this.indexTemplates = indexTemplates; - } - - public List getIndexTemplates() { - return indexTemplates; - } - - - public static GetIndexTemplatesResponse fromXContent(XContentParser parser) throws IOException { - final List templates = new ArrayList<>(); - for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { - if (token == XContentParser.Token.FIELD_NAME) { - final IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()); - templates.add(templateMetaData); - } - } - return new GetIndexTemplatesResponse(templates); - } - - @Override - public int hashCode() { - List sortedList = new ArrayList<>(this.indexTemplates); - sortedList.sort(Comparator.comparing(IndexTemplateMetaData::name)); - return Objects.hash(sortedList); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - // To compare results we need to make sure the templates are listed in the same order - GetIndexTemplatesResponse other = (GetIndexTemplatesResponse) obj; - List thisList = new ArrayList<>(this.indexTemplates); - List otherList = new ArrayList<>(other.indexTemplates); - thisList.sort(Comparator.comparing(IndexTemplateMetaData::name)); - otherList.sort(Comparator.comparing(IndexTemplateMetaData::name)); - return Objects.equals(thisList, otherList); - } - - -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java deleted file mode 100644 index 12fc747ab3473..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.client.indices; - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.MapperService; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class IndexTemplateMetaData { - - - private final String name; - - private final int order; - - /** - * The version is an arbitrary number managed by the user so that they can easily and quickly verify the existence of a given template. - * Expected usage: - * 


-     * PUT /_template/my_template
-     * {
-     *   "index_patterns": ["my_index-*"],
-     *   "mappings": { ... },
-     *   "version": 1
-     * }
-     *
- * Then, some process from the user can occasionally verify that the template exists with the appropriate version without having to - * check the template's content: - * 

-     * GET /_template/my_template?filter_path=*.version
-     *
- */ - @Nullable - private final Integer version; - - private final List patterns; - - private final Settings settings; - - private final MappingMetaData mappings; - - private final ImmutableOpenMap aliases; - - public IndexTemplateMetaData(String name, int order, Integer version, - List patterns, Settings settings, - MappingMetaData mappings, - ImmutableOpenMap aliases) { - if (patterns == null || patterns.isEmpty()) { - throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns); - } - this.name = name; - this.order = order; - this.version = version; - this.patterns= patterns; - this.settings = settings; - this.mappings = mappings; - this.aliases = aliases; - } - - public String name() { - return this.name; - } - - public int order() { - return this.order; - } - - @Nullable - public Integer version() { - return version; - } - - public List patterns() { - return this.patterns; - } - - public Settings settings() { - return this.settings; - } - - public MappingMetaData mappings() { - return this.mappings; - } - - public ImmutableOpenMap aliases() { - return this.aliases; - } - - public static Builder builder(String name) { - return new Builder(name); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - IndexTemplateMetaData that = (IndexTemplateMetaData) o; - - if (order != that.order) return false; - if (!Objects.equals(mappings, that.mappings)) return false; - if (!name.equals(that.name)) return false; - if (!settings.equals(that.settings)) return false; - if (!patterns.equals(that.patterns)) return false; - - return Objects.equals(version, that.version); - } - - @Override - public int hashCode() { - return Objects.hash(name, order, version, patterns, settings, mappings); - } - - public static class Builder { - - private static final Set VALID_FIELDS = Sets.newHashSet( - "template", "order", "mappings", "settings", "index_patterns", "aliases", "version"); - - private String name; - - private int order; - - private Integer version; - - private List indexPatterns; - - private Settings settings = Settings.Builder.EMPTY_SETTINGS; - - private MappingMetaData mappings; - - private final ImmutableOpenMap.Builder aliases; - - public Builder(String name) { - this.name = name; - mappings = null; - aliases = ImmutableOpenMap.builder(); - } - - public Builder(IndexTemplateMetaData indexTemplateMetaData) { - this.name = indexTemplateMetaData.name(); - order(indexTemplateMetaData.order()); - version(indexTemplateMetaData.version()); - patterns(indexTemplateMetaData.patterns()); - settings(indexTemplateMetaData.settings()); - - mappings = indexTemplateMetaData.mappings(); - aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases()); - } - - public Builder order(int order) { - this.order = order; - return this; - } - - public Builder version(Integer version) { - this.version = version; - return this; - } - - public Builder patterns(List indexPatterns) { - this.indexPatterns = indexPatterns; - return this; - } - - - public Builder settings(Settings.Builder settings) { - this.settings = settings.build(); - return this; - } - - public Builder settings(Settings settings) { - this.settings = settings; - return this; - } - - public Builder mapping(MappingMetaData mappings) { - this.mappings = mappings; - return this; - } - - public Builder putAlias(AliasMetaData aliasMetaData) { - aliases.put(aliasMetaData.alias(), aliasMetaData); - return this; - } - - public Builder putAlias(AliasMetaData.Builder aliasMetaData) { - aliases.put(aliasMetaData.alias(), aliasMetaData.build()); - return this; - } - - public IndexTemplateMetaData build() { - return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings, aliases.build()); - } - - - public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException { - Builder builder = new Builder(templateName); - - String currentFieldName = skipTemplateName(parser); - XContentParser.Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.START_OBJECT) { - if ("settings".equals(currentFieldName)) { - Settings.Builder templateSettingsBuilder = Settings.builder(); - templateSettingsBuilder.put(Settings.fromXContent(parser)); - templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX); - builder.settings(templateSettingsBuilder.build()); - } else if ("mappings".equals(currentFieldName)) { - Map mapping = parser.map(); - if (mapping.isEmpty() == false) { - MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping); - builder.mapping(md); - } - } else if ("aliases".equals(currentFieldName)) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); - } - } else { - throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if ("mappings".equals(currentFieldName)) { - // The server-side IndexTemplateMetaData has toXContent impl that can return mappings - // in an array but also a comment saying this never happens with typeless APIs. - throw new ElasticsearchParseException("Invalid response format - " - + "mappings are not expected to be returned in an array", currentFieldName); - } else if ("index_patterns".equals(currentFieldName)) { - List index_patterns = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - index_patterns.add(parser.text()); - } - builder.patterns(index_patterns); - } - } else if (token.isValue()) { - // Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009) - if("template".equals(currentFieldName)) { - builder.patterns(Collections.singletonList(parser.text())); - } else if ("order".equals(currentFieldName)) { - builder.order(parser.intValue()); - } else if ("version".equals(currentFieldName)) { - builder.version(parser.intValue()); - } - } - } - return builder.build(); - } - - private static String skipTemplateName(XContentParser parser) throws IOException { - XContentParser.Token token = parser.nextToken(); - if (token == XContentParser.Token.START_OBJECT) { - token = parser.nextToken(); - if (token == XContentParser.Token.FIELD_NAME) { - String currentFieldName = parser.currentName(); - if (VALID_FIELDS.contains(currentFieldName)) { - return currentFieldName; - } else { - // we just hit the template name, which should be ignored and we move on - parser.nextToken(); - } - } - } - - return null; - } - } - -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java deleted file mode 100644 index 5f22691b046eb..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutIndexTemplateRequest.java +++ /dev/null @@ -1,453 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.client.indices; - -import org.elasticsearch.ElasticsearchGenerationException; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.common.xcontent.support.XContentMapValues; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; - -/** - * A request to create an index template. - */ -public class PutIndexTemplateRequest extends MasterNodeRequest implements IndicesRequest, ToXContent { - - private String name; - - private String cause = ""; - - private List indexPatterns; - - private int order; - - private boolean create; - - private Settings settings = EMPTY_SETTINGS; - - private BytesReference mappings = null; - - private final Set aliases = new HashSet<>(); - - private Integer version; - - /** - * Constructs a new put index template request with the provided name. - */ - public PutIndexTemplateRequest(String name) { - this.name(name); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; - if (indexPatterns == null || indexPatterns.size() == 0) { - validationException = addValidationError("index patterns are missing", validationException); - } - return validationException; - } - - /** - * Sets the name of the index template. - */ - public PutIndexTemplateRequest name(String name) { - if(name == null) { - throw new IllegalArgumentException("Name cannot be null"); - } - this.name = name; - return this; - } - - /** - * The name of the index template. - */ - public String name() { - return this.name; - } - - public PutIndexTemplateRequest patterns(List indexPatterns) { - this.indexPatterns = indexPatterns; - return this; - } - - public List patterns() { - return this.indexPatterns; - } - - public PutIndexTemplateRequest order(int order) { - this.order = order; - return this; - } - - public int order() { - return this.order; - } - - public PutIndexTemplateRequest version(Integer version) { - this.version = version; - return this; - } - - public Integer version() { - return this.version; - } - - /** - * Set to {@code true} to force only creation, not an update of an index template. If it already - * exists, it will fail with an {@link IllegalArgumentException}. - */ - public PutIndexTemplateRequest create(boolean create) { - this.create = create; - return this; - } - - public boolean create() { - return create; - } - - /** - * The settings to create the index template with. - */ - public PutIndexTemplateRequest settings(Settings settings) { - this.settings = settings; - return this; - } - - /** - * The settings to create the index template with. - */ - public PutIndexTemplateRequest settings(Settings.Builder settings) { - this.settings = settings.build(); - return this; - } - - /** - * The settings to create the index template with (either json/yaml format). - */ - public PutIndexTemplateRequest settings(String source, XContentType xContentType) { - this.settings = Settings.builder().loadFromSource(source, xContentType).build(); - return this; - } - - /** - * The settings to create the index template with (either json or yaml format). - */ - public PutIndexTemplateRequest settings(Map source) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - builder.map(source); - settings(Strings.toString(builder), XContentType.JSON); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); - } - return this; - } - - public Settings settings() { - return this.settings; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * @param source The mapping source - * @param xContentType The type of content contained within the source - */ - public PutIndexTemplateRequest mapping(String source, XContentType xContentType) { - internalMapping(XContentHelper.convertToMap(new BytesArray(source), true, xContentType).v2()); - return this; - } - - /** - * The cause for this index template creation. - */ - public PutIndexTemplateRequest cause(String cause) { - this.cause = cause; - return this; - } - - public String cause() { - return this.cause; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * @param source The mapping source - */ - public PutIndexTemplateRequest mapping(XContentBuilder source) { - internalMapping(XContentHelper.convertToMap(BytesReference.bytes(source), - true, source.contentType()).v2()); - return this; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * @param source The mapping source - * @param xContentType the source content type - */ - public PutIndexTemplateRequest mapping(BytesReference source, XContentType xContentType) { - internalMapping(XContentHelper.convertToMap(source, true, xContentType).v2()); - return this; - } - - /** - * Adds mapping that will be added when the index gets created. - * - * @param source The mapping source - */ - public PutIndexTemplateRequest mapping(Map source) { - return internalMapping(source); - } - - private PutIndexTemplateRequest internalMapping(Map source) { - try { - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - builder.map(source); - Objects.requireNonNull(builder.contentType()); - try { - mappings = new BytesArray( - XContentHelper.convertToJson(BytesReference.bytes(builder), false, false, builder.contentType())); - return this; - } catch (IOException e) { - throw new UncheckedIOException("failed to convert source to json", e); - } - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); - } - } - - public BytesReference mappings() { - return this.mappings; - } - - /** - * The template source definition. - */ - public PutIndexTemplateRequest source(XContentBuilder templateBuilder) { - try { - return source(BytesReference.bytes(templateBuilder), templateBuilder.contentType()); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to build json for template request", e); - } - } - - /** - * The template source definition. - */ - @SuppressWarnings("unchecked") - public PutIndexTemplateRequest source(Map templateSource) { - Map source = templateSource; - for (Map.Entry entry : source.entrySet()) { - String name = entry.getKey(); - if (name.equals("template")) { - if(entry.getValue() instanceof String) { - patterns(Collections.singletonList((String) entry.getValue())); - } - } else if (name.equals("index_patterns")) { - if(entry.getValue() instanceof String) { - patterns(Collections.singletonList((String) entry.getValue())); - } else if (entry.getValue() instanceof List) { - List elements = ((List) entry.getValue()).stream().map(Object::toString).collect(Collectors.toList()); - patterns(elements); - } else { - throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings"); - } - } else if (name.equals("order")) { - order(XContentMapValues.nodeIntegerValue(entry.getValue(), order())); - } else if ("version".equals(name)) { - if ((entry.getValue() instanceof Integer) == false) { - throw new IllegalArgumentException("Malformed [version] value, should be an integer"); - } - version((Integer)entry.getValue()); - } else if (name.equals("settings")) { - if ((entry.getValue() instanceof Map) == false) { - throw new IllegalArgumentException("Malformed [settings] section, should include an inner object"); - } - settings((Map) entry.getValue()); - } else if (name.equals("mappings")) { - Map mappings = (Map) entry.getValue(); - mapping(mappings); - } else if (name.equals("aliases")) { - aliases((Map) entry.getValue()); - } else { - throw new ElasticsearchParseException("unknown key [{}] in the template ", name); - } - } - return this; - } - - /** - * The template source definition. - */ - public PutIndexTemplateRequest source(String templateSource, XContentType xContentType) { - return source(XContentHelper.convertToMap(xContentType.xContent(), templateSource, true)); - } - - /** - * The template source definition. - */ - public PutIndexTemplateRequest source(byte[] source, XContentType xContentType) { - return source(source, 0, source.length, xContentType); - } - - /** - * The template source definition. - */ - public PutIndexTemplateRequest source(byte[] source, int offset, int length, XContentType xContentType) { - return source(new BytesArray(source, offset, length), xContentType); - } - - /** - * The template source definition. - */ - public PutIndexTemplateRequest source(BytesReference source, XContentType xContentType) { - return source(XContentHelper.convertToMap(source, true, xContentType).v2()); - } - - - public Set aliases() { - return this.aliases; - } - - /** - * Sets the aliases that will be associated with the index when it gets created - */ - public PutIndexTemplateRequest aliases(Map source) { - try { - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.map(source); - return aliases(BytesReference.bytes(builder)); - } catch (IOException e) { - throw new ElasticsearchGenerationException("Failed to generate [" + source + "]", e); - } - } - - /** - * Sets the aliases that will be associated with the index when it gets created - */ - public PutIndexTemplateRequest aliases(XContentBuilder source) { - return aliases(BytesReference.bytes(source)); - } - - /** - * Sets the aliases that will be associated with the index when it gets created - */ - public PutIndexTemplateRequest aliases(String source) { - return aliases(new BytesArray(source)); - } - - /** - * Sets the aliases that will be associated with the index when it gets created - */ - public PutIndexTemplateRequest aliases(BytesReference source) { - // EMPTY is safe here because we never call namedObject - try (XContentParser parser = XContentHelper - .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, source)) { - //move to the first alias - parser.nextToken(); - while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { - alias(Alias.fromXContent(parser)); - } - return this; - } catch(IOException e) { - throw new ElasticsearchParseException("Failed to parse aliases", e); - } - } - - /** - * Adds an alias that will be added when the index gets created. - * - * @param alias The metadata for the new alias - * @return the index template creation request - */ - public PutIndexTemplateRequest alias(Alias alias) { - aliases.add(alias); - return this; - } - - @Override - public String[] indices() { - return indexPatterns.toArray(new String[indexPatterns.size()]); - } - - @Override - public IndicesOptions indicesOptions() { - return IndicesOptions.strictExpand(); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("index_patterns", indexPatterns); - builder.field("order", order); - if (version != null) { - builder.field("version", version); - } - - builder.startObject("settings"); - settings.toXContent(builder, params); - builder.endObject(); - - if (mappings != null) { - builder.field("mappings"); - try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappings.utf8ToString())) { - builder.copyCurrentStructure(parser); - } - } - - builder.startObject("aliases"); - for (Alias alias : aliases) { - alias.toXContent(builder, params); - } - builder.endObject(); - - return builder; - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 59c834fba0fee..c549399d193a4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -56,6 +56,8 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.elasticsearch.action.index.IndexRequest; @@ -71,14 +73,12 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; -import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; @@ -87,7 +87,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexSettings; @@ -99,8 +98,6 @@ import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; -import org.elasticsearch.rest.action.admin.indices.RestGetIndexTemplateAction; -import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import java.io.IOException; import java.util.Arrays; @@ -1448,9 +1445,8 @@ public void testIndexPutSettingNonExistent() throws IOException { } @SuppressWarnings("unchecked") - public void testPutTemplateWithTypes() throws Exception { - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = - new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() + public void testPutTemplate() throws Exception { + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest() .name("my-template") .patterns(Arrays.asList("pattern-1", "name-*")) .order(10) @@ -1460,9 +1456,7 @@ public void testPutTemplateWithTypes() throws Exception { .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, - expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) - ); + highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); Map templates = getAsMap("/_template/my-template"); @@ -1476,94 +1470,6 @@ public void testPutTemplateWithTypes() throws Exception { assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); } - - @SuppressWarnings("unchecked") - public void testPutTemplate() throws Exception { - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") - .patterns(Arrays.asList("pattern-1", "name-*")) - .order(10) - .create(randomBoolean()) - .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) - .mapping("{ \"properties\":{" - + "\"host_name\": {\"type\":\"keyword\"}" - + "}" - + "}", XContentType.JSON) - .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); - - AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync); - assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); - - Map templates = getAsMap("/_template/my-template?include_type_name=false"); - assertThat(templates.keySet(), hasSize(1)); - assertThat(extractValue("my-template.order", templates), equalTo(10)); - assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); - assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); - assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); - assertThat(extractValue("my-template.mappings.properties.host_name.type", templates), equalTo("keyword")); - assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); - assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); - } - - public void testPutTemplateWithTypesUsingUntypedAPI() throws Exception { - PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest("my-template") - .patterns(Arrays.asList("pattern-1", "name-*")) - .order(10) - .create(randomBoolean()) - .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) - .mapping("{ " - + "\"my_doc_type\":{" - + "\"properties\":{" - + "\"host_name\": {\"type\":\"keyword\"}" - + "}" - + "}" - + "}", XContentType.JSON) - .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); - - - ElasticsearchStatusException badMappingError = expectThrows(ElasticsearchStatusException.class, - () -> execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync)); - assertThat(badMappingError.getDetailedMessage(), - containsString("Root mapping definition has unsupported parameters: [my_doc_type")); - } - - @SuppressWarnings("unchecked") - public void testPutTemplateWithNoTypesUsingTypedApi() throws Exception { - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = - new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() - .name("my-template") - .patterns(Arrays.asList("pattern-1", "name-*")) - .order(10) - .create(randomBoolean()) - .settings(Settings.builder().put("number_of_shards", "3").put("number_of_replicas", "0")) - .mapping("my_doc_type", - // Note that the declared type is missing from the mapping - "{ " - + "\"properties\":{" - + "\"host_name\": {\"type\":\"keyword\"}," - + "\"description\": {\"type\":\"text\"}" - + "}" - + "}", XContentType.JSON) - .alias(new Alias("alias-1").indexRouting("abc")).alias(new Alias("{index}-write").searchRouting("xyz")); - - AcknowledgedResponse putTemplateResponse = execute(putTemplateRequest, - highLevelClient().indices()::putTemplate, highLevelClient().indices()::putTemplateAsync, - expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE) - ); - assertThat(putTemplateResponse.isAcknowledged(), equalTo(true)); - - Map templates = getAsMap("/_template/my-template"); - assertThat(templates.keySet(), hasSize(1)); - assertThat(extractValue("my-template.order", templates), equalTo(10)); - assertThat(extractRawValues("my-template.index_patterns", templates), contains("pattern-1", "name-*")); - assertThat(extractValue("my-template.settings.index.number_of_shards", templates), equalTo("3")); - assertThat(extractValue("my-template.settings.index.number_of_replicas", templates), equalTo("0")); - assertThat(extractValue("my-template.mappings.my_doc_type.properties.host_name.type", templates), equalTo("keyword")); - assertThat(extractValue("my-template.mappings.my_doc_type.properties.description.type", templates), equalTo("text")); - assertThat((Map) extractValue("my-template.aliases.alias-1", templates), hasEntry("index_routing", "abc")); - assertThat((Map) extractValue("my-template.aliases.{index}-write", templates), hasEntry("search_routing", "xyz")); - } public void testPutTemplateBadRequests() throws Exception { RestHighLevelClient client = highLevelClient(); @@ -1628,167 +1534,68 @@ public void testInvalidValidateQuery() throws IOException{ assertFalse(response.isValid()); } - // Tests the deprecated form of the API that returns templates with doc types (using the server-side's GetIndexTemplateResponse) - public void testCRUDIndexTemplateWithTypes() throws Exception { - RestHighLevelClient client = highLevelClient(); - - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate1 = - new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-1") - .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); - assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync - , expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) - .isAcknowledged(), equalTo(true)); - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplate2 = - new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest().name("template-2") - .patterns(Arrays.asList("pattern-2", "name-2")) - .mapping("custom_doc_type", "name", "type=text") - .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync, - expectWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)) - .isAcknowledged(), equalTo(true)); - - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate1 = execute( - new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); - assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); - org.elasticsearch.cluster.metadata.IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); - assertThat(template1.name(), equalTo("template-1")); - assertThat(template1.patterns(), contains("pattern-1", "name-1")); - assertTrue(template1.aliases().containsKey("alias-1")); - - //Check the typed version of the call - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getTemplate2 = - execute(new GetIndexTemplatesRequest("template-2"), - client.indices()::getTemplate, client.indices()::getTemplateAsync); - assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); - org.elasticsearch.cluster.metadata.IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); - assertThat(template2.name(), equalTo("template-2")); - assertThat(template2.patterns(), contains("pattern-2", "name-2")); - assertTrue(template2.aliases().isEmpty()); - assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); - // Ugly deprecated form of API requires use of doc type to get at mapping object which is CompressedXContent - assertTrue(template2.mappings().containsKey("custom_doc_type")); - - List names = randomBoolean() - ? Arrays.asList("*-1", "template-2") - : Arrays.asList("template-*"); - GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getBoth = execute( - getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); - assertThat(getBoth.getIndexTemplates(), hasSize(2)); - assertThat(getBoth.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName).toArray(), - arrayContainingInAnyOrder("template-1", "template-2")); - - GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse getAll = execute( - getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)); - assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); - assertThat(getAll.getIndexTemplates().stream().map(org.elasticsearch.cluster.metadata.IndexTemplateMetaData::getName) - .collect(Collectors.toList()), - hasItems("template-1", "template-2")); - - assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), - client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); - assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); - assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), - client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); - - assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates(), hasSize(1)); - assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE)).getIndexTemplates() - .get(0).name(), equalTo("template-2")); - - assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), - client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); - assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getTemplate, client.indices()::getTemplateAsync, - expectWarnings(RestGetIndexTemplateAction.TYPES_DEPRECATION_MESSAGE))).status(), equalTo(RestStatus.NOT_FOUND)); - } - - public void testCRUDIndexTemplate() throws Exception { RestHighLevelClient client = highLevelClient(); - PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest("template-1") + PutIndexTemplateRequest putTemplate1 = new PutIndexTemplateRequest().name("template-1") .patterns(Arrays.asList("pattern-1", "name-1")).alias(new Alias("alias-1")); assertThat(execute(putTemplate1, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), equalTo(true)); - PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest("template-2") + PutIndexTemplateRequest putTemplate2 = new PutIndexTemplateRequest().name("template-2") .patterns(Arrays.asList("pattern-2", "name-2")) - .mapping("{\"properties\": { \"name\": { \"type\": \"text\" }}}", XContentType.JSON) .settings(Settings.builder().put("number_of_shards", "2").put("number_of_replicas", "0")); - assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync) - .isAcknowledged(), equalTo(true)); + assertThat(execute(putTemplate2, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged(), + equalTo(true)); - GetIndexTemplatesResponse getTemplate1 = execute( - new GetIndexTemplatesRequest("template-1"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); + GetIndexTemplatesResponse getTemplate1 = execute(new GetIndexTemplatesRequest("template-1"), + client.indices()::getTemplate, client.indices()::getTemplateAsync); assertThat(getTemplate1.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template1 = getTemplate1.getIndexTemplates().get(0); assertThat(template1.name(), equalTo("template-1")); assertThat(template1.patterns(), contains("pattern-1", "name-1")); assertTrue(template1.aliases().containsKey("alias-1")); - + GetIndexTemplatesResponse getTemplate2 = execute(new GetIndexTemplatesRequest("template-2"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); + client.indices()::getTemplate, client.indices()::getTemplateAsync); assertThat(getTemplate2.getIndexTemplates(), hasSize(1)); IndexTemplateMetaData template2 = getTemplate2.getIndexTemplates().get(0); assertThat(template2.name(), equalTo("template-2")); assertThat(template2.patterns(), contains("pattern-2", "name-2")); assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); - assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); - // New API returns a MappingMetaData class rather than CompressedXContent for the mapping - assertTrue(template2.mappings().sourceAsMap().containsKey("properties")); - @SuppressWarnings("unchecked") - Map props = (Map) template2.mappings().sourceAsMap().get("properties"); - assertTrue(props.containsKey("name")); - - + assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); List names = randomBoolean() ? Arrays.asList("*-1", "template-2") : Arrays.asList("template-*"); GetIndexTemplatesRequest getBothRequest = new GetIndexTemplatesRequest(names); - GetIndexTemplatesResponse getBoth = execute( - getBothRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); + GetIndexTemplatesResponse getBoth = execute(getBothRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); assertThat(getBoth.getIndexTemplates(), hasSize(2)); - assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::name).toArray(), + assertThat(getBoth.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).toArray(), arrayContainingInAnyOrder("template-1", "template-2")); GetIndexTemplatesRequest getAllRequest = new GetIndexTemplatesRequest(); - GetIndexTemplatesResponse getAll = execute( - getAllRequest, client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync); + GetIndexTemplatesResponse getAll = execute(getAllRequest, client.indices()::getTemplate, client.indices()::getTemplateAsync); assertThat(getAll.getIndexTemplates().size(), greaterThanOrEqualTo(2)); - assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::name) - .collect(Collectors.toList()), + assertThat(getAll.getIndexTemplates().stream().map(IndexTemplateMetaData::getName).collect(Collectors.toList()), hasItems("template-1", "template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-1"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new DeleteIndexTemplateRequest("template-1"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates(), hasSize(1)); + client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates(), hasSize(1)); assertThat(execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync).getIndexTemplates() - .get(0).name(), equalTo("template-2")); + client.indices()::getTemplate, client.indices()::getTemplateAsync).getIndexTemplates().get(0).name(), equalTo("template-2")); assertTrue(execute(new DeleteIndexTemplateRequest("template-*"), client.indices()::deleteTemplate, client.indices()::deleteTemplateAsync).isAcknowledged()); assertThat(expectThrows(ElasticsearchException.class, () -> execute(new GetIndexTemplatesRequest("template-*"), - client.indices()::getIndexTemplate, client.indices()::getIndexTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); + client.indices()::getTemplate, client.indices()::getTemplateAsync)).status(), equalTo(RestStatus.NOT_FOUND)); } public void testIndexTemplatesExist() throws Exception { @@ -1797,7 +1604,8 @@ public void testIndexTemplatesExist() throws Exception { { for (String suffix : Arrays.asList("1", "2")) { - final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("template-" + suffix) + final PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest() + .name("template-" + suffix) .patterns(Arrays.asList("pattern-" + suffix, "name-" + suffix)) .alias(new Alias("alias-" + suffix)); assertTrue(execute(putRequest, client.indices()::putTemplate, client.indices()::putTemplateAsync).isAcknowledged()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index ba471baaa42c8..5ddaf873b9d65 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedRequest; @@ -53,7 +54,6 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; import org.elasticsearch.common.CheckedFunction; @@ -61,7 +61,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.junit.Assert; @@ -927,16 +926,14 @@ public void testIndexPutSettings() throws IOException { Assert.assertEquals(expectedParams, request.getParameters()); } - public void testPutTemplateRequestWithTypes() throws Exception { + public void testPutTemplateRequest() throws Exception { Map names = new HashMap<>(); names.put("log", "log"); names.put("template#1", "template%231"); names.put("-#template", "-%23template"); names.put("foo^bar", "foo%5Ebar"); - org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest putTemplateRequest = - new org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest() - .name(ESTestCase.randomFrom(names.keySet())) + PutIndexTemplateRequest putTemplateRequest = new PutIndexTemplateRequest().name(ESTestCase.randomFrom(names.keySet())) .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); if (ESTestCase.randomBoolean()) { putTemplateRequest.order(ESTestCase.randomInt()); @@ -947,59 +944,14 @@ public void testPutTemplateRequestWithTypes() throws Exception { if (ESTestCase.randomBoolean()) { putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); } - Map expectedParams = new HashMap<>(); if (ESTestCase.randomBoolean()) { putTemplateRequest.mapping("doc-" + ESTestCase.randomInt(), "field-" + ESTestCase.randomInt(), "type=" + ESTestCase.randomFrom("text", "keyword")); } - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); } - if (ESTestCase.randomBoolean()) { - expectedParams.put("create", Boolean.TRUE.toString()); - putTemplateRequest.create(true); - } - if (ESTestCase.randomBoolean()) { - String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); - putTemplateRequest.cause(cause); - expectedParams.put("cause", cause); - } - RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); - Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); - Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); - Assert.assertThat(request.getParameters(), equalTo(expectedParams)); - RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); - } - - public void testPutTemplateRequest() throws Exception { - Map names = new HashMap<>(); - names.put("log", "log"); - names.put("template#1", "template%231"); - names.put("-#template", "-%23template"); - names.put("foo^bar", "foo%5Ebar"); - - PutIndexTemplateRequest putTemplateRequest = - new PutIndexTemplateRequest(ESTestCase.randomFrom(names.keySet())) - .patterns(Arrays.asList(ESTestCase.generateRandomStringArray(20, 100, false, false))); - if (ESTestCase.randomBoolean()) { - putTemplateRequest.order(ESTestCase.randomInt()); - } - if (ESTestCase.randomBoolean()) { - putTemplateRequest.version(ESTestCase.randomInt()); - } - if (ESTestCase.randomBoolean()) { - putTemplateRequest.settings(Settings.builder().put("setting-" + ESTestCase.randomInt(), ESTestCase.randomTimeValue())); - } Map expectedParams = new HashMap<>(); - if (ESTestCase.randomBoolean()) { - putTemplateRequest.mapping("{ \"properties\": { \"field-" + ESTestCase.randomInt() + - "\" : { \"type\" : \"" + ESTestCase.randomFrom("text", "keyword") + "\" }}}", XContentType.JSON); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "false"); - } - if (ESTestCase.randomBoolean()) { - putTemplateRequest.alias(new Alias("alias-" + ESTestCase.randomInt())); - } if (ESTestCase.randomBoolean()) { expectedParams.put("create", Boolean.TRUE.toString()); putTemplateRequest.create(true); @@ -1008,14 +960,14 @@ public void testPutTemplateRequest() throws Exception { String cause = ESTestCase.randomUnicodeOfCodepointLengthBetween(1, 50); putTemplateRequest.cause(cause); expectedParams.put("cause", cause); - } + } RequestConvertersTests.setRandomMasterTimeout(putTemplateRequest, expectedParams); - Request request = IndicesRequestConverters.putTemplate(putTemplateRequest); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.get(putTemplateRequest.name()))); Assert.assertThat(request.getParameters(), equalTo(expectedParams)); RequestConvertersTests.assertToXContentBody(putTemplateRequest, request.getEntity()); } + public void testValidateQuery() throws Exception { String[] indices = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); String[] types = ESTestCase.randomBoolean() ? ESTestCase.generateRandomStringArray(5, 5, false, false) : null; @@ -1063,8 +1015,7 @@ public void testGetTemplateRequest() throws Exception { Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomMasterTimeout(getTemplatesRequest::setMasterNodeTimeout, expectedParams); RequestConvertersTests.setRandomLocal(getTemplatesRequest::setLocal, expectedParams); - Request request = IndicesRequestConverters.getTemplatesWithDocumentTypes(getTemplatesRequest); - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + Request request = IndicesRequestConverters.getTemplates(getTemplatesRequest); Assert.assertThat(request.getEndpoint(), equalTo("/_template/" + names.stream().map(encodes::get).collect(Collectors.joining(",")))); Assert.assertThat(request.getParameters(), equalTo(expectedParams)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 09ace08723a83..8064ba2f2b565 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -762,14 +762,6 @@ public void testApiNamingConventions() throws Exception { .collect(Collectors.groupingBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet()))); - // TODO remove in 8.0 - we will undeprecate indices.get_template because the current getIndexTemplate - // impl will replace the existing getTemplate method. - // The above general-purpose code ignores all deprecated methods which in this case leaves `getTemplate` - // looking like it doesn't have a valid implementatation when it does. - apiUnsupported.remove("indices.get_template"); - - - for (Map.Entry> entry : methods.entrySet()) { String apiName = entry.getKey(); @@ -802,10 +794,7 @@ public void testApiNamingConventions() throws Exception { apiName.startsWith("security.") == false && apiName.startsWith("index_lifecycle.") == false && apiName.startsWith("ccr.") == false && - apiName.endsWith("freeze") == false && - // IndicesClientIT.getIndexTemplate should be renamed "getTemplate" in version 8.0 when we - // can get rid of 7.0's deprecated "getTemplate" - apiName.equals("indices.get_index_template") == false) { + apiName.endsWith("freeze") == false) { apiNotFound.add(apiName); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index fc4f69e281da9..c20389c24b10a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -55,6 +55,8 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse; @@ -74,13 +76,12 @@ import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; -import org.elasticsearch.client.indices.GetIndexTemplatesResponse; -import org.elasticsearch.client.indices.IndexTemplateMetaData; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; -import org.elasticsearch.client.indices.PutIndexTemplateRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -2084,14 +2085,16 @@ public void testPutTemplate() throws Exception { { // tag::put-template-request-mappings-json - request.mapping(// <1> + request.mapping("_doc", // <1> "{\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + + " \"_doc\": {\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + " }\n" + " }\n" + - "}", + "}", // <2> XContentType.JSON); // end::put-template-request-mappings-json assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); @@ -2099,16 +2102,14 @@ public void testPutTemplate() throws Exception { { //tag::put-template-request-mappings-map Map jsonMap = new HashMap<>(); - { - Map properties = new HashMap<>(); - { - Map message = new HashMap<>(); - message.put("type", "text"); - properties.put("message", message); - } - jsonMap.put("properties", properties); - } - request.mapping(jsonMap); // <1> + Map message = new HashMap<>(); + message.put("type", "text"); + Map properties = new HashMap<>(); + properties.put("message", message); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap.put("_doc", mapping); + request.mapping("_doc", jsonMap); // <1> //end::put-template-request-mappings-map assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } @@ -2117,21 +2118,31 @@ public void testPutTemplate() throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); { - builder.startObject("properties"); + builder.startObject("_doc"); { - builder.startObject("message"); + builder.startObject("properties"); { - builder.field("type", "text"); + builder.startObject("message"); + { + builder.field("type", "text"); + } + builder.endObject(); } builder.endObject(); } builder.endObject(); } builder.endObject(); - request.mapping(builder); // <1> + request.mapping("_doc", builder); // <1> //end::put-template-request-mappings-xcontent assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); } + { + //tag::put-template-request-mappings-shortcut + request.mapping("_doc", "message", "type=text"); // <1> + //end::put-template-request-mappings-shortcut + assertTrue(client.indices().putTemplate(request, RequestOptions.DEFAULT).isAcknowledged()); + } // tag::put-template-request-aliases request.alias(new Alias("twitter_alias").filter(QueryBuilders.termQuery("user", "kimchy"))); // <1> @@ -2157,9 +2168,11 @@ public void testPutTemplate() throws Exception { " \"number_of_shards\": 1\n" + " },\n" + " \"mappings\": {\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + + " \"_doc\": {\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + " }\n" + " }\n" + " },\n" + @@ -2222,11 +2235,13 @@ public void testGetTemplates() throws Exception { PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest("my-template"); putRequest.patterns(Arrays.asList("pattern-1", "log-*")); putRequest.settings(Settings.builder().put("index.number_of_shards", 3).put("index.number_of_replicas", 1)); - putRequest.mapping( + putRequest.mapping("_doc", "{\n" + - " \"properties\": {\n" + - " \"message\": {\n" + - " \"type\": \"text\"\n" + + " \"_doc\": {\n" + + " \"properties\": {\n" + + " \"message\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + " }\n" + " }\n" + "}", XContentType.JSON); @@ -2245,7 +2260,7 @@ public void testGetTemplates() throws Exception { // end::get-templates-request-masterTimeout // tag::get-templates-execute - GetIndexTemplatesResponse getTemplatesResponse = client.indices().getIndexTemplate(request, RequestOptions.DEFAULT); + GetIndexTemplatesResponse getTemplatesResponse = client.indices().getTemplate(request, RequestOptions.DEFAULT); // end::get-templates-execute // tag::get-templates-response @@ -2275,7 +2290,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::get-templates-execute-async - client.indices().getIndexTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.indices().getTemplateAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::get-templates-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java deleted file mode 100644 index d6bca30085165..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.indices; - -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.ToXContent.MapParams; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; - -public class GetIndexTemplatesResponseTests extends ESTestCase { - - static final String mappingString = "{\"properties\":{" - + "\"f1\": {\"type\":\"text\"}," - + "\"f2\": {\"type\":\"keyword\"}" - + "}}"; - - - public void testFromXContent() throws IOException { - xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent, - GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false) - .assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances) - .shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"}) - .test(); - } - - private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) { - assertEquals(expectedInstance, newInstance); - // Check there's no doc types at the root of the mapping - Map expectedMap = XContentHelper.convertToMap( - new BytesArray(mappingString), true, XContentType.JSON).v2(); - for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) { - MappingMetaData mappingMD = template.mappings(); - if(mappingMD!=null) { - Map mappingAsMap = mappingMD.sourceAsMap(); - assertEquals(expectedMap, mappingAsMap); - } - } - } - - static GetIndexTemplatesResponse createTestInstance() { - List templates = new ArrayList<>(); - int numTemplates = between(0, 10); - for (int t = 0; t < numTemplates; t++) { - IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder("template-" + t); - templateBuilder.patterns(IntStream.range(0, between(1, 5)).mapToObj(i -> "pattern-" + i).collect(Collectors.toList())); - int numAlias = between(0, 5); - for (int i = 0; i < numAlias; i++) { - templateBuilder.putAlias(AliasMetaData.builder(randomAlphaOfLengthBetween(1, 10))); - } - if (randomBoolean()) { - templateBuilder.settings(Settings.builder().put("index.setting-1", randomLong())); - } - if (randomBoolean()) { - templateBuilder.order(randomInt()); - } - if (randomBoolean()) { - templateBuilder.version(between(0, 100)); - } - if (randomBoolean()) { - try { - Map map = XContentHelper.convertToMap(new BytesArray(mappingString), true, XContentType.JSON).v2(); - MappingMetaData mapping = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, map); - templateBuilder.mapping(mapping); - } catch (IOException ex) { - throw new UncheckedIOException(ex); - } - } - templates.add(templateBuilder.build()); - } - return new GetIndexTemplatesResponse(templates); - } - - // As the client class GetIndexTemplatesResponse doesn't have toXContent method, adding this method here only for the test - static void toXContent(GetIndexTemplatesResponse response, XContentBuilder builder) throws IOException { - - //Create a server-side counterpart for the client-side class and call toXContent on it - - List serverIndexTemplates = new ArrayList<>(); - List clientIndexTemplates = response.getIndexTemplates(); - for (IndexTemplateMetaData clientITMD : clientIndexTemplates) { - org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder serverTemplateBuilder = - org.elasticsearch.cluster.metadata.IndexTemplateMetaData.builder(clientITMD.name()); - - serverTemplateBuilder.patterns(clientITMD.patterns()); - - Iterator aliases = clientITMD.aliases().valuesIt(); - aliases.forEachRemaining((a)->serverTemplateBuilder.putAlias(a)); - - serverTemplateBuilder.settings(clientITMD.settings()); - serverTemplateBuilder.order(clientITMD.order()); - serverTemplateBuilder.version(clientITMD.version()); - if (clientITMD.mappings() != null) { - serverTemplateBuilder.putMapping(MapperService.SINGLE_MAPPING_NAME, clientITMD.mappings().source()); - } - serverIndexTemplates.add(serverTemplateBuilder.build()); - - } - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse serverResponse = new - org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates); - MapParams params = - new MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); - serverResponse.toXContent(builder, params); - } -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java deleted file mode 100644 index 8aab973982fc0..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/PutIndexTemplateRequestTests.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.client.indices; - -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.AbstractXContentTestCase; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Arrays; -import java.util.Collections; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.nullValue; -import static org.hamcrest.core.Is.is; - -public class PutIndexTemplateRequestTests extends AbstractXContentTestCase { - public void testValidateErrorMessage() throws Exception { - expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest(null)); - expectThrows(IllegalArgumentException.class, () -> new PutIndexTemplateRequest("test").name(null)); - PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); - ActionRequestValidationException withoutPattern = request.validate(); - assertThat(withoutPattern.getMessage(), containsString("index patterns are missing")); - - request.name("foo"); - ActionRequestValidationException withoutIndexPatterns = request.validate(); - assertThat(withoutIndexPatterns.validationErrors(), hasSize(1)); - assertThat(withoutIndexPatterns.getMessage(), containsString("index patterns are missing")); - - request.patterns(Collections.singletonList("test-*")); - ActionRequestValidationException noError = request.validate(); - assertThat(noError, is(nullValue())); - } - - @Override - protected PutIndexTemplateRequest createTestInstance() { - PutIndexTemplateRequest request = new PutIndexTemplateRequest("test"); - if (randomBoolean()) { - request.version(randomInt()); - } - if (randomBoolean()) { - request.order(randomInt()); - } - request.patterns(Arrays.asList(generateRandomStringArray(20, 100, false, false))); - int numAlias = between(0, 5); - for (int i = 0; i < numAlias; i++) { - // some ASCII or Latin-1 control characters, especially newline, can lead to - // problems with yaml parsers, that's why we filter them here (see #30911) - Alias alias = new Alias(randomRealisticUnicodeOfLengthBetween(1, 10).replaceAll("\\p{Cc}", "")); - if (randomBoolean()) { - alias.indexRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); - } - if (randomBoolean()) { - alias.searchRouting(randomRealisticUnicodeOfLengthBetween(1, 10)); - } - request.alias(alias); - } - if (randomBoolean()) { - try { - request.mapping(XContentFactory.jsonBuilder().startObject() - .startObject("properties") - .startObject("field-" + randomInt()).field("type", randomFrom("keyword", "text")).endObject() - .endObject().endObject()); - } catch (IOException ex) { - throw new UncheckedIOException(ex); - } - } - if (randomBoolean()) { - request.settings(Settings.builder().put("setting1", randomLong()).put("setting2", randomTimeValue()).build()); - } - return request; - } - - @Override - protected PutIndexTemplateRequest doParseInstance(XContentParser parser) throws IOException { - return new PutIndexTemplateRequest("test").source(parser.map()); - } - - @Override - protected void assertEqualInstances(PutIndexTemplateRequest expected, PutIndexTemplateRequest actual) { - assertNotSame(expected, actual); - assertThat(actual.version(), equalTo(expected.version())); - assertThat(actual.order(), equalTo(expected.order())); - assertThat(actual.patterns(), equalTo(expected.patterns())); - assertThat(actual.aliases(), equalTo(expected.aliases())); - assertThat(actual.mappings(), equalTo(expected.mappings())); - assertThat(actual.settings(), equalTo(expected.settings())); - } - - @Override - protected boolean supportsUnknownFields() { - return false; - } -} diff --git a/docs/java-rest/high-level/indices/put_template.asciidoc b/docs/java-rest/high-level/indices/put_template.asciidoc index 3e3954308736d..7618fc8ce7af5 100644 --- a/docs/java-rest/high-level/indices/put_template.asciidoc +++ b/docs/java-rest/high-level/indices/put_template.asciidoc @@ -39,7 +39,8 @@ template's patterns. -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-mappings-json] -------------------------------------------------- -<1> The mapping, provided as a JSON string +<1> The type to define +<2> The mapping for this type, provided as a JSON string The mapping source can be provided in different ways in addition to the `String` example shown above: @@ -58,6 +59,13 @@ include-tagged::{doc-tests-file}[{api}-request-mappings-xcontent] <1> Mapping source provided as an `XContentBuilder` object, the Elasticsearch built-in helpers to generate JSON content +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-mappings-shortcut] +-------------------------------------------------- +<1> Mapping source provided as `Object` key-pairs, which gets converted to +JSON format + ==== Aliases The aliases of the template will define aliasing to the index whose name matches the template's patterns. A placeholder `{index}` can be used in an alias of a template. diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java index 9ddea863d48f4..2bdc966c74e59 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java @@ -41,7 +41,7 @@ public class GetIndexTemplatesResponse extends ActionResponse implements ToXCont indexTemplates = new ArrayList<>(); } - public GetIndexTemplatesResponse(List indexTemplates) { + GetIndexTemplatesResponse(List indexTemplates) { this.indexTemplates = indexTemplates; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java index ddaa31b6bbad7..50370797aa6f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndexTemplateAction.java @@ -19,12 +19,10 @@ package org.elasticsearch.rest.action.admin.indices; -import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; @@ -49,11 +47,7 @@ public class RestGetIndexTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMETERS = Collections.unmodifiableSet(Sets.union( Collections.singleton(INCLUDE_TYPE_NAME_PARAMETER), Settings.FORMAT_PARAMS)); - private static final DeprecationLogger deprecationLogger = new DeprecationLogger( - LogManager.getLogger(RestGetIndexTemplateAction.class)); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of get index template requests will change" - + " in the next major version. Please start using the `include_type_name` parameter set to `false` in the request to " - + "move to the new, typeless response format that will be the default in 7.0."; + public RestGetIndexTemplateAction(final Settings settings, final RestController controller) { super(settings); controller.registerHandler(GET, "/_template", this); @@ -71,9 +65,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC final String[] names = Strings.splitStringByCommaToArray(request.param("name")); final GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names); - if (request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY)) { - deprecationLogger.deprecatedAndMaybeLog("get_index_template_with_types", TYPES_DEPRECATION_MESSAGE); - } getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local())); getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java index b1dd13324c85c..8a292859d6031 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java @@ -42,9 +42,6 @@ public class RestPutIndexTemplateAction extends BaseRestHandler { private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger( LogManager.getLogger(RestPutIndexTemplateAction.class)); - public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in put index template " + - "requests is deprecated. To be compatible with 7.0, the mapping definition should not be nested under " + - "the type name, and the parameter include_type_name must be provided and set to false."; public RestPutIndexTemplateAction(Settings settings, RestController controller) { super(settings); @@ -71,7 +68,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC putRequest.create(request.paramAsBoolean("create", false)); putRequest.cause(request.param("cause", "")); - boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, true); Map sourceAsMap = prepareRequestSource(request, includeTypeName); putRequest.source(sourceAsMap); @@ -86,9 +83,6 @@ Map prepareRequestSource(RestRequest request, boolean includeTyp newSourceAsMap.put("mappings", Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, sourceAsMap.get("mappings"))); return newSourceAsMap; } else { - if(includeTypeName && sourceAsMap.containsKey("mappings") ) { - DEPRECATION_LOGGER.deprecatedAndMaybeLog("put_index_template_with_types", TYPES_DEPRECATION_MESSAGE); - } return sourceAsMap; } } diff --git a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java index 5da864207b621..ac0eb8f0d81a6 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.rest.action.admin.indices; -import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -32,12 +31,8 @@ import org.junit.Before; import java.io.IOException; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; -import static org.mockito.Mockito.mock; - public class RestPutIndexTemplateActionTests extends RestActionTestCase { private RestPutIndexTemplateAction action; @@ -50,8 +45,7 @@ public void testPrepareTypelessRequest() throws IOException { XContentBuilder content = XContentFactory.jsonBuilder().startObject() .startObject("mappings") .startObject("properties") - .startObject("field1").field("type", "keyword").endObject() - .startObject("field2").field("type", "text").endObject() + .startObject("field").field("type", "keyword").endObject() .endObject() .endObject() .startObject("aliases") @@ -59,19 +53,11 @@ public void testPrepareTypelessRequest() throws IOException { .endObject() .endObject(); - Map params = new HashMap<>(); - params.put(INCLUDE_TYPE_NAME_PARAMETER, "false"); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) .withMethod(RestRequest.Method.PUT) - .withParams(params) .withPath("/_template/_some_template") .withContent(BytesReference.bytes(content), XContentType.JSON) .build(); - action.prepareRequest(request, mock(NodeClient.class)); - - // Internally the above prepareRequest method calls prepareRequestSource to inject a - // default type into the mapping. Here we test that this does what is expected by - // explicitly calling that same helper function boolean includeTypeName = false; Map source = action.prepareRequestSource(request, includeTypeName); @@ -79,8 +65,7 @@ public void testPrepareTypelessRequest() throws IOException { .startObject("mappings") .startObject("_doc") .startObject("properties") - .startObject("field1").field("type", "keyword").endObject() - .startObject("field2").field("type", "text").endObject() + .startObject("field").field("type", "keyword").endObject() .endObject() .endObject() .endObject() @@ -93,49 +78,4 @@ public void testPrepareTypelessRequest() throws IOException { assertEquals(expectedContentAsMap, source); } - - public void testIncludeTypeName() throws IOException { - XContentBuilder typedContent = XContentFactory.jsonBuilder().startObject() - .startObject("mappings") - .startObject("my_doc") - .startObject("properties") - .startObject("field1").field("type", "keyword").endObject() - .startObject("field2").field("type", "text").endObject() - .endObject() - .endObject() - .endObject() - .startObject("aliases") - .startObject("read_alias").endObject() - .endObject() - .endObject(); - - RestRequest request = new FakeRestRequest.Builder(xContentRegistry()) - .withMethod(RestRequest.Method.PUT) - .withPath("/_template/_some_template") - .withContent(BytesReference.bytes(typedContent), XContentType.JSON) - .build(); - action.prepareRequest(request, mock(NodeClient.class)); - assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); - boolean includeTypeName = true; - Map source = action.prepareRequestSource(request, includeTypeName); - assertWarnings(RestPutIndexTemplateAction.TYPES_DEPRECATION_MESSAGE); - - XContentBuilder expectedContent = XContentFactory.jsonBuilder().startObject() - .startObject("mappings") - .startObject("my_doc") - .startObject("properties") - .startObject("field1").field("type", "keyword").endObject() - .startObject("field2").field("type", "text").endObject() - .endObject() - .endObject() - .endObject() - .startObject("aliases") - .startObject("read_alias").endObject() - .endObject() - .endObject(); - Map expectedContentAsMap = XContentHelper.convertToMap( - BytesReference.bytes(expectedContent), true, expectedContent.contentType()).v2(); - - assertEquals(expectedContentAsMap, source); - } } From bfa1a8cd968645d6c872082eb3f10c42d4146df8 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 5 Feb 2019 09:36:54 -0600 Subject: [PATCH 06/25] Fix ILM explain response to allow unknown fields (#38363) IndexLifecycleExplainResponse did not allow unknown fields. This commit fixes the test and ConstructingObjectParser such that it allows unknown fields. Relates #36938 Backport of #38054 --- .../IndexLifecycleExplainResponse.java | 2 +- .../IndexLifecycleExplainResponseTests.java | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java index 8bdc3b195acd0..772dfbc0c5c13 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java @@ -54,7 +54,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject { private static final ParseField PHASE_EXECUTION_INFO = new ParseField("phase_execution"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "index_lifecycle_explain_response", + "index_lifecycle_explain_response", true, a -> new IndexLifecycleExplainResponse( (String) a[0], (boolean) a[1], diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java index 29f7a8db89f57..89e580dfd33dd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponseTests.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; @@ -99,7 +100,16 @@ protected IndexLifecycleExplainResponse doParseInstance(XContentParser parser) t @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return (field) -> + // actions are plucked from the named registry, and it fails if the action is not in the named registry + field.endsWith("phase_definition.actions") + // This is a bytes reference, so any new fields are tested for equality in this bytes reference. + || field.contains("step_info"); } private static class RandomStepInfo implements ToXContentObject { From 80ae5da1f3b5519f737e0b7db8393de720254e4f Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 5 Feb 2019 17:18:59 +0100 Subject: [PATCH 07/25] Backport changes to the release notes script. (#38346) Backport of #37967 and #38307. --- dev-tools/es_release_notes.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/es_release_notes.pl b/dev-tools/es_release_notes.pl index cf19a1cd9ddf5..16a00d4eff2ae 100755 --- a/dev-tools/es_release_notes.pl +++ b/dev-tools/es_release_notes.pl @@ -32,7 +32,7 @@ ">enhancement", ">bug", ">regression", ">upgrade" ); my %Ignore = map { $_ => 1 } - ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" ); + ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ">test-mute", ":Core/Infra/Build", "backport" ); my %Group_Labels = ( '>breaking' => 'Breaking changes', @@ -48,7 +48,7 @@ my %Area_Overrides = ( ':ml' => 'Machine Learning', - ':beats' => 'Beats Plugin', + ':Beats' => 'Beats Plugin', ':Docs' => 'Docs Infrastructure' ); From 62933dbbfb2521bcbf0eacaaa0896a0dc5ec41b0 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 5 Feb 2019 18:15:26 +0200 Subject: [PATCH 08/25] SQL: Implement CURRENT_DATE (#38175) Since DATE data type is now available, this implements the `CURRENT_DATE/CURRENT_DATE()/TODAY()` similar to `CURRENT_TIMESTAMP`. Closes: #38160 --- .../sql/functions/date-time.asciidoc | 79 +- .../xpack/sql/qa/cli/ShowTestCase.java | 1 + .../qa/src/main/resources/command.csv-spec | 6 +- .../sql/qa/src/main/resources/date.csv-spec | 87 +++ .../qa/src/main/resources/datetime.sql-spec | 7 + .../sql/qa/src/main/resources/docs.csv-spec | 74 +- x-pack/plugin/sql/src/main/antlr/SqlBase.g4 | 7 +- .../plugin/sql/src/main/antlr/SqlBase.tokens | 4 +- .../sql/src/main/antlr/SqlBaseLexer.tokens | 4 +- .../expression/function/FunctionRegistry.java | 4 +- .../scalar/ConfigurationFunction.java | 2 +- .../function/scalar/datetime/CurrentDate.java | 25 + .../scalar/datetime/CurrentDateTime.java | 52 +- .../scalar/datetime/CurrentFunction.java | 49 ++ .../xpack/sql/parser/ExpressionBuilder.java | 10 +- .../xpack/sql/parser/SqlBaseLexer.java | 723 +++++++++--------- .../xpack/sql/parser/SqlBaseParser.java | 656 ++++++++-------- .../scalar/datetime/CurrentDateTests.java | 46 ++ .../scalar/datetime/CurrentDateTimeTests.java | 64 +- .../xpack/sql/parser/ExpressionTests.java | 18 +- .../xpack/sql/tree/NodeSubclassTests.java | 7 +- 21 files changed, 1161 insertions(+), 764 deletions(-) create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDate.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentFunction.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTests.java diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc index 15fdba39026ef..2d2678a61704d 100644 --- a/docs/reference/sql/functions/date-time.asciidoc +++ b/docs/reference/sql/functions/date-time.asciidoc @@ -93,6 +93,48 @@ include-tagged::{sql-specs}/docs.csv-spec[dtIntervalMul] beta[] +[[sql-functions-current-date]] +==== `CURRENT_DATE/CURDATE` + +.Synopsis: +[source, sql] +-------------------------------------------------- +CURRENT_DATE +CURRENT_DATE() +-------------------------------------------------- + +*Input*: _none_ + +*Output*: date + +.Description: + +Returns the date (no time part) when the current query reached the server. +It can be used both as a keyword: `CURRENT_DATE` or as a function with no arguments: `CURRENT_DATE()`. + +[NOTE] +Unlike CURRENT_DATE, `CURDATE()` can only be used as a function with no arguments and not as a keyword. + +This method always returns the same value for its every occurrence within the same query. + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[curDate] +-------------------------------------------------- + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[curDateFunction] +-------------------------------------------------- + +Typically, this function (as well as its twin <> function +is used for relative date filtering: + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[filterToday] +-------------------------------------------------- + [[sql-functions-current-timestamp]] ==== `CURRENT_TIMESTAMP` @@ -115,7 +157,7 @@ Returns the date/time when the current query reached the server. As a function, `CURRENT_TIMESTAMP()` accepts _precision_ as an optional parameter for rounding the second fractional digits (nanoseconds). -This method always returns the same value within a query. +This method always returns the same value for its every occurrence within the same query. ["source","sql",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -422,7 +464,8 @@ NOW() .Description: This function offers the same functionality as <> function: returns -the datetime when the current query reached the server. This method always returns the same value within a query. +the datetime when the current query reached the server. This method always returns the same value for its every +occurrence within the same query. ["source","sql",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -485,6 +528,38 @@ Extract the year quarter the date/datetime falls in. include-tagged::{sql-specs}/docs.csv-spec[quarter] -------------------------------------------------- +[[sql-functions-today]] +==== `TODAY` + +.Synopsis: +[source, sql] +-------------------------------------------------- +TODAY() +-------------------------------------------------- + +*Input*: _none_ + +*Output*: date + +.Description: + +This function offers the same functionality as <> function: returns +the date when the current query reached the server. This method always returns the same value for its every occurrence +within the same query. + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[todayFunction] +-------------------------------------------------- + +Typically, this function (as well as its twin <> function is used +for relative date filtering: + +["source","sql",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{sql-specs}/docs.csv-spec[filterToday] +-------------------------------------------------- + [[sql-functions-datetime-week]] ==== `WEEK_OF_YEAR/WEEK` diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java index 382442b5bffec..8d5e399017cf6 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java @@ -91,6 +91,7 @@ public void testShowFunctionsLikeInfix() throws IOException { assertThat(readLine(), RegexMatcher.matches("\\s*ISODAYOFWEEK\\s*\\|\\s*SCALAR\\s*")); assertThat(readLine(), RegexMatcher.matches("\\s*ISO_DAY_OF_WEEK\\s*\\|\\s*SCALAR\\s*")); assertThat(readLine(), RegexMatcher.matches("\\s*MINUTE_OF_DAY\\s*\\|\\s*SCALAR\\s*")); + assertThat(readLine(), RegexMatcher.matches("\\s*TODAY\\s*\\|\\s*SCALAR\\s*")); assertEquals("", readLine()); } } diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec index c7ebf9420c9a0..15f9f58495deb 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec @@ -31,6 +31,8 @@ ISNULL |CONDITIONAL LEAST |CONDITIONAL NULLIF |CONDITIONAL NVL |CONDITIONAL +CURDATE |SCALAR +CURRENT_DATE |SCALAR CURRENT_TIMESTAMP|SCALAR DAY |SCALAR DAYNAME |SCALAR @@ -65,7 +67,8 @@ MONTH_OF_YEAR |SCALAR NOW |SCALAR QUARTER |SCALAR SECOND |SCALAR -SECOND_OF_MINUTE |SCALAR +SECOND_OF_MINUTE |SCALAR +TODAY |SCALAR WEEK |SCALAR WEEK_OF_YEAR |SCALAR YEAR |SCALAR @@ -175,6 +178,7 @@ HOUR_OF_DAY |SCALAR ISODAYOFWEEK |SCALAR ISO_DAY_OF_WEEK|SCALAR MINUTE_OF_DAY |SCALAR +TODAY |SCALAR ; showTables diff --git a/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec index f744ea9ca6c70..35db16541babf 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec @@ -2,6 +2,73 @@ // Date // +currentDateKeywordWithDivision +SELECT YEAR(CURRENT_TIMESTAMP) / 1000 AS result; + + result +--------------- +2 +; + +currentDateFunctionNoArgsWithDivision +SELECT YEAR(CURRENT_TIMESTAMP()) / 1000 AS result; + + result +--------------- +2 +; + +todayWithDivision +SELECT YEAR(TODAY()) / 1000 AS result; + + result +--------------- +2 +; + +todayIntervalSubstraction +SELECT TRUNCATE(YEAR(TODAY() - INTERVAL 50 YEARS) / 1000) AS result; + + result +--------------- +1 +; + + +currentDateFilter +SELECT first_name FROM test_emp WHERE hire_date > CURRENT_DATE() - INTERVAL 25 YEARS ORDER BY first_name ASC LIMIT 10; + + first_name +----------------- +Kazuhito +Kenroku +Lillian +Mayumi +Mingsen +Sailaja +Saniya +Shahaf +Suzette +Tuval +; + +currentDateFilterScript +SELECT first_name, TRUNCATE(YEAR(hire_date) - YEAR(TODAY()) / 1000) AS filter FROM test_emp +WHERE TRUNCATE(YEAR(hire_date) - YEAR(TODAY()) / 1000) > 1990 ORDER BY first_name ASC LIMIT 10; + + first_name | filter +Cristinel |1991 +Kazuhito |1993 +Kenroku |1992 +Lillian |1997 +Magy |1991 +Mayumi |1993 +Mingsen |1992 +Sailaja |1994 +Saniya |1992 +Shahaf |1993 +; + dateExtractDateParts SELECT DAY(CAST(birth_date AS DATE)) d, @@ -75,3 +142,23 @@ SELECT YEAR(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS y, MON y:i | m:i 2020 | 3 ; + +orderByCurrentDate +SELECT first_name FROM test_emp ORDER BY TODAY(), first_name LIMIT 5; + + first_name +--------------- +Alejandro +Amabile +Anneke +Anoosh +Arumugam +; + +groupByCurrentDate +SELECT MAX(salary) FROM test_emp GROUP BY TODAY(); + + MAX(salary) +--------------- +74999 +; diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec index 1bdc090ea232f..1c21e9a7e6f9a 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec @@ -120,6 +120,12 @@ SELECT DAY_OF_WEEK(birth_date) day, COUNT(*) c FROM test_emp WHERE DAY_OF_WEEK(b currentTimestampYear SELECT YEAR(CURRENT_TIMESTAMP()) AS result; +orderByCurrentTimestamp +SELECT first_name FROM test_emp ORDER BY NOW(), first_name NULLS LAST LIMIT 5; + +groupByCurrentTimestamp +SELECT MAX(salary) AS max FROM test_emp GROUP BY NOW(); + // // H2 uses the local timezone instead of the specified one // @@ -131,3 +137,4 @@ SELECT HOUR(CURRENT_TIMESTAMP()) AS result; currentTimestampMinute-Ignore SELECT MINUTE(CURRENT_TIMESTAMP()) AS result; + diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec index 7b3ca6c398fa1..532f564a1ca2d 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec @@ -208,6 +208,8 @@ ISNULL |CONDITIONAL LEAST |CONDITIONAL NULLIF |CONDITIONAL NVL |CONDITIONAL +CURDATE |SCALAR +CURRENT_DATE |SCALAR CURRENT_TIMESTAMP|SCALAR DAY |SCALAR DAYNAME |SCALAR @@ -242,7 +244,8 @@ MONTH_OF_YEAR |SCALAR NOW |SCALAR QUARTER |SCALAR SECOND |SCALAR -SECOND_OF_MINUTE |SCALAR +SECOND_OF_MINUTE |SCALAR +TODAY |SCALAR WEEK |SCALAR WEEK_OF_YEAR |SCALAR YEAR |SCALAR @@ -365,6 +368,7 @@ HOUR_OF_DAY |SCALAR ISODAYOFWEEK |SCALAR ISO_DAY_OF_WEEK|SCALAR MINUTE_OF_DAY |SCALAR +TODAY |SCALAR // end::showFunctionsWithPattern ; @@ -2227,18 +2231,50 @@ SELECT WEEK(CAST('1988-01-05T09:22:10Z' AS TIMESTAMP)) AS week, ISOWEEK(CAST('19 ; -currentNow -// tag::filterNow -SELECT first_name FROM emp WHERE hire_date > NOW() - INTERVAL 100 YEARS ORDER BY first_name ASC LIMIT 5; - first_name ---------------- -Alejandro -Amabile -Anneke -Anoosh -Arumugam -// end::filterNow + +currentDate-Ignore +// tag::curDate +SELECT CURRENT_TIMESTAMP AS result; + + result +------------------------ +2018-12-12 +// end::curDate +; + +currentDateFunction-Ignore +// tag::curDateFunction +SELECT CURRENT_TIMESTAMP() AS result; + + result +------------------------ +2018-12-12 +// end::curDateFunction +; + +todayFunction-Ignore +// tag::todayFunction +SELECT TODAY() AS result; + + result +------------------------ +2018-12-12 +// end::todayFunction +; + +filterToday +// tag::filterToday +SELECT first_name FROM emp WHERE hire_date > TODAY() - INTERVAL 25 YEARS ORDER BY first_name ASC LIMIT 5; + + first_name +------------ +Kazuhito +Kenroku +Lillian +Mayumi +Mingsen +// end::filterToday ; currentTimestamp-Ignore @@ -2282,6 +2318,20 @@ SELECT NOW() AS result; // end::nowFunction ; +filterNow +// tag::filterNow +SELECT first_name FROM emp WHERE hire_date > NOW() - INTERVAL 100 YEARS ORDER BY first_name ASC LIMIT 5; + + first_name +--------------- +Alejandro +Amabile +Anneke +Anoosh +Arumugam +// end::filterNow +; + //////////// // Next two queries need to have the same output, as they should be equivalent. // They are used in the "SQL Limitations" page. diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 index e7cb2e2b1b258..a11121feaa2bc 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 +++ b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 @@ -236,7 +236,8 @@ castTemplate ; builtinDateTimeFunction - : name=CURRENT_TIMESTAMP ('(' precision=INTEGER_VALUE? ')')? + : name=CURRENT_DATE ('(' ')')? + | name=CURRENT_TIMESTAMP ('(' precision=INTEGER_VALUE? ')')? ; convertTemplate @@ -337,7 +338,7 @@ string // http://developer.mimer.se/validator/sql-reserved-words.tml nonReserved : ANALYZE | ANALYZED - | CATALOGS | COLUMNS | CURRENT + | CATALOGS | COLUMNS | DAY | DEBUG | EXECUTABLE | EXPLAIN | FIRST | FORMAT | FULL | FUNCTIONS @@ -370,7 +371,7 @@ CATALOG: 'CATALOG'; CATALOGS: 'CATALOGS'; COLUMNS: 'COLUMNS'; CONVERT: 'CONVERT'; -CURRENT: 'CURRENT'; +CURRENT_DATE : 'CURRENT_DATE'; CURRENT_TIMESTAMP : 'CURRENT_TIMESTAMP'; DAY: 'DAY'; DAYS: 'DAYS'; diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens b/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens index f2d522b2bc757..4c3cc5de06421 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens +++ b/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens @@ -16,7 +16,7 @@ CATALOG=15 CATALOGS=16 COLUMNS=17 CONVERT=18 -CURRENT=19 +CURRENT_DATE=19 CURRENT_TIMESTAMP=20 DAY=21 DAYS=22 @@ -143,7 +143,7 @@ DELIMITER=127 'CATALOGS'=16 'COLUMNS'=17 'CONVERT'=18 -'CURRENT'=19 +'CURRENT_DATE'=19 'CURRENT_TIMESTAMP'=20 'DAY'=21 'DAYS'=22 diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens b/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens index 8b586035f8520..6d4252c0e723e 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens +++ b/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens @@ -16,7 +16,7 @@ CATALOG=15 CATALOGS=16 COLUMNS=17 CONVERT=18 -CURRENT=19 +CURRENT_DATE=19 CURRENT_TIMESTAMP=20 DAY=21 DAYS=22 @@ -142,7 +142,7 @@ UNRECOGNIZED=126 'CATALOGS'=16 'COLUMNS'=17 'CONVERT'=18 -'CURRENT'=19 +'CURRENT_DATE'=19 'CURRENT_TIMESTAMP'=20 'DAY'=21 'DAYS'=22 diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java index 876af256294c8..ea9f8cba24b7e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.function.scalar.Database; import org.elasticsearch.xpack.sql.expression.function.scalar.User; +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.CurrentDate; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.CurrentDateTime; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayName; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DayOfMonth; @@ -169,7 +170,8 @@ private void defineDefaultFunctions() { def(Greatest.class, Greatest::new, "GREATEST"), def(Least.class, Least::new, "LEAST")); // Date - addToMap(def(CurrentDateTime.class, CurrentDateTime::new, "CURRENT_TIMESTAMP", "NOW"), + addToMap(def(CurrentDate.class, CurrentDate::new, "CURRENT_DATE", "CURDATE", "TODAY"), + def(CurrentDateTime.class, CurrentDateTime::new, "CURRENT_TIMESTAMP", "NOW"), def(DayName.class, DayName::new, "DAY_NAME", "DAYNAME"), def(DayOfMonth.class, DayOfMonth::new, "DAY_OF_MONTH", "DAYOFMONTH", "DAY", "DOM"), def(DayOfWeek.class, DayOfWeek::new, "DAY_OF_WEEK", "DAYOFWEEK", "DOW"), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ConfigurationFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ConfigurationFunction.java index a24fba1b13569..dc49d2a950a63 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ConfigurationFunction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ConfigurationFunction.java @@ -32,7 +32,7 @@ public Expression replaceChildren(List newChildren) { throw new UnsupportedOperationException("this node doesn't have any children"); } - protected Configuration configuration() { + public Configuration configuration() { return configuration; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDate.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDate.java new file mode 100644 index 0000000000000..03b5567e9266c --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDate.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; + +import org.elasticsearch.xpack.sql.session.Configuration; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.util.DateUtils; + +public class CurrentDate extends CurrentFunction { + + public CurrentDate(Source source, Configuration configuration) { + super(source, configuration, DateUtils.asDateOnly(configuration.now()), DataType.DATE); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CurrentDate::new, configuration()); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTime.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTime.java index 82556795b5961..8224ef090b78c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTime.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTime.java @@ -7,29 +7,25 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.expression.Expression; -import org.elasticsearch.xpack.sql.expression.function.scalar.ConfigurationFunction; +import org.elasticsearch.xpack.sql.expression.Foldables; import org.elasticsearch.xpack.sql.session.Configuration; -import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataType; import java.time.ZonedDateTime; -import java.util.Objects; -public class CurrentDateTime extends ConfigurationFunction { +public class CurrentDateTime extends CurrentFunction { + private final Expression precision; - private final ZonedDateTime dateTime; public CurrentDateTime(Source source, Expression precision, Configuration configuration) { - super(source, configuration, DataType.DATETIME); + super(source, configuration, nanoPrecision(configuration.now(), precision), DataType.DATETIME); this.precision = precision; - int p = precision != null ? ((Number) precision.fold()).intValue() : 0; - this.dateTime = nanoPrecision(configuration().now(), p); } - @Override - public Object fold() { - return dateTime; + Expression precision() { + return precision; } @Override @@ -37,33 +33,13 @@ protected NodeInfo info() { return NodeInfo.create(this, CurrentDateTime::new, precision, configuration()); } - @Override - public int hashCode() { - return Objects.hash(dateTime); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - CurrentDateTime other = (CurrentDateTime) obj; - return Objects.equals(dateTime, other.dateTime); - } - - static ZonedDateTime nanoPrecision(ZonedDateTime zdt, int precision) { - if (zdt != null) { - int nano = zdt.getNano(); - if (precision >= 0 && precision < 10) { - // remove the remainder - nano = nano - nano % (int) Math.pow(10, (9 - precision)); - return zdt.withNano(nano); - } + static ZonedDateTime nanoPrecision(ZonedDateTime zdt, Expression precisionExpression) { + int precision = precisionExpression != null ? Foldables.intValueOf(precisionExpression) : 0; + int nano = zdt.getNano(); + if (precision >= 0 && precision < 10) { + // remove the remainder + nano = nano - nano % (int) Math.pow(10, (9 - precision)); + return zdt.withNano(nano); } return zdt; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentFunction.java new file mode 100644 index 0000000000000..3aae08903afa5 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentFunction.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; + +import org.elasticsearch.xpack.sql.expression.function.scalar.ConfigurationFunction; +import org.elasticsearch.xpack.sql.session.Configuration; +import org.elasticsearch.xpack.sql.tree.Source; +import org.elasticsearch.xpack.sql.type.DataType; + +import java.time.ZonedDateTime; +import java.util.Objects; + +abstract class CurrentFunction extends ConfigurationFunction { + + private final ZonedDateTime date; + + CurrentFunction(Source source, Configuration configuration, ZonedDateTime date, DataType dataType) { + super(source, configuration, dataType); + this.date = date; + } + + @Override + public Object fold() { + return date; + } + + @Override + public int hashCode() { + return Objects.hash(date); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + CurrentFunction other = (CurrentFunction) obj; + return Objects.equals(date, other.date); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 432872891e5c2..fe8f5ac9925b1 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -465,7 +465,7 @@ public Function visitExtractExpression(ExtractExpressionContext ctx) { @Override public Object visitBuiltinDateTimeFunction(BuiltinDateTimeFunctionContext ctx) { - // maps current_XXX to their respective functions + // maps CURRENT_XXX to its respective function e.g: CURRENT_TIMESTAMP() // since the functions need access to the Configuration, the parser only registers the definition and not the actual function Source source = source(ctx); Literal p = null; @@ -484,13 +484,15 @@ public Object visitBuiltinDateTimeFunction(BuiltinDateTimeFunctionContext ctx) { } String functionName = ctx.name.getText(); - + switch (ctx.name.getType()) { + case SqlBaseLexer.CURRENT_DATE: + return new UnresolvedFunction(source, functionName, ResolutionType.STANDARD, emptyList()); case SqlBaseLexer.CURRENT_TIMESTAMP: return new UnresolvedFunction(source, functionName, ResolutionType.STANDARD, p != null ? singletonList(p) : emptyList()); + default: + throw new ParsingException(source, "Unknown function [{}]", functionName); } - - throw new ParsingException(source, "Unknown function [{}]", functionName); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java index fc113cd58c67c..f62130b14fb07 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java @@ -19,7 +19,7 @@ class SqlBaseLexer extends Lexer { public static final int T__0=1, T__1=2, T__2=3, T__3=4, ALL=5, ANALYZE=6, ANALYZED=7, AND=8, ANY=9, AS=10, ASC=11, BETWEEN=12, BY=13, CAST=14, CATALOG=15, CATALOGS=16, COLUMNS=17, - CONVERT=18, CURRENT=19, CURRENT_TIMESTAMP=20, DAY=21, DAYS=22, DEBUG=23, + CONVERT=18, CURRENT_DATE=19, CURRENT_TIMESTAMP=20, DAY=21, DAYS=22, DEBUG=23, DESC=24, DESCRIBE=25, DISTINCT=26, ESCAPE=27, EXECUTABLE=28, EXISTS=29, EXPLAIN=30, EXTRACT=31, FALSE=32, FIRST=33, FORMAT=34, FROM=35, FULL=36, FUNCTIONS=37, GRAPHVIZ=38, GROUP=39, HAVING=40, HOUR=41, HOURS=42, IN=43, @@ -43,16 +43,16 @@ class SqlBaseLexer extends Lexer { public static final String[] ruleNames = { "T__0", "T__1", "T__2", "T__3", "ALL", "ANALYZE", "ANALYZED", "AND", "ANY", "AS", "ASC", "BETWEEN", "BY", "CAST", "CATALOG", "CATALOGS", "COLUMNS", - "CONVERT", "CURRENT", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", "DESC", - "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", "EXTRACT", - "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", "GROUP", - "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", "LAST", - "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", "MONTH", - "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", "OR", - "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", - "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", - "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", - "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", + "CONVERT", "CURRENT_DATE", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", + "DESC", "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", + "EXTRACT", "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", + "GROUP", "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", + "LAST", "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", + "MONTH", "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", + "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", + "QUERY", "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", + "TABLES", "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", + "WITH", "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", "GUID_ESC", "ESC_END", "EQ", "NULLEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", @@ -64,7 +64,7 @@ class SqlBaseLexer extends Lexer { private static final String[] _LITERAL_NAMES = { null, "'('", "')'", "','", "':'", "'ALL'", "'ANALYZE'", "'ANALYZED'", "'AND'", "'ANY'", "'AS'", "'ASC'", "'BETWEEN'", "'BY'", "'CAST'", "'CATALOG'", - "'CATALOGS'", "'COLUMNS'", "'CONVERT'", "'CURRENT'", "'CURRENT_TIMESTAMP'", + "'CATALOGS'", "'COLUMNS'", "'CONVERT'", "'CURRENT_DATE'", "'CURRENT_TIMESTAMP'", "'DAY'", "'DAYS'", "'DEBUG'", "'DESC'", "'DESCRIBE'", "'DISTINCT'", "'ESCAPE'", "'EXECUTABLE'", "'EXISTS'", "'EXPLAIN'", "'EXTRACT'", "'FALSE'", "'FIRST'", "'FORMAT'", "'FROM'", "'FULL'", "'FUNCTIONS'", "'GRAPHVIZ'", "'GROUP'", @@ -83,16 +83,16 @@ class SqlBaseLexer extends Lexer { private static final String[] _SYMBOLIC_NAMES = { null, null, null, null, null, "ALL", "ANALYZE", "ANALYZED", "AND", "ANY", "AS", "ASC", "BETWEEN", "BY", "CAST", "CATALOG", "CATALOGS", "COLUMNS", - "CONVERT", "CURRENT", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", "DESC", - "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", "EXTRACT", - "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", "GROUP", - "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", "LAST", - "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", "MONTH", - "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", "OR", - "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", - "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", - "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", - "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", + "CONVERT", "CURRENT_DATE", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", + "DESC", "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", + "EXTRACT", "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", + "GROUP", "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", + "LAST", "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", + "MONTH", "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", + "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", + "QUERY", "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", + "TABLES", "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", + "WITH", "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", "GUID_ESC", "ESC_END", "EQ", "NULLEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", @@ -155,7 +155,7 @@ public SqlBaseLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2\u0080\u0423\b\1\4"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2\u0080\u0428\b\1\4"+ "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+ "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+ "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+ @@ -176,345 +176,346 @@ public SqlBaseLexer(CharStream input) { "\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\21\3\21\3\21\3\21"+ "\3\21\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23"+ "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24"+ - "\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25"+ - "\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27"+ - "\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32"+ - "\3\32\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33"+ - "\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\35\3\35"+ - "\3\35\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\37\3\37"+ - "\3\37\3\37\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3 \3 \3 \3 \3!\3!\3!\3!\3!"+ - "\3!\3\"\3\"\3\"\3\"\3\"\3\"\3#\3#\3#\3#\3#\3#\3#\3$\3$\3$\3$\3$\3%\3%"+ - "\3%\3%\3%\3&\3&\3&\3&\3&\3&\3&\3&\3&\3&\3\'\3\'\3\'\3\'\3\'\3\'\3\'\3"+ - "\'\3\'\3(\3(\3(\3(\3(\3(\3)\3)\3)\3)\3)\3)\3)\3*\3*\3*\3*\3*\3+\3+\3+"+ - "\3+\3+\3+\3,\3,\3,\3-\3-\3-\3-\3-\3-\3.\3.\3.\3.\3.\3.\3.\3.\3.\3/\3/"+ - "\3/\3\60\3\60\3\60\3\60\3\60\3\61\3\61\3\61\3\61\3\61\3\62\3\62\3\62\3"+ - "\62\3\62\3\63\3\63\3\63\3\63\3\63\3\64\3\64\3\64\3\64\3\64\3\64\3\65\3"+ - "\65\3\65\3\65\3\65\3\65\3\65\3\66\3\66\3\66\3\66\3\66\3\66\3\67\3\67\3"+ - "\67\3\67\3\67\3\67\3\67\38\38\38\38\38\38\38\38\39\39\39\39\39\39\3:\3"+ - ":\3:\3:\3:\3:\3:\3;\3;\3;\3;\3;\3;\3;\3;\3<\3<\3<\3<\3=\3=\3=\3=\3=\3"+ - ">\3>\3>\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3@\3@\3@\3@\3@\3@\3A\3A\3A\3B\3"+ - "B\3B\3B\3B\3B\3C\3C\3C\3C\3C\3C\3D\3D\3D\3D\3D\3D\3D\3E\3E\3E\3E\3E\3"+ - "E\3E\3E\3E\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3G\3H\3H\3H\3H\3H\3H\3I\3I\3"+ - "I\3I\3I\3I\3J\3J\3J\3J\3J\3J\3J\3J\3K\3K\3K\3K\3K\3K\3K\3L\3L\3L\3L\3"+ - "L\3L\3L\3L\3M\3M\3M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3O\3O\3O\3O\3P\3P\3P\3"+ - "P\3P\3P\3Q\3Q\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\3R\3S\3S\3S\3S\3S\3T\3T\3T\3"+ - "U\3U\3U\3U\3U\3V\3V\3V\3V\3V\3V\3W\3W\3W\3W\3W\3W\3X\3X\3X\3X\3X\3X\3"+ - "X\3Y\3Y\3Y\3Y\3Y\3Y\3Z\3Z\3Z\3Z\3Z\3[\3[\3[\3[\3[\3\\\3\\\3\\\3\\\3\\"+ - "\3\\\3]\3]\3]\3]\3]\3]\3]\3]\3^\3^\3^\3^\3_\3_\3_\3_\3_\3_\3_\3`\3`\3"+ - "`\3a\3a\3a\3b\3b\3b\3b\3c\3c\3c\3c\3c\3c\3d\3d\3e\3e\3f\3f\3f\3f\3g\3"+ - "g\3g\3g\5g\u0364\ng\3h\3h\3i\3i\3i\3j\3j\3k\3k\3k\3l\3l\3m\3m\3n\3n\3"+ - "o\3o\3p\3p\3q\3q\3q\3r\3r\3s\3s\3t\3t\3t\3t\7t\u0385\nt\ft\16t\u0388\13"+ - "t\3t\3t\3u\6u\u038d\nu\ru\16u\u038e\3v\6v\u0392\nv\rv\16v\u0393\3v\3v"+ - "\7v\u0398\nv\fv\16v\u039b\13v\3v\3v\6v\u039f\nv\rv\16v\u03a0\3v\6v\u03a4"+ - "\nv\rv\16v\u03a5\3v\3v\7v\u03aa\nv\fv\16v\u03ad\13v\5v\u03af\nv\3v\3v"+ - "\3v\3v\6v\u03b5\nv\rv\16v\u03b6\3v\3v\5v\u03bb\nv\3w\3w\5w\u03bf\nw\3"+ - "w\3w\3w\7w\u03c4\nw\fw\16w\u03c7\13w\3x\3x\3x\3x\6x\u03cd\nx\rx\16x\u03ce"+ - "\3y\3y\3y\6y\u03d4\ny\ry\16y\u03d5\3z\3z\3z\3z\7z\u03dc\nz\fz\16z\u03df"+ - "\13z\3z\3z\3{\3{\3{\3{\7{\u03e7\n{\f{\16{\u03ea\13{\3{\3{\3|\3|\5|\u03f0"+ - "\n|\3|\6|\u03f3\n|\r|\16|\u03f4\3}\3}\3~\3~\3\177\3\177\3\177\3\177\7"+ - "\177\u03ff\n\177\f\177\16\177\u0402\13\177\3\177\5\177\u0405\n\177\3\177"+ - "\5\177\u0408\n\177\3\177\3\177\3\u0080\3\u0080\3\u0080\3\u0080\3\u0080"+ - "\7\u0080\u0411\n\u0080\f\u0080\16\u0080\u0414\13\u0080\3\u0080\3\u0080"+ - "\3\u0080\3\u0080\3\u0080\3\u0081\6\u0081\u041c\n\u0081\r\u0081\16\u0081"+ - "\u041d\3\u0081\3\u0081\3\u0082\3\u0082\3\u0412\2\u0083\3\3\5\4\7\5\t\6"+ - "\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31\16\33\17\35\20\37\21!\22#\23%\24"+ - "\'\25)\26+\27-\30/\31\61\32\63\33\65\34\67\359\36;\37= ?!A\"C#E$G%I&K"+ - "\'M(O)Q*S+U,W-Y.[/]\60_\61a\62c\63e\64g\65i\66k\67m8o9q:s;u{?}@\177"+ - "A\u0081B\u0083C\u0085D\u0087E\u0089F\u008bG\u008dH\u008fI\u0091J\u0093"+ - "K\u0095L\u0097M\u0099N\u009bO\u009dP\u009fQ\u00a1R\u00a3S\u00a5T\u00a7"+ - "U\u00a9V\u00abW\u00adX\u00afY\u00b1Z\u00b3[\u00b5\\\u00b7]\u00b9^\u00bb"+ - "_\u00bd`\u00bfa\u00c1b\u00c3c\u00c5d\u00c7e\u00c9f\u00cbg\u00cdh\u00cf"+ - "i\u00d1j\u00d3k\u00d5l\u00d7m\u00d9n\u00dbo\u00ddp\u00dfq\u00e1r\u00e3"+ - "s\u00e5t\u00e7u\u00e9v\u00ebw\u00edx\u00efy\u00f1z\u00f3{\u00f5|\u00f7"+ - "\2\u00f9\2\u00fb\2\u00fd}\u00ff~\u0101\177\u0103\u0080\3\2\f\3\2))\4\2"+ - "BBaa\5\2<\3\2\2\2\u01c3\u01c4\7G\2\2\u01c4\u01c5\7Z\2\2\u01c5\u01c6\7V\2"+ - "\2\u01c6\u01c7\7T\2\2\u01c7\u01c8\7C\2\2\u01c8\u01c9\7E\2\2\u01c9\u01ca"+ - "\7V\2\2\u01ca@\3\2\2\2\u01cb\u01cc\7H\2\2\u01cc\u01cd\7C\2\2\u01cd\u01ce"+ - "\7N\2\2\u01ce\u01cf\7U\2\2\u01cf\u01d0\7G\2\2\u01d0B\3\2\2\2\u01d1\u01d2"+ - "\7H\2\2\u01d2\u01d3\7K\2\2\u01d3\u01d4\7T\2\2\u01d4\u01d5\7U\2\2\u01d5"+ - "\u01d6\7V\2\2\u01d6D\3\2\2\2\u01d7\u01d8\7H\2\2\u01d8\u01d9\7Q\2\2\u01d9"+ - "\u01da\7T\2\2\u01da\u01db\7O\2\2\u01db\u01dc\7C\2\2\u01dc\u01dd\7V\2\2"+ - "\u01ddF\3\2\2\2\u01de\u01df\7H\2\2\u01df\u01e0\7T\2\2\u01e0\u01e1\7Q\2"+ - "\2\u01e1\u01e2\7O\2\2\u01e2H\3\2\2\2\u01e3\u01e4\7H\2\2\u01e4\u01e5\7"+ - "W\2\2\u01e5\u01e6\7N\2\2\u01e6\u01e7\7N\2\2\u01e7J\3\2\2\2\u01e8\u01e9"+ - "\7H\2\2\u01e9\u01ea\7W\2\2\u01ea\u01eb\7P\2\2\u01eb\u01ec\7E\2\2\u01ec"+ - "\u01ed\7V\2\2\u01ed\u01ee\7K\2\2\u01ee\u01ef\7Q\2\2\u01ef\u01f0\7P\2\2"+ - "\u01f0\u01f1\7U\2\2\u01f1L\3\2\2\2\u01f2\u01f3\7I\2\2\u01f3\u01f4\7T\2"+ - "\2\u01f4\u01f5\7C\2\2\u01f5\u01f6\7R\2\2\u01f6\u01f7\7J\2\2\u01f7\u01f8"+ - "\7X\2\2\u01f8\u01f9\7K\2\2\u01f9\u01fa\7\\\2\2\u01faN\3\2\2\2\u01fb\u01fc"+ - "\7I\2\2\u01fc\u01fd\7T\2\2\u01fd\u01fe\7Q\2\2\u01fe\u01ff\7W\2\2\u01ff"+ - "\u0200\7R\2\2\u0200P\3\2\2\2\u0201\u0202\7J\2\2\u0202\u0203\7C\2\2\u0203"+ - "\u0204\7X\2\2\u0204\u0205\7K\2\2\u0205\u0206\7P\2\2\u0206\u0207\7I\2\2"+ - "\u0207R\3\2\2\2\u0208\u0209\7J\2\2\u0209\u020a\7Q\2\2\u020a\u020b\7W\2"+ - "\2\u020b\u020c\7T\2\2\u020cT\3\2\2\2\u020d\u020e\7J\2\2\u020e\u020f\7"+ - "Q\2\2\u020f\u0210\7W\2\2\u0210\u0211\7T\2\2\u0211\u0212\7U\2\2\u0212V"+ - "\3\2\2\2\u0213\u0214\7K\2\2\u0214\u0215\7P\2\2\u0215X\3\2\2\2\u0216\u0217"+ - "\7K\2\2\u0217\u0218\7P\2\2\u0218\u0219\7P\2\2\u0219\u021a\7G\2\2\u021a"+ - "\u021b\7T\2\2\u021bZ\3\2\2\2\u021c\u021d\7K\2\2\u021d\u021e\7P\2\2\u021e"+ - "\u021f\7V\2\2\u021f\u0220\7G\2\2\u0220\u0221\7T\2\2\u0221\u0222\7X\2\2"+ - "\u0222\u0223\7C\2\2\u0223\u0224\7N\2\2\u0224\\\3\2\2\2\u0225\u0226\7K"+ - "\2\2\u0226\u0227\7U\2\2\u0227^\3\2\2\2\u0228\u0229\7L\2\2\u0229\u022a"+ - "\7Q\2\2\u022a\u022b\7K\2\2\u022b\u022c\7P\2\2\u022c`\3\2\2\2\u022d\u022e"+ - "\7N\2\2\u022e\u022f\7C\2\2\u022f\u0230\7U\2\2\u0230\u0231\7V\2\2\u0231"+ - "b\3\2\2\2\u0232\u0233\7N\2\2\u0233\u0234\7G\2\2\u0234\u0235\7H\2\2\u0235"+ - "\u0236\7V\2\2\u0236d\3\2\2\2\u0237\u0238\7N\2\2\u0238\u0239\7K\2\2\u0239"+ - "\u023a\7M\2\2\u023a\u023b\7G\2\2\u023bf\3\2\2\2\u023c\u023d\7N\2\2\u023d"+ - "\u023e\7K\2\2\u023e\u023f\7O\2\2\u023f\u0240\7K\2\2\u0240\u0241\7V\2\2"+ - "\u0241h\3\2\2\2\u0242\u0243\7O\2\2\u0243\u0244\7C\2\2\u0244\u0245\7R\2"+ - "\2\u0245\u0246\7R\2\2\u0246\u0247\7G\2\2\u0247\u0248\7F\2\2\u0248j\3\2"+ - "\2\2\u0249\u024a\7O\2\2\u024a\u024b\7C\2\2\u024b\u024c\7V\2\2\u024c\u024d"+ - "\7E\2\2\u024d\u024e\7J\2\2\u024el\3\2\2\2\u024f\u0250\7O\2\2\u0250\u0251"+ - "\7K\2\2\u0251\u0252\7P\2\2\u0252\u0253\7W\2\2\u0253\u0254\7V\2\2\u0254"+ - "\u0255\7G\2\2\u0255n\3\2\2\2\u0256\u0257\7O\2\2\u0257\u0258\7K\2\2\u0258"+ - "\u0259\7P\2\2\u0259\u025a\7W\2\2\u025a\u025b\7V\2\2\u025b\u025c\7G\2\2"+ - "\u025c\u025d\7U\2\2\u025dp\3\2\2\2\u025e\u025f\7O\2\2\u025f\u0260\7Q\2"+ - "\2\u0260\u0261\7P\2\2\u0261\u0262\7V\2\2\u0262\u0263\7J\2\2\u0263r\3\2"+ - "\2\2\u0264\u0265\7O\2\2\u0265\u0266\7Q\2\2\u0266\u0267\7P\2\2\u0267\u0268"+ - "\7V\2\2\u0268\u0269\7J\2\2\u0269\u026a\7U\2\2\u026at\3\2\2\2\u026b\u026c"+ - "\7P\2\2\u026c\u026d\7C\2\2\u026d\u026e\7V\2\2\u026e\u026f\7W\2\2\u026f"+ - "\u0270\7T\2\2\u0270\u0271\7C\2\2\u0271\u0272\7N\2\2\u0272v\3\2\2\2\u0273"+ - "\u0274\7P\2\2\u0274\u0275\7Q\2\2\u0275\u0276\7V\2\2\u0276x\3\2\2\2\u0277"+ - "\u0278\7P\2\2\u0278\u0279\7W\2\2\u0279\u027a\7N\2\2\u027a\u027b\7N\2\2"+ - "\u027bz\3\2\2\2\u027c\u027d\7P\2\2\u027d\u027e\7W\2\2\u027e\u027f\7N\2"+ - "\2\u027f\u0280\7N\2\2\u0280\u0281\7U\2\2\u0281|\3\2\2\2\u0282\u0283\7"+ - "Q\2\2\u0283\u0284\7P\2\2\u0284~\3\2\2\2\u0285\u0286\7Q\2\2\u0286\u0287"+ - "\7R\2\2\u0287\u0288\7V\2\2\u0288\u0289\7K\2\2\u0289\u028a\7O\2\2\u028a"+ - "\u028b\7K\2\2\u028b\u028c\7\\\2\2\u028c\u028d\7G\2\2\u028d\u028e\7F\2"+ - "\2\u028e\u0080\3\2\2\2\u028f\u0290\7Q\2\2\u0290\u0291\7T\2\2\u0291\u0082"+ - "\3\2\2\2\u0292\u0293\7Q\2\2\u0293\u0294\7T\2\2\u0294\u0295\7F\2\2\u0295"+ - "\u0296\7G\2\2\u0296\u0297\7T\2\2\u0297\u0084\3\2\2\2\u0298\u0299\7Q\2"+ - "\2\u0299\u029a\7W\2\2\u029a\u029b\7V\2\2\u029b\u029c\7G\2\2\u029c\u029d"+ - "\7T\2\2\u029d\u0086\3\2\2\2\u029e\u029f\7R\2\2\u029f\u02a0\7C\2\2\u02a0"+ - "\u02a1\7T\2\2\u02a1\u02a2\7U\2\2\u02a2\u02a3\7G\2\2\u02a3\u02a4\7F\2\2"+ - "\u02a4\u0088\3\2\2\2\u02a5\u02a6\7R\2\2\u02a6\u02a7\7J\2\2\u02a7\u02a8"+ - "\7[\2\2\u02a8\u02a9\7U\2\2\u02a9\u02aa\7K\2\2\u02aa\u02ab\7E\2\2\u02ab"+ - "\u02ac\7C\2\2\u02ac\u02ad\7N\2\2\u02ad\u008a\3\2\2\2\u02ae\u02af\7R\2"+ - "\2\u02af\u02b0\7N\2\2\u02b0\u02b1\7C\2\2\u02b1\u02b2\7P\2\2\u02b2\u008c"+ - "\3\2\2\2\u02b3\u02b4\7T\2\2\u02b4\u02b5\7K\2\2\u02b5\u02b6\7I\2\2\u02b6"+ - "\u02b7\7J\2\2\u02b7\u02b8\7V\2\2\u02b8\u008e\3\2\2\2\u02b9\u02ba\7T\2"+ - "\2\u02ba\u02bb\7N\2\2\u02bb\u02bc\7K\2\2\u02bc\u02bd\7M\2\2\u02bd\u02be"+ - "\7G\2\2\u02be\u0090\3\2\2\2\u02bf\u02c0\7S\2\2\u02c0\u02c1\7W\2\2\u02c1"+ - "\u02c2\7G\2\2\u02c2\u02c3\7T\2\2\u02c3\u02c4\7[\2\2\u02c4\u0092\3\2\2"+ - "\2\u02c5\u02c6\7U\2\2\u02c6\u02c7\7E\2\2\u02c7\u02c8\7J\2\2\u02c8\u02c9"+ - "\7G\2\2\u02c9\u02ca\7O\2\2\u02ca\u02cb\7C\2\2\u02cb\u02cc\7U\2\2\u02cc"+ - "\u0094\3\2\2\2\u02cd\u02ce\7U\2\2\u02ce\u02cf\7G\2\2\u02cf\u02d0\7E\2"+ - "\2\u02d0\u02d1\7Q\2\2\u02d1\u02d2\7P\2\2\u02d2\u02d3\7F\2\2\u02d3\u0096"+ - "\3\2\2\2\u02d4\u02d5\7U\2\2\u02d5\u02d6\7G\2\2\u02d6\u02d7\7E\2\2\u02d7"+ - "\u02d8\7Q\2\2\u02d8\u02d9\7P\2\2\u02d9\u02da\7F\2\2\u02da\u02db\7U\2\2"+ - "\u02db\u0098\3\2\2\2\u02dc\u02dd\7U\2\2\u02dd\u02de\7G\2\2\u02de\u02df"+ - "\7N\2\2\u02df\u02e0\7G\2\2\u02e0\u02e1\7E\2\2\u02e1\u02e2\7V\2\2\u02e2"+ - "\u009a\3\2\2\2\u02e3\u02e4\7U\2\2\u02e4\u02e5\7J\2\2\u02e5\u02e6\7Q\2"+ - "\2\u02e6\u02e7\7Y\2\2\u02e7\u009c\3\2\2\2\u02e8\u02e9\7U\2\2\u02e9\u02ea"+ - "\7[\2\2\u02ea\u02eb\7U\2\2\u02eb\u009e\3\2\2\2\u02ec\u02ed\7V\2\2\u02ed"+ - "\u02ee\7C\2\2\u02ee\u02ef\7D\2\2\u02ef\u02f0\7N\2\2\u02f0\u02f1\7G\2\2"+ - "\u02f1\u00a0\3\2\2\2\u02f2\u02f3\7V\2\2\u02f3\u02f4\7C\2\2\u02f4\u02f5"+ - "\7D\2\2\u02f5\u02f6\7N\2\2\u02f6\u02f7\7G\2\2\u02f7\u02f8\7U\2\2\u02f8"+ - "\u00a2\3\2\2\2\u02f9\u02fa\7V\2\2\u02fa\u02fb\7G\2\2\u02fb\u02fc\7Z\2"+ - "\2\u02fc\u02fd\7V\2\2\u02fd\u00a4\3\2\2\2\u02fe\u02ff\7V\2\2\u02ff\u0300"+ - "\7T\2\2\u0300\u0301\7W\2\2\u0301\u0302\7G\2\2\u0302\u00a6\3\2\2\2\u0303"+ - "\u0304\7V\2\2\u0304\u0305\7Q\2\2\u0305\u00a8\3\2\2\2\u0306\u0307\7V\2"+ - "\2\u0307\u0308\7[\2\2\u0308\u0309\7R\2\2\u0309\u030a\7G\2\2\u030a\u00aa"+ - "\3\2\2\2\u030b\u030c\7V\2\2\u030c\u030d\7[\2\2\u030d\u030e\7R\2\2\u030e"+ - "\u030f\7G\2\2\u030f\u0310\7U\2\2\u0310\u00ac\3\2\2\2\u0311\u0312\7W\2"+ - "\2\u0312\u0313\7U\2\2\u0313\u0314\7K\2\2\u0314\u0315\7P\2\2\u0315\u0316"+ - "\7I\2\2\u0316\u00ae\3\2\2\2\u0317\u0318\7X\2\2\u0318\u0319\7G\2\2\u0319"+ - "\u031a\7T\2\2\u031a\u031b\7K\2\2\u031b\u031c\7H\2\2\u031c\u031d\7[\2\2"+ - "\u031d\u00b0\3\2\2\2\u031e\u031f\7Y\2\2\u031f\u0320\7J\2\2\u0320\u0321"+ - "\7G\2\2\u0321\u0322\7T\2\2\u0322\u0323\7G\2\2\u0323\u00b2\3\2\2\2\u0324"+ - "\u0325\7Y\2\2\u0325\u0326\7K\2\2\u0326\u0327\7V\2\2\u0327\u0328\7J\2\2"+ - "\u0328\u00b4\3\2\2\2\u0329\u032a\7[\2\2\u032a\u032b\7G\2\2\u032b\u032c"+ - "\7C\2\2\u032c\u032d\7T\2\2\u032d\u00b6\3\2\2\2\u032e\u032f\7[\2\2\u032f"+ - "\u0330\7G\2\2\u0330\u0331\7C\2\2\u0331\u0332\7T\2\2\u0332\u0333\7U\2\2"+ - "\u0333\u00b8\3\2\2\2\u0334\u0335\7}\2\2\u0335\u0336\7G\2\2\u0336\u0337"+ - "\7U\2\2\u0337\u0338\7E\2\2\u0338\u0339\7C\2\2\u0339\u033a\7R\2\2\u033a"+ - "\u033b\7G\2\2\u033b\u00ba\3\2\2\2\u033c\u033d\7}\2\2\u033d\u033e\7H\2"+ - "\2\u033e\u033f\7P\2\2\u033f\u00bc\3\2\2\2\u0340\u0341\7}\2\2\u0341\u0342"+ - "\7N\2\2\u0342\u0343\7K\2\2\u0343\u0344\7O\2\2\u0344\u0345\7K\2\2\u0345"+ - "\u0346\7V\2\2\u0346\u00be\3\2\2\2\u0347\u0348\7}\2\2\u0348\u0349\7F\2"+ - "\2\u0349\u00c0\3\2\2\2\u034a\u034b\7}\2\2\u034b\u034c\7V\2\2\u034c\u00c2"+ - "\3\2\2\2\u034d\u034e\7}\2\2\u034e\u034f\7V\2\2\u034f\u0350\7U\2\2\u0350"+ - "\u00c4\3\2\2\2\u0351\u0352\7}\2\2\u0352\u0353\7I\2\2\u0353\u0354\7W\2"+ - "\2\u0354\u0355\7K\2\2\u0355\u0356\7F\2\2\u0356\u00c6\3\2\2\2\u0357\u0358"+ - "\7\177\2\2\u0358\u00c8\3\2\2\2\u0359\u035a\7?\2\2\u035a\u00ca\3\2\2\2"+ - "\u035b\u035c\7>\2\2\u035c\u035d\7?\2\2\u035d\u035e\7@\2\2\u035e\u00cc"+ - "\3\2\2\2\u035f\u0360\7>\2\2\u0360\u0364\7@\2\2\u0361\u0362\7#\2\2\u0362"+ - "\u0364\7?\2\2\u0363\u035f\3\2\2\2\u0363\u0361\3\2\2\2\u0364\u00ce\3\2"+ - "\2\2\u0365\u0366\7>\2\2\u0366\u00d0\3\2\2\2\u0367\u0368\7>\2\2\u0368\u0369"+ - "\7?\2\2\u0369\u00d2\3\2\2\2\u036a\u036b\7@\2\2\u036b\u00d4\3\2\2\2\u036c"+ - "\u036d\7@\2\2\u036d\u036e\7?\2\2\u036e\u00d6\3\2\2\2\u036f\u0370\7-\2"+ - "\2\u0370\u00d8\3\2\2\2\u0371\u0372\7/\2\2\u0372\u00da\3\2\2\2\u0373\u0374"+ - "\7,\2\2\u0374\u00dc\3\2\2\2\u0375\u0376\7\61\2\2\u0376\u00de\3\2\2\2\u0377"+ - "\u0378\7\'\2\2\u0378\u00e0\3\2\2\2\u0379\u037a\7~\2\2\u037a\u037b\7~\2"+ - "\2\u037b\u00e2\3\2\2\2\u037c\u037d\7\60\2\2\u037d\u00e4\3\2\2\2\u037e"+ - "\u037f\7A\2\2\u037f\u00e6\3\2\2\2\u0380\u0386\7)\2\2\u0381\u0385\n\2\2"+ - "\2\u0382\u0383\7)\2\2\u0383\u0385\7)\2\2\u0384\u0381\3\2\2\2\u0384\u0382"+ - "\3\2\2\2\u0385\u0388\3\2\2\2\u0386\u0384\3\2\2\2\u0386\u0387\3\2\2\2\u0387"+ - "\u0389\3\2\2\2\u0388\u0386\3\2\2\2\u0389\u038a\7)\2\2\u038a\u00e8\3\2"+ - "\2\2\u038b\u038d\5\u00f9}\2\u038c\u038b\3\2\2\2\u038d\u038e\3\2\2\2\u038e"+ - "\u038c\3\2\2\2\u038e\u038f\3\2\2\2\u038f\u00ea\3\2\2\2\u0390\u0392\5\u00f9"+ + "\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25"+ + "\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26"+ + "\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31"+ + "\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33"+ + "\3\33\3\33\3\33\3\33\3\33\3\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\35"+ + "\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36"+ + "\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3"+ + " \3 \3 \3 \3!\3!\3!\3!\3!\3!\3\"\3\"\3\"\3\"\3\"\3\"\3#\3#\3#\3#\3#\3"+ + "#\3#\3$\3$\3$\3$\3$\3%\3%\3%\3%\3%\3&\3&\3&\3&\3&\3&\3&\3&\3&\3&\3\'\3"+ + "\'\3\'\3\'\3\'\3\'\3\'\3\'\3\'\3(\3(\3(\3(\3(\3(\3)\3)\3)\3)\3)\3)\3)"+ + "\3*\3*\3*\3*\3*\3+\3+\3+\3+\3+\3+\3,\3,\3,\3-\3-\3-\3-\3-\3-\3.\3.\3."+ + "\3.\3.\3.\3.\3.\3.\3/\3/\3/\3\60\3\60\3\60\3\60\3\60\3\61\3\61\3\61\3"+ + "\61\3\61\3\62\3\62\3\62\3\62\3\62\3\63\3\63\3\63\3\63\3\63\3\64\3\64\3"+ + "\64\3\64\3\64\3\64\3\65\3\65\3\65\3\65\3\65\3\65\3\65\3\66\3\66\3\66\3"+ + "\66\3\66\3\66\3\67\3\67\3\67\3\67\3\67\3\67\3\67\38\38\38\38\38\38\38"+ + "\38\39\39\39\39\39\39\3:\3:\3:\3:\3:\3:\3:\3;\3;\3;\3;\3;\3;\3;\3;\3<"+ + "\3<\3<\3<\3=\3=\3=\3=\3=\3>\3>\3>\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3@\3@"+ + "\3@\3@\3@\3@\3A\3A\3A\3B\3B\3B\3B\3B\3B\3C\3C\3C\3C\3C\3C\3D\3D\3D\3D"+ + "\3D\3D\3D\3E\3E\3E\3E\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3G"+ + "\3H\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3I\3J\3J\3J\3J\3J\3J\3J\3J\3K\3K\3K"+ + "\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3L\3L\3M\3M\3M\3M\3M\3M\3M\3N\3N\3N\3N"+ + "\3N\3O\3O\3O\3O\3P\3P\3P\3P\3P\3P\3Q\3Q\3Q\3Q\3Q\3Q\3Q\3R\3R\3R\3R\3R"+ + "\3S\3S\3S\3S\3S\3T\3T\3T\3U\3U\3U\3U\3U\3V\3V\3V\3V\3V\3V\3W\3W\3W\3W"+ + "\3W\3W\3X\3X\3X\3X\3X\3X\3X\3Y\3Y\3Y\3Y\3Y\3Y\3Z\3Z\3Z\3Z\3Z\3[\3[\3["+ + "\3[\3[\3\\\3\\\3\\\3\\\3\\\3\\\3]\3]\3]\3]\3]\3]\3]\3]\3^\3^\3^\3^\3_"+ + "\3_\3_\3_\3_\3_\3_\3`\3`\3`\3a\3a\3a\3b\3b\3b\3b\3c\3c\3c\3c\3c\3c\3d"+ + "\3d\3e\3e\3f\3f\3f\3f\3g\3g\3g\3g\5g\u0369\ng\3h\3h\3i\3i\3i\3j\3j\3k"+ + "\3k\3k\3l\3l\3m\3m\3n\3n\3o\3o\3p\3p\3q\3q\3q\3r\3r\3s\3s\3t\3t\3t\3t"+ + "\7t\u038a\nt\ft\16t\u038d\13t\3t\3t\3u\6u\u0392\nu\ru\16u\u0393\3v\6v"+ + "\u0397\nv\rv\16v\u0398\3v\3v\7v\u039d\nv\fv\16v\u03a0\13v\3v\3v\6v\u03a4"+ + "\nv\rv\16v\u03a5\3v\6v\u03a9\nv\rv\16v\u03aa\3v\3v\7v\u03af\nv\fv\16v"+ + "\u03b2\13v\5v\u03b4\nv\3v\3v\3v\3v\6v\u03ba\nv\rv\16v\u03bb\3v\3v\5v\u03c0"+ + "\nv\3w\3w\5w\u03c4\nw\3w\3w\3w\7w\u03c9\nw\fw\16w\u03cc\13w\3x\3x\3x\3"+ + "x\6x\u03d2\nx\rx\16x\u03d3\3y\3y\3y\6y\u03d9\ny\ry\16y\u03da\3z\3z\3z"+ + "\3z\7z\u03e1\nz\fz\16z\u03e4\13z\3z\3z\3{\3{\3{\3{\7{\u03ec\n{\f{\16{"+ + "\u03ef\13{\3{\3{\3|\3|\5|\u03f5\n|\3|\6|\u03f8\n|\r|\16|\u03f9\3}\3}\3"+ + "~\3~\3\177\3\177\3\177\3\177\7\177\u0404\n\177\f\177\16\177\u0407\13\177"+ + "\3\177\5\177\u040a\n\177\3\177\5\177\u040d\n\177\3\177\3\177\3\u0080\3"+ + "\u0080\3\u0080\3\u0080\3\u0080\7\u0080\u0416\n\u0080\f\u0080\16\u0080"+ + "\u0419\13\u0080\3\u0080\3\u0080\3\u0080\3\u0080\3\u0080\3\u0081\6\u0081"+ + "\u0421\n\u0081\r\u0081\16\u0081\u0422\3\u0081\3\u0081\3\u0082\3\u0082"+ + "\3\u0417\2\u0083\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31"+ + "\16\33\17\35\20\37\21!\22#\23%\24\'\25)\26+\27-\30/\31\61\32\63\33\65"+ + "\34\67\359\36;\37= ?!A\"C#E$G%I&K\'M(O)Q*S+U,W-Y.[/]\60_\61a\62c\63e\64"+ + "g\65i\66k\67m8o9q:s;u{?}@\177A\u0081B\u0083C\u0085D\u0087E\u0089"+ + "F\u008bG\u008dH\u008fI\u0091J\u0093K\u0095L\u0097M\u0099N\u009bO\u009d"+ + "P\u009fQ\u00a1R\u00a3S\u00a5T\u00a7U\u00a9V\u00abW\u00adX\u00afY\u00b1"+ + "Z\u00b3[\u00b5\\\u00b7]\u00b9^\u00bb_\u00bd`\u00bfa\u00c1b\u00c3c\u00c5"+ + "d\u00c7e\u00c9f\u00cbg\u00cdh\u00cfi\u00d1j\u00d3k\u00d5l\u00d7m\u00d9"+ + "n\u00dbo\u00ddp\u00dfq\u00e1r\u00e3s\u00e5t\u00e7u\u00e9v\u00ebw\u00ed"+ + "x\u00efy\u00f1z\u00f3{\u00f5|\u00f7\2\u00f9\2\u00fb\2\u00fd}\u00ff~\u0101"+ + "\177\u0103\u0080\3\2\f\3\2))\4\2BBaa\5\2<\3\2\2"+ + "\2\u01c8\u01c9\7G\2\2\u01c9\u01ca\7Z\2\2\u01ca\u01cb\7V\2\2\u01cb\u01cc"+ + "\7T\2\2\u01cc\u01cd\7C\2\2\u01cd\u01ce\7E\2\2\u01ce\u01cf\7V\2\2\u01cf"+ + "@\3\2\2\2\u01d0\u01d1\7H\2\2\u01d1\u01d2\7C\2\2\u01d2\u01d3\7N\2\2\u01d3"+ + "\u01d4\7U\2\2\u01d4\u01d5\7G\2\2\u01d5B\3\2\2\2\u01d6\u01d7\7H\2\2\u01d7"+ + "\u01d8\7K\2\2\u01d8\u01d9\7T\2\2\u01d9\u01da\7U\2\2\u01da\u01db\7V\2\2"+ + "\u01dbD\3\2\2\2\u01dc\u01dd\7H\2\2\u01dd\u01de\7Q\2\2\u01de\u01df\7T\2"+ + "\2\u01df\u01e0\7O\2\2\u01e0\u01e1\7C\2\2\u01e1\u01e2\7V\2\2\u01e2F\3\2"+ + "\2\2\u01e3\u01e4\7H\2\2\u01e4\u01e5\7T\2\2\u01e5\u01e6\7Q\2\2\u01e6\u01e7"+ + "\7O\2\2\u01e7H\3\2\2\2\u01e8\u01e9\7H\2\2\u01e9\u01ea\7W\2\2\u01ea\u01eb"+ + "\7N\2\2\u01eb\u01ec\7N\2\2\u01ecJ\3\2\2\2\u01ed\u01ee\7H\2\2\u01ee\u01ef"+ + "\7W\2\2\u01ef\u01f0\7P\2\2\u01f0\u01f1\7E\2\2\u01f1\u01f2\7V\2\2\u01f2"+ + "\u01f3\7K\2\2\u01f3\u01f4\7Q\2\2\u01f4\u01f5\7P\2\2\u01f5\u01f6\7U\2\2"+ + "\u01f6L\3\2\2\2\u01f7\u01f8\7I\2\2\u01f8\u01f9\7T\2\2\u01f9\u01fa\7C\2"+ + "\2\u01fa\u01fb\7R\2\2\u01fb\u01fc\7J\2\2\u01fc\u01fd\7X\2\2\u01fd\u01fe"+ + "\7K\2\2\u01fe\u01ff\7\\\2\2\u01ffN\3\2\2\2\u0200\u0201\7I\2\2\u0201\u0202"+ + "\7T\2\2\u0202\u0203\7Q\2\2\u0203\u0204\7W\2\2\u0204\u0205\7R\2\2\u0205"+ + "P\3\2\2\2\u0206\u0207\7J\2\2\u0207\u0208\7C\2\2\u0208\u0209\7X\2\2\u0209"+ + "\u020a\7K\2\2\u020a\u020b\7P\2\2\u020b\u020c\7I\2\2\u020cR\3\2\2\2\u020d"+ + "\u020e\7J\2\2\u020e\u020f\7Q\2\2\u020f\u0210\7W\2\2\u0210\u0211\7T\2\2"+ + "\u0211T\3\2\2\2\u0212\u0213\7J\2\2\u0213\u0214\7Q\2\2\u0214\u0215\7W\2"+ + "\2\u0215\u0216\7T\2\2\u0216\u0217\7U\2\2\u0217V\3\2\2\2\u0218\u0219\7"+ + "K\2\2\u0219\u021a\7P\2\2\u021aX\3\2\2\2\u021b\u021c\7K\2\2\u021c\u021d"+ + "\7P\2\2\u021d\u021e\7P\2\2\u021e\u021f\7G\2\2\u021f\u0220\7T\2\2\u0220"+ + "Z\3\2\2\2\u0221\u0222\7K\2\2\u0222\u0223\7P\2\2\u0223\u0224\7V\2\2\u0224"+ + "\u0225\7G\2\2\u0225\u0226\7T\2\2\u0226\u0227\7X\2\2\u0227\u0228\7C\2\2"+ + "\u0228\u0229\7N\2\2\u0229\\\3\2\2\2\u022a\u022b\7K\2\2\u022b\u022c\7U"+ + "\2\2\u022c^\3\2\2\2\u022d\u022e\7L\2\2\u022e\u022f\7Q\2\2\u022f\u0230"+ + "\7K\2\2\u0230\u0231\7P\2\2\u0231`\3\2\2\2\u0232\u0233\7N\2\2\u0233\u0234"+ + "\7C\2\2\u0234\u0235\7U\2\2\u0235\u0236\7V\2\2\u0236b\3\2\2\2\u0237\u0238"+ + "\7N\2\2\u0238\u0239\7G\2\2\u0239\u023a\7H\2\2\u023a\u023b\7V\2\2\u023b"+ + "d\3\2\2\2\u023c\u023d\7N\2\2\u023d\u023e\7K\2\2\u023e\u023f\7M\2\2\u023f"+ + "\u0240\7G\2\2\u0240f\3\2\2\2\u0241\u0242\7N\2\2\u0242\u0243\7K\2\2\u0243"+ + "\u0244\7O\2\2\u0244\u0245\7K\2\2\u0245\u0246\7V\2\2\u0246h\3\2\2\2\u0247"+ + "\u0248\7O\2\2\u0248\u0249\7C\2\2\u0249\u024a\7R\2\2\u024a\u024b\7R\2\2"+ + "\u024b\u024c\7G\2\2\u024c\u024d\7F\2\2\u024dj\3\2\2\2\u024e\u024f\7O\2"+ + "\2\u024f\u0250\7C\2\2\u0250\u0251\7V\2\2\u0251\u0252\7E\2\2\u0252\u0253"+ + "\7J\2\2\u0253l\3\2\2\2\u0254\u0255\7O\2\2\u0255\u0256\7K\2\2\u0256\u0257"+ + "\7P\2\2\u0257\u0258\7W\2\2\u0258\u0259\7V\2\2\u0259\u025a\7G\2\2\u025a"+ + "n\3\2\2\2\u025b\u025c\7O\2\2\u025c\u025d\7K\2\2\u025d\u025e\7P\2\2\u025e"+ + "\u025f\7W\2\2\u025f\u0260\7V\2\2\u0260\u0261\7G\2\2\u0261\u0262\7U\2\2"+ + "\u0262p\3\2\2\2\u0263\u0264\7O\2\2\u0264\u0265\7Q\2\2\u0265\u0266\7P\2"+ + "\2\u0266\u0267\7V\2\2\u0267\u0268\7J\2\2\u0268r\3\2\2\2\u0269\u026a\7"+ + "O\2\2\u026a\u026b\7Q\2\2\u026b\u026c\7P\2\2\u026c\u026d\7V\2\2\u026d\u026e"+ + "\7J\2\2\u026e\u026f\7U\2\2\u026ft\3\2\2\2\u0270\u0271\7P\2\2\u0271\u0272"+ + "\7C\2\2\u0272\u0273\7V\2\2\u0273\u0274\7W\2\2\u0274\u0275\7T\2\2\u0275"+ + "\u0276\7C\2\2\u0276\u0277\7N\2\2\u0277v\3\2\2\2\u0278\u0279\7P\2\2\u0279"+ + "\u027a\7Q\2\2\u027a\u027b\7V\2\2\u027bx\3\2\2\2\u027c\u027d\7P\2\2\u027d"+ + "\u027e\7W\2\2\u027e\u027f\7N\2\2\u027f\u0280\7N\2\2\u0280z\3\2\2\2\u0281"+ + "\u0282\7P\2\2\u0282\u0283\7W\2\2\u0283\u0284\7N\2\2\u0284\u0285\7N\2\2"+ + "\u0285\u0286\7U\2\2\u0286|\3\2\2\2\u0287\u0288\7Q\2\2\u0288\u0289\7P\2"+ + "\2\u0289~\3\2\2\2\u028a\u028b\7Q\2\2\u028b\u028c\7R\2\2\u028c\u028d\7"+ + "V\2\2\u028d\u028e\7K\2\2\u028e\u028f\7O\2\2\u028f\u0290\7K\2\2\u0290\u0291"+ + "\7\\\2\2\u0291\u0292\7G\2\2\u0292\u0293\7F\2\2\u0293\u0080\3\2\2\2\u0294"+ + "\u0295\7Q\2\2\u0295\u0296\7T\2\2\u0296\u0082\3\2\2\2\u0297\u0298\7Q\2"+ + "\2\u0298\u0299\7T\2\2\u0299\u029a\7F\2\2\u029a\u029b\7G\2\2\u029b\u029c"+ + "\7T\2\2\u029c\u0084\3\2\2\2\u029d\u029e\7Q\2\2\u029e\u029f\7W\2\2\u029f"+ + "\u02a0\7V\2\2\u02a0\u02a1\7G\2\2\u02a1\u02a2\7T\2\2\u02a2\u0086\3\2\2"+ + "\2\u02a3\u02a4\7R\2\2\u02a4\u02a5\7C\2\2\u02a5\u02a6\7T\2\2\u02a6\u02a7"+ + "\7U\2\2\u02a7\u02a8\7G\2\2\u02a8\u02a9\7F\2\2\u02a9\u0088\3\2\2\2\u02aa"+ + "\u02ab\7R\2\2\u02ab\u02ac\7J\2\2\u02ac\u02ad\7[\2\2\u02ad\u02ae\7U\2\2"+ + "\u02ae\u02af\7K\2\2\u02af\u02b0\7E\2\2\u02b0\u02b1\7C\2\2\u02b1\u02b2"+ + "\7N\2\2\u02b2\u008a\3\2\2\2\u02b3\u02b4\7R\2\2\u02b4\u02b5\7N\2\2\u02b5"+ + "\u02b6\7C\2\2\u02b6\u02b7\7P\2\2\u02b7\u008c\3\2\2\2\u02b8\u02b9\7T\2"+ + "\2\u02b9\u02ba\7K\2\2\u02ba\u02bb\7I\2\2\u02bb\u02bc\7J\2\2\u02bc\u02bd"+ + "\7V\2\2\u02bd\u008e\3\2\2\2\u02be\u02bf\7T\2\2\u02bf\u02c0\7N\2\2\u02c0"+ + "\u02c1\7K\2\2\u02c1\u02c2\7M\2\2\u02c2\u02c3\7G\2\2\u02c3\u0090\3\2\2"+ + "\2\u02c4\u02c5\7S\2\2\u02c5\u02c6\7W\2\2\u02c6\u02c7\7G\2\2\u02c7\u02c8"+ + "\7T\2\2\u02c8\u02c9\7[\2\2\u02c9\u0092\3\2\2\2\u02ca\u02cb\7U\2\2\u02cb"+ + "\u02cc\7E\2\2\u02cc\u02cd\7J\2\2\u02cd\u02ce\7G\2\2\u02ce\u02cf\7O\2\2"+ + "\u02cf\u02d0\7C\2\2\u02d0\u02d1\7U\2\2\u02d1\u0094\3\2\2\2\u02d2\u02d3"+ + "\7U\2\2\u02d3\u02d4\7G\2\2\u02d4\u02d5\7E\2\2\u02d5\u02d6\7Q\2\2\u02d6"+ + "\u02d7\7P\2\2\u02d7\u02d8\7F\2\2\u02d8\u0096\3\2\2\2\u02d9\u02da\7U\2"+ + "\2\u02da\u02db\7G\2\2\u02db\u02dc\7E\2\2\u02dc\u02dd\7Q\2\2\u02dd\u02de"+ + "\7P\2\2\u02de\u02df\7F\2\2\u02df\u02e0\7U\2\2\u02e0\u0098\3\2\2\2\u02e1"+ + "\u02e2\7U\2\2\u02e2\u02e3\7G\2\2\u02e3\u02e4\7N\2\2\u02e4\u02e5\7G\2\2"+ + "\u02e5\u02e6\7E\2\2\u02e6\u02e7\7V\2\2\u02e7\u009a\3\2\2\2\u02e8\u02e9"+ + "\7U\2\2\u02e9\u02ea\7J\2\2\u02ea\u02eb\7Q\2\2\u02eb\u02ec\7Y\2\2\u02ec"+ + "\u009c\3\2\2\2\u02ed\u02ee\7U\2\2\u02ee\u02ef\7[\2\2\u02ef\u02f0\7U\2"+ + "\2\u02f0\u009e\3\2\2\2\u02f1\u02f2\7V\2\2\u02f2\u02f3\7C\2\2\u02f3\u02f4"+ + "\7D\2\2\u02f4\u02f5\7N\2\2\u02f5\u02f6\7G\2\2\u02f6\u00a0\3\2\2\2\u02f7"+ + "\u02f8\7V\2\2\u02f8\u02f9\7C\2\2\u02f9\u02fa\7D\2\2\u02fa\u02fb\7N\2\2"+ + "\u02fb\u02fc\7G\2\2\u02fc\u02fd\7U\2\2\u02fd\u00a2\3\2\2\2\u02fe\u02ff"+ + "\7V\2\2\u02ff\u0300\7G\2\2\u0300\u0301\7Z\2\2\u0301\u0302\7V\2\2\u0302"+ + "\u00a4\3\2\2\2\u0303\u0304\7V\2\2\u0304\u0305\7T\2\2\u0305\u0306\7W\2"+ + "\2\u0306\u0307\7G\2\2\u0307\u00a6\3\2\2\2\u0308\u0309\7V\2\2\u0309\u030a"+ + "\7Q\2\2\u030a\u00a8\3\2\2\2\u030b\u030c\7V\2\2\u030c\u030d\7[\2\2\u030d"+ + "\u030e\7R\2\2\u030e\u030f\7G\2\2\u030f\u00aa\3\2\2\2\u0310\u0311\7V\2"+ + "\2\u0311\u0312\7[\2\2\u0312\u0313\7R\2\2\u0313\u0314\7G\2\2\u0314\u0315"+ + "\7U\2\2\u0315\u00ac\3\2\2\2\u0316\u0317\7W\2\2\u0317\u0318\7U\2\2\u0318"+ + "\u0319\7K\2\2\u0319\u031a\7P\2\2\u031a\u031b\7I\2\2\u031b\u00ae\3\2\2"+ + "\2\u031c\u031d\7X\2\2\u031d\u031e\7G\2\2\u031e\u031f\7T\2\2\u031f\u0320"+ + "\7K\2\2\u0320\u0321\7H\2\2\u0321\u0322\7[\2\2\u0322\u00b0\3\2\2\2\u0323"+ + "\u0324\7Y\2\2\u0324\u0325\7J\2\2\u0325\u0326\7G\2\2\u0326\u0327\7T\2\2"+ + "\u0327\u0328\7G\2\2\u0328\u00b2\3\2\2\2\u0329\u032a\7Y\2\2\u032a\u032b"+ + "\7K\2\2\u032b\u032c\7V\2\2\u032c\u032d\7J\2\2\u032d\u00b4\3\2\2\2\u032e"+ + "\u032f\7[\2\2\u032f\u0330\7G\2\2\u0330\u0331\7C\2\2\u0331\u0332\7T\2\2"+ + "\u0332\u00b6\3\2\2\2\u0333\u0334\7[\2\2\u0334\u0335\7G\2\2\u0335\u0336"+ + "\7C\2\2\u0336\u0337\7T\2\2\u0337\u0338\7U\2\2\u0338\u00b8\3\2\2\2\u0339"+ + "\u033a\7}\2\2\u033a\u033b\7G\2\2\u033b\u033c\7U\2\2\u033c\u033d\7E\2\2"+ + "\u033d\u033e\7C\2\2\u033e\u033f\7R\2\2\u033f\u0340\7G\2\2\u0340\u00ba"+ + "\3\2\2\2\u0341\u0342\7}\2\2\u0342\u0343\7H\2\2\u0343\u0344\7P\2\2\u0344"+ + "\u00bc\3\2\2\2\u0345\u0346\7}\2\2\u0346\u0347\7N\2\2\u0347\u0348\7K\2"+ + "\2\u0348\u0349\7O\2\2\u0349\u034a\7K\2\2\u034a\u034b\7V\2\2\u034b\u00be"+ + "\3\2\2\2\u034c\u034d\7}\2\2\u034d\u034e\7F\2\2\u034e\u00c0\3\2\2\2\u034f"+ + "\u0350\7}\2\2\u0350\u0351\7V\2\2\u0351\u00c2\3\2\2\2\u0352\u0353\7}\2"+ + "\2\u0353\u0354\7V\2\2\u0354\u0355\7U\2\2\u0355\u00c4\3\2\2\2\u0356\u0357"+ + "\7}\2\2\u0357\u0358\7I\2\2\u0358\u0359\7W\2\2\u0359\u035a\7K\2\2\u035a"+ + "\u035b\7F\2\2\u035b\u00c6\3\2\2\2\u035c\u035d\7\177\2\2\u035d\u00c8\3"+ + "\2\2\2\u035e\u035f\7?\2\2\u035f\u00ca\3\2\2\2\u0360\u0361\7>\2\2\u0361"+ + "\u0362\7?\2\2\u0362\u0363\7@\2\2\u0363\u00cc\3\2\2\2\u0364\u0365\7>\2"+ + "\2\u0365\u0369\7@\2\2\u0366\u0367\7#\2\2\u0367\u0369\7?\2\2\u0368\u0364"+ + "\3\2\2\2\u0368\u0366\3\2\2\2\u0369\u00ce\3\2\2\2\u036a\u036b\7>\2\2\u036b"+ + "\u00d0\3\2\2\2\u036c\u036d\7>\2\2\u036d\u036e\7?\2\2\u036e\u00d2\3\2\2"+ + "\2\u036f\u0370\7@\2\2\u0370\u00d4\3\2\2\2\u0371\u0372\7@\2\2\u0372\u0373"+ + "\7?\2\2\u0373\u00d6\3\2\2\2\u0374\u0375\7-\2\2\u0375\u00d8\3\2\2\2\u0376"+ + "\u0377\7/\2\2\u0377\u00da\3\2\2\2\u0378\u0379\7,\2\2\u0379\u00dc\3\2\2"+ + "\2\u037a\u037b\7\61\2\2\u037b\u00de\3\2\2\2\u037c\u037d\7\'\2\2\u037d"+ + "\u00e0\3\2\2\2\u037e\u037f\7~\2\2\u037f\u0380\7~\2\2\u0380\u00e2\3\2\2"+ + "\2\u0381\u0382\7\60\2\2\u0382\u00e4\3\2\2\2\u0383\u0384\7A\2\2\u0384\u00e6"+ + "\3\2\2\2\u0385\u038b\7)\2\2\u0386\u038a\n\2\2\2\u0387\u0388\7)\2\2\u0388"+ + "\u038a\7)\2\2\u0389\u0386\3\2\2\2\u0389\u0387\3\2\2\2\u038a\u038d\3\2"+ + "\2\2\u038b\u0389\3\2\2\2\u038b\u038c\3\2\2\2\u038c\u038e\3\2\2\2\u038d"+ + "\u038b\3\2\2\2\u038e\u038f\7)\2\2\u038f\u00e8\3\2\2\2\u0390\u0392\5\u00f9"+ "}\2\u0391\u0390\3\2\2\2\u0392\u0393\3\2\2\2\u0393\u0391\3\2\2\2\u0393"+ - "\u0394\3\2\2\2\u0394\u0395\3\2\2\2\u0395\u0399\5\u00e3r\2\u0396\u0398"+ - "\5\u00f9}\2\u0397\u0396\3\2\2\2\u0398\u039b\3\2\2\2\u0399\u0397\3\2\2"+ - "\2\u0399\u039a\3\2\2\2\u039a\u03bb\3\2\2\2\u039b\u0399\3\2\2\2\u039c\u039e"+ - "\5\u00e3r\2\u039d\u039f\5\u00f9}\2\u039e\u039d\3\2\2\2\u039f\u03a0\3\2"+ - "\2\2\u03a0\u039e\3\2\2\2\u03a0\u03a1\3\2\2\2\u03a1\u03bb\3\2\2\2\u03a2"+ - "\u03a4\5\u00f9}\2\u03a3\u03a2\3\2\2\2\u03a4\u03a5\3\2\2\2\u03a5\u03a3"+ - "\3\2\2\2\u03a5\u03a6\3\2\2\2\u03a6\u03ae\3\2\2\2\u03a7\u03ab\5\u00e3r"+ - "\2\u03a8\u03aa\5\u00f9}\2\u03a9\u03a8\3\2\2\2\u03aa\u03ad\3\2\2\2\u03ab"+ - "\u03a9\3\2\2\2\u03ab\u03ac\3\2\2\2\u03ac\u03af\3\2\2\2\u03ad\u03ab\3\2"+ - "\2\2\u03ae\u03a7\3\2\2\2\u03ae\u03af\3\2\2\2\u03af\u03b0\3\2\2\2\u03b0"+ - "\u03b1\5\u00f7|\2\u03b1\u03bb\3\2\2\2\u03b2\u03b4\5\u00e3r\2\u03b3\u03b5"+ - "\5\u00f9}\2\u03b4\u03b3\3\2\2\2\u03b5\u03b6\3\2\2\2\u03b6\u03b4\3\2\2"+ - "\2\u03b6\u03b7\3\2\2\2\u03b7\u03b8\3\2\2\2\u03b8\u03b9\5\u00f7|\2\u03b9"+ - "\u03bb\3\2\2\2\u03ba\u0391\3\2\2\2\u03ba\u039c\3\2\2\2\u03ba\u03a3\3\2"+ - "\2\2\u03ba\u03b2\3\2\2\2\u03bb\u00ec\3\2\2\2\u03bc\u03bf\5\u00fb~\2\u03bd"+ - "\u03bf\7a\2\2\u03be\u03bc\3\2\2\2\u03be\u03bd\3\2\2\2\u03bf\u03c5\3\2"+ - "\2\2\u03c0\u03c4\5\u00fb~\2\u03c1\u03c4\5\u00f9}\2\u03c2\u03c4\t\3\2\2"+ - "\u03c3\u03c0\3\2\2\2\u03c3\u03c1\3\2\2\2\u03c3\u03c2\3\2\2\2\u03c4\u03c7"+ - "\3\2\2\2\u03c5\u03c3\3\2\2\2\u03c5\u03c6\3\2\2\2\u03c6\u00ee\3\2\2\2\u03c7"+ - "\u03c5\3\2\2\2\u03c8\u03cc\5\u00f9}\2\u03c9\u03cd\5\u00fb~\2\u03ca\u03cd"+ - "\5\u00f9}\2\u03cb\u03cd\t\4\2\2\u03cc\u03c9\3\2\2\2\u03cc\u03ca\3\2\2"+ - "\2\u03cc\u03cb\3\2\2\2\u03cd\u03ce\3\2\2\2\u03ce\u03cc\3\2\2\2\u03ce\u03cf"+ - "\3\2\2\2\u03cf\u00f0\3\2\2\2\u03d0\u03d4\5\u00fb~\2\u03d1\u03d4\5\u00f9"+ - "}\2\u03d2\u03d4\7a\2\2\u03d3\u03d0\3\2\2\2\u03d3\u03d1\3\2\2\2\u03d3\u03d2"+ - "\3\2\2\2\u03d4\u03d5\3\2\2\2\u03d5\u03d3\3\2\2\2\u03d5\u03d6\3\2\2\2\u03d6"+ - "\u00f2\3\2\2\2\u03d7\u03dd\7$\2\2\u03d8\u03dc\n\5\2\2\u03d9\u03da\7$\2"+ - "\2\u03da\u03dc\7$\2\2\u03db\u03d8\3\2\2\2\u03db\u03d9\3\2\2\2\u03dc\u03df"+ - "\3\2\2\2\u03dd\u03db\3\2\2\2\u03dd\u03de\3\2\2\2\u03de\u03e0\3\2\2\2\u03df"+ - "\u03dd\3\2\2\2\u03e0\u03e1\7$\2\2\u03e1\u00f4\3\2\2\2\u03e2\u03e8\7b\2"+ - "\2\u03e3\u03e7\n\6\2\2\u03e4\u03e5\7b\2\2\u03e5\u03e7\7b\2\2\u03e6\u03e3"+ - "\3\2\2\2\u03e6\u03e4\3\2\2\2\u03e7\u03ea\3\2\2\2\u03e8\u03e6\3\2\2\2\u03e8"+ - "\u03e9\3\2\2\2\u03e9\u03eb\3\2\2\2\u03ea\u03e8\3\2\2\2\u03eb\u03ec\7b"+ - "\2\2\u03ec\u00f6\3\2\2\2\u03ed\u03ef\7G\2\2\u03ee\u03f0\t\7\2\2\u03ef"+ - "\u03ee\3\2\2\2\u03ef\u03f0\3\2\2\2\u03f0\u03f2\3\2\2\2\u03f1\u03f3\5\u00f9"+ - "}\2\u03f2\u03f1\3\2\2\2\u03f3\u03f4\3\2\2\2\u03f4\u03f2\3\2\2\2\u03f4"+ - "\u03f5\3\2\2\2\u03f5\u00f8\3\2\2\2\u03f6\u03f7\t\b\2\2\u03f7\u00fa\3\2"+ - "\2\2\u03f8\u03f9\t\t\2\2\u03f9\u00fc\3\2\2\2\u03fa\u03fb\7/\2\2\u03fb"+ - "\u03fc\7/\2\2\u03fc\u0400\3\2\2\2\u03fd\u03ff\n\n\2\2\u03fe\u03fd\3\2"+ - "\2\2\u03ff\u0402\3\2\2\2\u0400\u03fe\3\2\2\2\u0400\u0401\3\2\2\2\u0401"+ - "\u0404\3\2\2\2\u0402\u0400\3\2\2\2\u0403\u0405\7\17\2\2\u0404\u0403\3"+ - "\2\2\2\u0404\u0405\3\2\2\2\u0405\u0407\3\2\2\2\u0406\u0408\7\f\2\2\u0407"+ - "\u0406\3\2\2\2\u0407\u0408\3\2\2\2\u0408\u0409\3\2\2\2\u0409\u040a\b\177"+ - "\2\2\u040a\u00fe\3\2\2\2\u040b\u040c\7\61\2\2\u040c\u040d\7,\2\2\u040d"+ - "\u0412\3\2\2\2\u040e\u0411\5\u00ff\u0080\2\u040f\u0411\13\2\2\2\u0410"+ - "\u040e\3\2\2\2\u0410\u040f\3\2\2\2\u0411\u0414\3\2\2\2\u0412\u0413\3\2"+ - "\2\2\u0412\u0410\3\2\2\2\u0413\u0415\3\2\2\2\u0414\u0412\3\2\2\2\u0415"+ - "\u0416\7,\2\2\u0416\u0417\7\61\2\2\u0417\u0418\3\2\2\2\u0418\u0419\b\u0080"+ - "\2\2\u0419\u0100\3\2\2\2\u041a\u041c\t\13\2\2\u041b\u041a\3\2\2\2\u041c"+ - "\u041d\3\2\2\2\u041d\u041b\3\2\2\2\u041d\u041e\3\2\2\2\u041e\u041f\3\2"+ - "\2\2\u041f\u0420\b\u0081\2\2\u0420\u0102\3\2\2\2\u0421\u0422\13\2\2\2"+ - "\u0422\u0104\3\2\2\2\"\2\u0363\u0384\u0386\u038e\u0393\u0399\u03a0\u03a5"+ - "\u03ab\u03ae\u03b6\u03ba\u03be\u03c3\u03c5\u03cc\u03ce\u03d3\u03d5\u03db"+ - "\u03dd\u03e6\u03e8\u03ef\u03f4\u0400\u0404\u0407\u0410\u0412\u041d\3\2"+ - "\3\2"; + "\u0394\3\2\2\2\u0394\u00ea\3\2\2\2\u0395\u0397\5\u00f9}\2\u0396\u0395"+ + "\3\2\2\2\u0397\u0398\3\2\2\2\u0398\u0396\3\2\2\2\u0398\u0399\3\2\2\2\u0399"+ + "\u039a\3\2\2\2\u039a\u039e\5\u00e3r\2\u039b\u039d\5\u00f9}\2\u039c\u039b"+ + "\3\2\2\2\u039d\u03a0\3\2\2\2\u039e\u039c\3\2\2\2\u039e\u039f\3\2\2\2\u039f"+ + "\u03c0\3\2\2\2\u03a0\u039e\3\2\2\2\u03a1\u03a3\5\u00e3r\2\u03a2\u03a4"+ + "\5\u00f9}\2\u03a3\u03a2\3\2\2\2\u03a4\u03a5\3\2\2\2\u03a5\u03a3\3\2\2"+ + "\2\u03a5\u03a6\3\2\2\2\u03a6\u03c0\3\2\2\2\u03a7\u03a9\5\u00f9}\2\u03a8"+ + "\u03a7\3\2\2\2\u03a9\u03aa\3\2\2\2\u03aa\u03a8\3\2\2\2\u03aa\u03ab\3\2"+ + "\2\2\u03ab\u03b3\3\2\2\2\u03ac\u03b0\5\u00e3r\2\u03ad\u03af\5\u00f9}\2"+ + "\u03ae\u03ad\3\2\2\2\u03af\u03b2\3\2\2\2\u03b0\u03ae\3\2\2\2\u03b0\u03b1"+ + "\3\2\2\2\u03b1\u03b4\3\2\2\2\u03b2\u03b0\3\2\2\2\u03b3\u03ac\3\2\2\2\u03b3"+ + "\u03b4\3\2\2\2\u03b4\u03b5\3\2\2\2\u03b5\u03b6\5\u00f7|\2\u03b6\u03c0"+ + "\3\2\2\2\u03b7\u03b9\5\u00e3r\2\u03b8\u03ba\5\u00f9}\2\u03b9\u03b8\3\2"+ + "\2\2\u03ba\u03bb\3\2\2\2\u03bb\u03b9\3\2\2\2\u03bb\u03bc\3\2\2\2\u03bc"+ + "\u03bd\3\2\2\2\u03bd\u03be\5\u00f7|\2\u03be\u03c0\3\2\2\2\u03bf\u0396"+ + "\3\2\2\2\u03bf\u03a1\3\2\2\2\u03bf\u03a8\3\2\2\2\u03bf\u03b7\3\2\2\2\u03c0"+ + "\u00ec\3\2\2\2\u03c1\u03c4\5\u00fb~\2\u03c2\u03c4\7a\2\2\u03c3\u03c1\3"+ + "\2\2\2\u03c3\u03c2\3\2\2\2\u03c4\u03ca\3\2\2\2\u03c5\u03c9\5\u00fb~\2"+ + "\u03c6\u03c9\5\u00f9}\2\u03c7\u03c9\t\3\2\2\u03c8\u03c5\3\2\2\2\u03c8"+ + "\u03c6\3\2\2\2\u03c8\u03c7\3\2\2\2\u03c9\u03cc\3\2\2\2\u03ca\u03c8\3\2"+ + "\2\2\u03ca\u03cb\3\2\2\2\u03cb\u00ee\3\2\2\2\u03cc\u03ca\3\2\2\2\u03cd"+ + "\u03d1\5\u00f9}\2\u03ce\u03d2\5\u00fb~\2\u03cf\u03d2\5\u00f9}\2\u03d0"+ + "\u03d2\t\4\2\2\u03d1\u03ce\3\2\2\2\u03d1\u03cf\3\2\2\2\u03d1\u03d0\3\2"+ + "\2\2\u03d2\u03d3\3\2\2\2\u03d3\u03d1\3\2\2\2\u03d3\u03d4\3\2\2\2\u03d4"+ + "\u00f0\3\2\2\2\u03d5\u03d9\5\u00fb~\2\u03d6\u03d9\5\u00f9}\2\u03d7\u03d9"+ + "\7a\2\2\u03d8\u03d5\3\2\2\2\u03d8\u03d6\3\2\2\2\u03d8\u03d7\3\2\2\2\u03d9"+ + "\u03da\3\2\2\2\u03da\u03d8\3\2\2\2\u03da\u03db\3\2\2\2\u03db\u00f2\3\2"+ + "\2\2\u03dc\u03e2\7$\2\2\u03dd\u03e1\n\5\2\2\u03de\u03df\7$\2\2\u03df\u03e1"+ + "\7$\2\2\u03e0\u03dd\3\2\2\2\u03e0\u03de\3\2\2\2\u03e1\u03e4\3\2\2\2\u03e2"+ + "\u03e0\3\2\2\2\u03e2\u03e3\3\2\2\2\u03e3\u03e5\3\2\2\2\u03e4\u03e2\3\2"+ + "\2\2\u03e5\u03e6\7$\2\2\u03e6\u00f4\3\2\2\2\u03e7\u03ed\7b\2\2\u03e8\u03ec"+ + "\n\6\2\2\u03e9\u03ea\7b\2\2\u03ea\u03ec\7b\2\2\u03eb\u03e8\3\2\2\2\u03eb"+ + "\u03e9\3\2\2\2\u03ec\u03ef\3\2\2\2\u03ed\u03eb\3\2\2\2\u03ed\u03ee\3\2"+ + "\2\2\u03ee\u03f0\3\2\2\2\u03ef\u03ed\3\2\2\2\u03f0\u03f1\7b\2\2\u03f1"+ + "\u00f6\3\2\2\2\u03f2\u03f4\7G\2\2\u03f3\u03f5\t\7\2\2\u03f4\u03f3\3\2"+ + "\2\2\u03f4\u03f5\3\2\2\2\u03f5\u03f7\3\2\2\2\u03f6\u03f8\5\u00f9}\2\u03f7"+ + "\u03f6\3\2\2\2\u03f8\u03f9\3\2\2\2\u03f9\u03f7\3\2\2\2\u03f9\u03fa\3\2"+ + "\2\2\u03fa\u00f8\3\2\2\2\u03fb\u03fc\t\b\2\2\u03fc\u00fa\3\2\2\2\u03fd"+ + "\u03fe\t\t\2\2\u03fe\u00fc\3\2\2\2\u03ff\u0400\7/\2\2\u0400\u0401\7/\2"+ + "\2\u0401\u0405\3\2\2\2\u0402\u0404\n\n\2\2\u0403\u0402\3\2\2\2\u0404\u0407"+ + "\3\2\2\2\u0405\u0403\3\2\2\2\u0405\u0406\3\2\2\2\u0406\u0409\3\2\2\2\u0407"+ + "\u0405\3\2\2\2\u0408\u040a\7\17\2\2\u0409\u0408\3\2\2\2\u0409\u040a\3"+ + "\2\2\2\u040a\u040c\3\2\2\2\u040b\u040d\7\f\2\2\u040c\u040b\3\2\2\2\u040c"+ + "\u040d\3\2\2\2\u040d\u040e\3\2\2\2\u040e\u040f\b\177\2\2\u040f\u00fe\3"+ + "\2\2\2\u0410\u0411\7\61\2\2\u0411\u0412\7,\2\2\u0412\u0417\3\2\2\2\u0413"+ + "\u0416\5\u00ff\u0080\2\u0414\u0416\13\2\2\2\u0415\u0413\3\2\2\2\u0415"+ + "\u0414\3\2\2\2\u0416\u0419\3\2\2\2\u0417\u0418\3\2\2\2\u0417\u0415\3\2"+ + "\2\2\u0418\u041a\3\2\2\2\u0419\u0417\3\2\2\2\u041a\u041b\7,\2\2\u041b"+ + "\u041c\7\61\2\2\u041c\u041d\3\2\2\2\u041d\u041e\b\u0080\2\2\u041e\u0100"+ + "\3\2\2\2\u041f\u0421\t\13\2\2\u0420\u041f\3\2\2\2\u0421\u0422\3\2\2\2"+ + "\u0422\u0420\3\2\2\2\u0422\u0423\3\2\2\2\u0423\u0424\3\2\2\2\u0424\u0425"+ + "\b\u0081\2\2\u0425\u0102\3\2\2\2\u0426\u0427\13\2\2\2\u0427\u0104\3\2"+ + "\2\2\"\2\u0368\u0389\u038b\u0393\u0398\u039e\u03a5\u03aa\u03b0\u03b3\u03bb"+ + "\u03bf\u03c3\u03c8\u03ca\u03d1\u03d3\u03d8\u03da\u03e0\u03e2\u03eb\u03ed"+ + "\u03f4\u03f9\u0405\u0409\u040c\u0415\u0417\u0422\3\2\3\2"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java index 7549bfab8320a..a690169409e81 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java @@ -19,7 +19,7 @@ class SqlBaseParser extends Parser { public static final int T__0=1, T__1=2, T__2=3, T__3=4, ALL=5, ANALYZE=6, ANALYZED=7, AND=8, ANY=9, AS=10, ASC=11, BETWEEN=12, BY=13, CAST=14, CATALOG=15, CATALOGS=16, COLUMNS=17, - CONVERT=18, CURRENT=19, CURRENT_TIMESTAMP=20, DAY=21, DAYS=22, DEBUG=23, + CONVERT=18, CURRENT_DATE=19, CURRENT_TIMESTAMP=20, DAY=21, DAYS=22, DEBUG=23, DESC=24, DESCRIBE=25, DISTINCT=26, ESCAPE=27, EXECUTABLE=28, EXISTS=29, EXPLAIN=30, EXTRACT=31, FALSE=32, FIRST=33, FORMAT=34, FROM=35, FULL=36, FUNCTIONS=37, GRAPHVIZ=38, GROUP=39, HAVING=40, HOUR=41, HOURS=42, IN=43, @@ -71,7 +71,7 @@ class SqlBaseParser extends Parser { private static final String[] _LITERAL_NAMES = { null, "'('", "')'", "','", "':'", "'ALL'", "'ANALYZE'", "'ANALYZED'", "'AND'", "'ANY'", "'AS'", "'ASC'", "'BETWEEN'", "'BY'", "'CAST'", "'CATALOG'", - "'CATALOGS'", "'COLUMNS'", "'CONVERT'", "'CURRENT'", "'CURRENT_TIMESTAMP'", + "'CATALOGS'", "'COLUMNS'", "'CONVERT'", "'CURRENT_DATE'", "'CURRENT_TIMESTAMP'", "'DAY'", "'DAYS'", "'DEBUG'", "'DESC'", "'DESCRIBE'", "'DISTINCT'", "'ESCAPE'", "'EXECUTABLE'", "'EXISTS'", "'EXPLAIN'", "'EXTRACT'", "'FALSE'", "'FIRST'", "'FORMAT'", "'FROM'", "'FULL'", "'FUNCTIONS'", "'GRAPHVIZ'", "'GROUP'", @@ -90,16 +90,16 @@ class SqlBaseParser extends Parser { private static final String[] _SYMBOLIC_NAMES = { null, null, null, null, null, "ALL", "ANALYZE", "ANALYZED", "AND", "ANY", "AS", "ASC", "BETWEEN", "BY", "CAST", "CATALOG", "CATALOGS", "COLUMNS", - "CONVERT", "CURRENT", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", "DESC", - "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", "EXTRACT", - "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", "GROUP", - "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", "LAST", - "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", "MONTH", - "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", "OR", - "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", - "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", - "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", - "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", + "CONVERT", "CURRENT_DATE", "CURRENT_TIMESTAMP", "DAY", "DAYS", "DEBUG", + "DESC", "DESCRIBE", "DISTINCT", "ESCAPE", "EXECUTABLE", "EXISTS", "EXPLAIN", + "EXTRACT", "FALSE", "FIRST", "FORMAT", "FROM", "FULL", "FUNCTIONS", "GRAPHVIZ", + "GROUP", "HAVING", "HOUR", "HOURS", "IN", "INNER", "INTERVAL", "IS", "JOIN", + "LAST", "LEFT", "LIKE", "LIMIT", "MAPPED", "MATCH", "MINUTE", "MINUTES", + "MONTH", "MONTHS", "NATURAL", "NOT", "NULL", "NULLS", "ON", "OPTIMIZED", + "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", + "QUERY", "SCHEMAS", "SECOND", "SECONDS", "SELECT", "SHOW", "SYS", "TABLE", + "TABLES", "TEXT", "TRUE", "TO", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", + "WITH", "YEAR", "YEARS", "ESCAPE_ESC", "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", "GUID_ESC", "ESC_END", "EQ", "NULLEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", @@ -767,7 +767,6 @@ public final StatementContext statement() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -844,7 +843,6 @@ public final StatementContext statement() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -915,7 +913,6 @@ public final StatementContext statement() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -1089,7 +1086,6 @@ public final StatementContext statement() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -2001,7 +1997,7 @@ public final GroupingExpressionsContext groupingExpressions() throws Recognition match(T__0); setState(327); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT) | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RIGHT - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TRUE - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (FUNCTION_ESC - 67)) | (1L << (DATE_ESC - 67)) | (1L << (TIME_ESC - 67)) | (1L << (TIMESTAMP_ESC - 67)) | (1L << (GUID_ESC - 67)) | (1L << (PLUS - 67)) | (1L << (MINUS - 67)) | (1L << (ASTERISK - 67)) | (1L << (PARAM - 67)) | (1L << (STRING - 67)) | (1L << (INTEGER_VALUE - 67)) | (1L << (DECIMAL_VALUE - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT_DATE) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT) | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RIGHT - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TRUE - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (FUNCTION_ESC - 67)) | (1L << (DATE_ESC - 67)) | (1L << (TIME_ESC - 67)) | (1L << (TIMESTAMP_ESC - 67)) | (1L << (GUID_ESC - 67)) | (1L << (PLUS - 67)) | (1L << (MINUS - 67)) | (1L << (ASTERISK - 67)) | (1L << (PARAM - 67)) | (1L << (STRING - 67)) | (1L << (INTEGER_VALUE - 67)) | (1L << (DECIMAL_VALUE - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { { setState(319); expression(); @@ -3805,7 +3801,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti case CATALOGS: case COLUMNS: case CONVERT: - case CURRENT: + case CURRENT_DATE: case CURRENT_TIMESTAMP: case DAY: case DEBUG: @@ -4203,7 +4199,7 @@ public final PrimaryExpressionContext primaryExpression() throws RecognitionExce { setState(576); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CURRENT) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { { setState(573); qualifiedName(); @@ -4415,6 +4411,7 @@ public final CastTemplateContext castTemplate() throws RecognitionException { public static class BuiltinDateTimeFunctionContext extends ParserRuleContext { public Token name; public Token precision; + public TerminalNode CURRENT_DATE() { return getToken(SqlBaseParser.CURRENT_DATE, 0); } public TerminalNode CURRENT_TIMESTAMP() { return getToken(SqlBaseParser.CURRENT_TIMESTAMP, 0); } public TerminalNode INTEGER_VALUE() { return getToken(SqlBaseParser.INTEGER_VALUE, 0); } public BuiltinDateTimeFunctionContext(ParserRuleContext parent, int invokingState) { @@ -4441,31 +4438,57 @@ public final BuiltinDateTimeFunctionContext builtinDateTimeFunction() throws Rec enterRule(_localctx, 66, RULE_builtinDateTimeFunction); int _la; try { - enterOuterAlt(_localctx, 1); - { - setState(610); - ((BuiltinDateTimeFunctionContext)_localctx).name = match(CURRENT_TIMESTAMP); - setState(616); - _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,82,_ctx) ) { - case 1: + setState(623); + switch (_input.LA(1)) { + case CURRENT_DATE: + enterOuterAlt(_localctx, 1); { - setState(611); - match(T__0); + setState(610); + ((BuiltinDateTimeFunctionContext)_localctx).name = match(CURRENT_DATE); setState(613); - _la = _input.LA(1); - if (_la==INTEGER_VALUE) { + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,81,_ctx) ) { + case 1: { + setState(611); + match(T__0); setState(612); - ((BuiltinDateTimeFunctionContext)_localctx).precision = match(INTEGER_VALUE); + match(T__1); } + break; } - + } + break; + case CURRENT_TIMESTAMP: + enterOuterAlt(_localctx, 2); + { setState(615); - match(T__1); + ((BuiltinDateTimeFunctionContext)_localctx).name = match(CURRENT_TIMESTAMP); + setState(621); + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,83,_ctx) ) { + case 1: + { + setState(616); + match(T__0); + setState(618); + _la = _input.LA(1); + if (_la==INTEGER_VALUE) { + { + setState(617); + ((BuiltinDateTimeFunctionContext)_localctx).precision = match(INTEGER_VALUE); + } + } + + setState(620); + match(T__1); + } + break; + } } break; - } + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -4512,17 +4535,17 @@ public final ConvertTemplateContext convertTemplate() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(618); + setState(625); match(CONVERT); - setState(619); + setState(626); match(T__0); - setState(620); + setState(627); expression(); - setState(621); + setState(628); match(T__2); - setState(622); + setState(629); dataType(); - setState(623); + setState(630); match(T__1); } } @@ -4566,23 +4589,23 @@ public final ExtractExpressionContext extractExpression() throws RecognitionExce ExtractExpressionContext _localctx = new ExtractExpressionContext(_ctx, getState()); enterRule(_localctx, 70, RULE_extractExpression); try { - setState(630); + setState(637); switch (_input.LA(1)) { case EXTRACT: enterOuterAlt(_localctx, 1); { - setState(625); + setState(632); extractTemplate(); } break; case FUNCTION_ESC: enterOuterAlt(_localctx, 2); { - setState(626); + setState(633); match(FUNCTION_ESC); - setState(627); + setState(634); extractTemplate(); - setState(628); + setState(635); match(ESC_END); } break; @@ -4636,17 +4659,17 @@ public final ExtractTemplateContext extractTemplate() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(632); + setState(639); match(EXTRACT); - setState(633); + setState(640); match(T__0); - setState(634); + setState(641); ((ExtractTemplateContext)_localctx).field = identifier(); - setState(635); + setState(642); match(FROM); - setState(636); + setState(643); valueExpression(0); - setState(637); + setState(644); match(T__1); } } @@ -4689,13 +4712,12 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx FunctionExpressionContext _localctx = new FunctionExpressionContext(_ctx, getState()); enterRule(_localctx, 74, RULE_functionExpression); try { - setState(644); + setState(651); switch (_input.LA(1)) { case ANALYZE: case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -4736,18 +4758,18 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case BACKQUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(639); + setState(646); functionTemplate(); } break; case FUNCTION_ESC: enterOuterAlt(_localctx, 2); { - setState(640); + setState(647); match(FUNCTION_ESC); - setState(641); + setState(648); functionTemplate(); - setState(642); + setState(649); match(ESC_END); } break; @@ -4805,45 +4827,45 @@ public final FunctionTemplateContext functionTemplate() throws RecognitionExcept try { enterOuterAlt(_localctx, 1); { - setState(646); + setState(653); functionName(); - setState(647); + setState(654); match(T__0); - setState(659); + setState(666); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT) | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RIGHT - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TRUE - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (FUNCTION_ESC - 67)) | (1L << (DATE_ESC - 67)) | (1L << (TIME_ESC - 67)) | (1L << (TIMESTAMP_ESC - 67)) | (1L << (GUID_ESC - 67)) | (1L << (PLUS - 67)) | (1L << (MINUS - 67)) | (1L << (ASTERISK - 67)) | (1L << (PARAM - 67)) | (1L << (STRING - 67)) | (1L << (INTEGER_VALUE - 67)) | (1L << (DECIMAL_VALUE - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CONVERT) | (1L << CURRENT_DATE) | (1L << CURRENT_TIMESTAMP) | (1L << DAY) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LEFT) | (1L << LIMIT) | (1L << MAPPED) | (1L << MATCH) | (1L << MINUTE) | (1L << MONTH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RIGHT - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TRUE - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (FUNCTION_ESC - 67)) | (1L << (DATE_ESC - 67)) | (1L << (TIME_ESC - 67)) | (1L << (TIMESTAMP_ESC - 67)) | (1L << (GUID_ESC - 67)) | (1L << (PLUS - 67)) | (1L << (MINUS - 67)) | (1L << (ASTERISK - 67)) | (1L << (PARAM - 67)) | (1L << (STRING - 67)) | (1L << (INTEGER_VALUE - 67)) | (1L << (DECIMAL_VALUE - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { { - setState(649); + setState(656); _la = _input.LA(1); if (_la==ALL || _la==DISTINCT) { { - setState(648); + setState(655); setQuantifier(); } } - setState(651); + setState(658); expression(); - setState(656); + setState(663); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(652); + setState(659); match(T__2); - setState(653); + setState(660); expression(); } } - setState(658); + setState(665); _errHandler.sync(this); _la = _input.LA(1); } } } - setState(661); + setState(668); match(T__1); } } @@ -4887,19 +4909,19 @@ public final FunctionNameContext functionName() throws RecognitionException { FunctionNameContext _localctx = new FunctionNameContext(_ctx, getState()); enterRule(_localctx, 78, RULE_functionName); try { - setState(666); + setState(673); switch (_input.LA(1)) { case LEFT: enterOuterAlt(_localctx, 1); { - setState(663); + setState(670); match(LEFT); } break; case RIGHT: enterOuterAlt(_localctx, 2); { - setState(664); + setState(671); match(RIGHT); } break; @@ -4907,7 +4929,6 @@ public final FunctionNameContext functionName() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -4946,7 +4967,7 @@ public final FunctionNameContext functionName() throws RecognitionException { case BACKQUOTED_IDENTIFIER: enterOuterAlt(_localctx, 3); { - setState(665); + setState(672); identifier(); } break; @@ -5177,13 +5198,13 @@ public final ConstantContext constant() throws RecognitionException { enterRule(_localctx, 80, RULE_constant); try { int _alt; - setState(694); + setState(701); switch (_input.LA(1)) { case NULL: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(668); + setState(675); match(NULL); } break; @@ -5191,7 +5212,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntervalLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(669); + setState(676); interval(); } break; @@ -5200,7 +5221,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(670); + setState(677); number(); } break; @@ -5209,7 +5230,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(671); + setState(678); booleanValue(); } break; @@ -5217,7 +5238,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(673); + setState(680); _errHandler.sync(this); _alt = 1; do { @@ -5225,7 +5246,7 @@ public final ConstantContext constant() throws RecognitionException { case 1: { { - setState(672); + setState(679); match(STRING); } } @@ -5233,9 +5254,9 @@ public final ConstantContext constant() throws RecognitionException { default: throw new NoViableAltException(this); } - setState(675); + setState(682); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,89,_ctx); + _alt = getInterpreter().adaptivePredict(_input,91,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); } break; @@ -5243,7 +5264,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new ParamLiteralContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(677); + setState(684); match(PARAM); } break; @@ -5251,11 +5272,11 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DateEscapedLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(678); + setState(685); match(DATE_ESC); - setState(679); + setState(686); string(); - setState(680); + setState(687); match(ESC_END); } break; @@ -5263,11 +5284,11 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new TimeEscapedLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(682); + setState(689); match(TIME_ESC); - setState(683); + setState(690); string(); - setState(684); + setState(691); match(ESC_END); } break; @@ -5275,11 +5296,11 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new TimestampEscapedLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(686); + setState(693); match(TIMESTAMP_ESC); - setState(687); + setState(694); string(); - setState(688); + setState(695); match(ESC_END); } break; @@ -5287,11 +5308,11 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new GuidEscapedLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(690); + setState(697); match(GUID_ESC); - setState(691); + setState(698); string(); - setState(692); + setState(699); match(ESC_END); } break; @@ -5344,7 +5365,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(696); + setState(703); _la = _input.LA(1); if ( !(((((_la - 100)) & ~0x3f) == 0 && ((1L << (_la - 100)) & ((1L << (EQ - 100)) | (1L << (NULLEQ - 100)) | (1L << (NEQ - 100)) | (1L << (LT - 100)) | (1L << (LTE - 100)) | (1L << (GT - 100)) | (1L << (GTE - 100)))) != 0)) ) { _errHandler.recoverInline(this); @@ -5393,7 +5414,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(698); + setState(705); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -5461,13 +5482,13 @@ public final IntervalContext interval() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(700); + setState(707); match(INTERVAL); - setState(702); + setState(709); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(701); + setState(708); ((IntervalContext)_localctx).sign = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -5478,35 +5499,35 @@ public final IntervalContext interval() throws RecognitionException { } } - setState(706); + setState(713); switch (_input.LA(1)) { case INTEGER_VALUE: case DECIMAL_VALUE: { - setState(704); + setState(711); ((IntervalContext)_localctx).valueNumeric = number(); } break; case PARAM: case STRING: { - setState(705); + setState(712); ((IntervalContext)_localctx).valuePattern = string(); } break; default: throw new NoViableAltException(this); } - setState(708); + setState(715); ((IntervalContext)_localctx).leading = intervalField(); - setState(711); + setState(718); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,93,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,95,_ctx) ) { case 1: { - setState(709); + setState(716); match(TO); - setState(710); + setState(717); ((IntervalContext)_localctx).trailing = intervalField(); } break; @@ -5563,7 +5584,7 @@ public final IntervalFieldContext intervalField() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(713); + setState(720); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << DAY) | (1L << DAYS) | (1L << HOUR) | (1L << HOURS) | (1L << MINUTE) | (1L << MINUTES) | (1L << MONTH) | (1L << MONTHS))) != 0) || ((((_la - 74)) & ~0x3f) == 0 && ((1L << (_la - 74)) & ((1L << (SECOND - 74)) | (1L << (SECONDS - 74)) | (1L << (YEAR - 74)) | (1L << (YEARS - 74)))) != 0)) ) { _errHandler.recoverInline(this); @@ -5621,7 +5642,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new PrimitiveDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(715); + setState(722); identifier(); } } @@ -5673,25 +5694,25 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(722); + setState(729); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,94,_ctx); + _alt = getInterpreter().adaptivePredict(_input,96,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(717); + setState(724); identifier(); - setState(718); + setState(725); match(DOT); } } } - setState(724); + setState(731); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,94,_ctx); + _alt = getInterpreter().adaptivePredict(_input,96,_ctx); } - setState(725); + setState(732); identifier(); } } @@ -5736,13 +5757,13 @@ public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); enterRule(_localctx, 94, RULE_identifier); try { - setState(729); + setState(736); switch (_input.LA(1)) { case QUOTED_IDENTIFIER: case BACKQUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(727); + setState(734); quoteIdentifier(); } break; @@ -5750,7 +5771,6 @@ public final IdentifierContext identifier() throws RecognitionException { case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -5787,7 +5807,7 @@ public final IdentifierContext identifier() throws RecognitionException { case DIGIT_IDENTIFIER: enterOuterAlt(_localctx, 2); { - setState(728); + setState(735); unquoteIdentifier(); } break; @@ -5840,43 +5860,43 @@ public final TableIdentifierContext tableIdentifier() throws RecognitionExceptio enterRule(_localctx, 96, RULE_tableIdentifier); int _la; try { - setState(743); + setState(750); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,98,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,100,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(734); + setState(741); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CURRENT) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)) | (1L << (IDENTIFIER - 67)) | (1L << (DIGIT_IDENTIFIER - 67)) | (1L << (QUOTED_IDENTIFIER - 67)) | (1L << (BACKQUOTED_IDENTIFIER - 67)))) != 0)) { { - setState(731); + setState(738); ((TableIdentifierContext)_localctx).catalog = identifier(); - setState(732); + setState(739); match(T__3); } } - setState(736); + setState(743); match(TABLE_IDENTIFIER); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(740); + setState(747); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,97,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,99,_ctx) ) { case 1: { - setState(737); + setState(744); ((TableIdentifierContext)_localctx).catalog = identifier(); - setState(738); + setState(745); match(T__3); } break; } - setState(742); + setState(749); ((TableIdentifierContext)_localctx).name = identifier(); } break; @@ -5943,13 +5963,13 @@ public final QuoteIdentifierContext quoteIdentifier() throws RecognitionExceptio QuoteIdentifierContext _localctx = new QuoteIdentifierContext(_ctx, getState()); enterRule(_localctx, 98, RULE_quoteIdentifier); try { - setState(747); + setState(754); switch (_input.LA(1)) { case QUOTED_IDENTIFIER: _localctx = new QuotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(745); + setState(752); match(QUOTED_IDENTIFIER); } break; @@ -5957,7 +5977,7 @@ public final QuoteIdentifierContext quoteIdentifier() throws RecognitionExceptio _localctx = new BackQuotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(746); + setState(753); match(BACKQUOTED_IDENTIFIER); } break; @@ -6029,13 +6049,13 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce UnquoteIdentifierContext _localctx = new UnquoteIdentifierContext(_ctx, getState()); enterRule(_localctx, 100, RULE_unquoteIdentifier); try { - setState(752); + setState(759); switch (_input.LA(1)) { case IDENTIFIER: _localctx = new UnquotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(749); + setState(756); match(IDENTIFIER); } break; @@ -6043,7 +6063,6 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce case ANALYZED: case CATALOGS: case COLUMNS: - case CURRENT: case DAY: case DEBUG: case EXECUTABLE: @@ -6079,7 +6098,7 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce _localctx = new UnquotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(750); + setState(757); nonReserved(); } break; @@ -6087,7 +6106,7 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce _localctx = new DigitIdentifierContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(751); + setState(758); match(DIGIT_IDENTIFIER); } break; @@ -6156,13 +6175,13 @@ public final NumberContext number() throws RecognitionException { NumberContext _localctx = new NumberContext(_ctx, getState()); enterRule(_localctx, 102, RULE_number); try { - setState(756); + setState(763); switch (_input.LA(1)) { case DECIMAL_VALUE: _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(754); + setState(761); match(DECIMAL_VALUE); } break; @@ -6170,7 +6189,7 @@ public final NumberContext number() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(755); + setState(762); match(INTEGER_VALUE); } break; @@ -6218,7 +6237,7 @@ public final StringContext string() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(758); + setState(765); _la = _input.LA(1); if ( !(_la==PARAM || _la==STRING) ) { _errHandler.recoverInline(this); @@ -6243,7 +6262,6 @@ public static class NonReservedContext extends ParserRuleContext { public TerminalNode ANALYZED() { return getToken(SqlBaseParser.ANALYZED, 0); } public TerminalNode CATALOGS() { return getToken(SqlBaseParser.CATALOGS, 0); } public TerminalNode COLUMNS() { return getToken(SqlBaseParser.COLUMNS, 0); } - public TerminalNode CURRENT() { return getToken(SqlBaseParser.CURRENT, 0); } public TerminalNode DAY() { return getToken(SqlBaseParser.DAY, 0); } public TerminalNode DEBUG() { return getToken(SqlBaseParser.DEBUG, 0); } public TerminalNode EXECUTABLE() { return getToken(SqlBaseParser.EXECUTABLE, 0); } @@ -6302,9 +6320,9 @@ public final NonReservedContext nonReserved() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(760); + setState(767); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << CURRENT) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)))) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DAY) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FIRST) | (1L << FORMAT) | (1L << FULL) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << HOUR) | (1L << INTERVAL) | (1L << LAST) | (1L << LIMIT) | (1L << MAPPED) | (1L << MINUTE) | (1L << MONTH) | (1L << OPTIMIZED))) != 0) || ((((_la - 67)) & ~0x3f) == 0 && ((1L << (_la - 67)) & ((1L << (PARSED - 67)) | (1L << (PHYSICAL - 67)) | (1L << (PLAN - 67)) | (1L << (RLIKE - 67)) | (1L << (QUERY - 67)) | (1L << (SCHEMAS - 67)) | (1L << (SECOND - 67)) | (1L << (SHOW - 67)) | (1L << (SYS - 67)) | (1L << (TABLES - 67)) | (1L << (TEXT - 67)) | (1L << (TYPE - 67)) | (1L << (TYPES - 67)) | (1L << (VERIFY - 67)) | (1L << (YEAR - 67)))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); @@ -6353,7 +6371,7 @@ private boolean valueExpression_sempred(ValueExpressionContext _localctx, int pr } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3\u0081\u02fd\4\2\t"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3\u0081\u0304\4\2\t"+ "\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13"+ "\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+ @@ -6400,114 +6418,114 @@ private boolean valueExpression_sempred(ValueExpressionContext _localctx, int pr "\n\37\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3\37\3\37\7\37\u0237\n\37"+ "\f\37\16\37\u023a\13\37\3 \3 \3 \3 \3 \3 \3 \5 \u0243\n \3 \3 \3 \3 \3"+ " \3 \3 \3 \3 \3 \3 \5 \u0250\n \3!\3!\3!\3!\3!\3!\3!\3!\3!\3!\5!\u025c"+ - "\n!\3\"\3\"\3\"\3\"\3\"\3\"\3\"\3#\3#\3#\5#\u0268\n#\3#\5#\u026b\n#\3"+ - "$\3$\3$\3$\3$\3$\3$\3%\3%\3%\3%\3%\5%\u0279\n%\3&\3&\3&\3&\3&\3&\3&\3"+ - "\'\3\'\3\'\3\'\3\'\5\'\u0287\n\'\3(\3(\3(\5(\u028c\n(\3(\3(\3(\7(\u0291"+ - "\n(\f(\16(\u0294\13(\5(\u0296\n(\3(\3(\3)\3)\3)\5)\u029d\n)\3*\3*\3*\3"+ - "*\3*\6*\u02a4\n*\r*\16*\u02a5\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3"+ - "*\3*\3*\3*\5*\u02b9\n*\3+\3+\3,\3,\3-\3-\5-\u02c1\n-\3-\3-\5-\u02c5\n"+ - "-\3-\3-\3-\5-\u02ca\n-\3.\3.\3/\3/\3\60\3\60\3\60\7\60\u02d3\n\60\f\60"+ - "\16\60\u02d6\13\60\3\60\3\60\3\61\3\61\5\61\u02dc\n\61\3\62\3\62\3\62"+ - "\5\62\u02e1\n\62\3\62\3\62\3\62\3\62\5\62\u02e7\n\62\3\62\5\62\u02ea\n"+ - "\62\3\63\3\63\5\63\u02ee\n\63\3\64\3\64\3\64\5\64\u02f3\n\64\3\65\3\65"+ - "\5\65\u02f7\n\65\3\66\3\66\3\67\3\67\3\67\2\4.<8\2\4\6\b\n\f\16\20\22"+ - "\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BDFHJLNPRTVXZ\\^`bdfhjl\2"+ - "\22\b\2\7\7\t\t\36\36\66\66AAEE\4\2((SS\4\2\t\tAA\4\2%%--\3\2\32\33\3"+ - "\2mn\4\2\7\7vv\4\2\r\r\32\32\4\2##\62\62\4\2\7\7\34\34\3\2oq\3\2fl\4\2"+ - "\"\"TT\7\2\27\30+,8;LM\\]\3\2tu\31\2\b\t\22\23\25\25\27\27\31\31\36\36"+ - " #$&(++//\62\62\65\6688::AAEGILOPRSVWYY\\\\\u0358\2n\3\2\2\2\4q\3\2\2"+ - "\2\6\u00d9\3\2\2\2\b\u00e4\3\2\2\2\n\u00e8\3\2\2\2\f\u00fd\3\2\2\2\16"+ - "\u0104\3\2\2\2\20\u0106\3\2\2\2\22\u010e\3\2\2\2\24\u012a\3\2\2\2\26\u0134"+ - "\3\2\2\2\30\u013e\3\2\2\2\32\u014d\3\2\2\2\34\u014f\3\2\2\2\36\u0155\3"+ - "\2\2\2 \u0157\3\2\2\2\"\u015e\3\2\2\2$\u0170\3\2\2\2&\u0181\3\2\2\2(\u0191"+ - "\3\2\2\2*\u01ac\3\2\2\2,\u01ae\3\2\2\2.\u01cf\3\2\2\2\60\u01e0\3\2\2\2"+ - "\62\u01e3\3\2\2\2\64\u0215\3\2\2\2\66\u0217\3\2\2\28\u021a\3\2\2\2:\u0224"+ - "\3\2\2\2<\u022a\3\2\2\2>\u024f\3\2\2\2@\u025b\3\2\2\2B\u025d\3\2\2\2D"+ - "\u0264\3\2\2\2F\u026c\3\2\2\2H\u0278\3\2\2\2J\u027a\3\2\2\2L\u0286\3\2"+ - "\2\2N\u0288\3\2\2\2P\u029c\3\2\2\2R\u02b8\3\2\2\2T\u02ba\3\2\2\2V\u02bc"+ - "\3\2\2\2X\u02be\3\2\2\2Z\u02cb\3\2\2\2\\\u02cd\3\2\2\2^\u02d4\3\2\2\2"+ - "`\u02db\3\2\2\2b\u02e9\3\2\2\2d\u02ed\3\2\2\2f\u02f2\3\2\2\2h\u02f6\3"+ - "\2\2\2j\u02f8\3\2\2\2l\u02fa\3\2\2\2no\5\6\4\2op\7\2\2\3p\3\3\2\2\2qr"+ - "\5,\27\2rs\7\2\2\3s\5\3\2\2\2t\u00da\5\b\5\2u\u0083\7 \2\2v\177\7\3\2"+ - "\2wx\7G\2\2x~\t\2\2\2yz\7$\2\2z~\t\3\2\2{|\7Y\2\2|~\5V,\2}w\3\2\2\2}y"+ - "\3\2\2\2}{\3\2\2\2~\u0081\3\2\2\2\177}\3\2\2\2\177\u0080\3\2\2\2\u0080"+ - "\u0082\3\2\2\2\u0081\177\3\2\2\2\u0082\u0084\7\4\2\2\u0083v\3\2\2\2\u0083"+ - "\u0084\3\2\2\2\u0084\u0085\3\2\2\2\u0085\u00da\5\6\4\2\u0086\u0092\7\31"+ - "\2\2\u0087\u008e\7\3\2\2\u0088\u0089\7G\2\2\u0089\u008d\t\4\2\2\u008a"+ - "\u008b\7$\2\2\u008b\u008d\t\3\2\2\u008c\u0088\3\2\2\2\u008c\u008a\3\2"+ - "\2\2\u008d\u0090\3\2\2\2\u008e\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f"+ - "\u0091\3\2\2\2\u0090\u008e\3\2\2\2\u0091\u0093\7\4\2\2\u0092\u0087\3\2"+ - "\2\2\u0092\u0093\3\2\2\2\u0093\u0094\3\2\2\2\u0094\u00da\5\6\4\2\u0095"+ - "\u0096\7O\2\2\u0096\u0099\7R\2\2\u0097\u009a\5\66\34\2\u0098\u009a\5b"+ - "\62\2\u0099\u0097\3\2\2\2\u0099\u0098\3\2\2\2\u0099\u009a\3\2\2\2\u009a"+ - "\u00da\3\2\2\2\u009b\u009c\7O\2\2\u009c\u009d\7\23\2\2\u009d\u00a0\t\5"+ - "\2\2\u009e\u00a1\5\66\34\2\u009f\u00a1\5b\62\2\u00a0\u009e\3\2\2\2\u00a0"+ - "\u009f\3\2\2\2\u00a1\u00da\3\2\2\2\u00a2\u00a5\t\6\2\2\u00a3\u00a6\5\66"+ - "\34\2\u00a4\u00a6\5b\62\2\u00a5\u00a3\3\2\2\2\u00a5\u00a4\3\2\2\2\u00a6"+ - "\u00da\3\2\2\2\u00a7\u00a8\7O\2\2\u00a8\u00aa\7\'\2\2\u00a9\u00ab\5\66"+ - "\34\2\u00aa\u00a9\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\u00da\3\2\2\2\u00ac"+ - "\u00ad\7O\2\2\u00ad\u00da\7K\2\2\u00ae\u00af\7P\2\2\u00af\u00b2\7R\2\2"+ - "\u00b0\u00b1\7\21\2\2\u00b1\u00b3\5\66\34\2\u00b2\u00b0\3\2\2\2\u00b2"+ - "\u00b3\3\2\2\2\u00b3\u00b6\3\2\2\2\u00b4\u00b7\5\66\34\2\u00b5\u00b7\5"+ - "b\62\2\u00b6\u00b4\3\2\2\2\u00b6\u00b5\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7"+ - "\u00c1\3\2\2\2\u00b8\u00b9\7V\2\2\u00b9\u00be\5j\66\2\u00ba\u00bb\7\5"+ - "\2\2\u00bb\u00bd\5j\66\2\u00bc\u00ba\3\2\2\2\u00bd\u00c0\3\2\2\2\u00be"+ - "\u00bc\3\2\2\2\u00be\u00bf\3\2\2\2\u00bf\u00c2\3\2\2\2\u00c0\u00be\3\2"+ - "\2\2\u00c1\u00b8\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00da\3\2\2\2\u00c3"+ - "\u00c4\7P\2\2\u00c4\u00c7\7\23\2\2\u00c5\u00c6\7\21\2\2\u00c6\u00c8\5"+ - "j\66\2\u00c7\u00c5\3\2\2\2\u00c7\u00c8\3\2\2\2\u00c8\u00cc\3\2\2\2\u00c9"+ - "\u00ca\7Q\2\2\u00ca\u00cd\5\66\34\2\u00cb\u00cd\5b\62\2\u00cc\u00c9\3"+ - "\2\2\2\u00cc\u00cb\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\u00cf\3\2\2\2\u00ce"+ - "\u00d0\5\66\34\2\u00cf\u00ce\3\2\2\2\u00cf\u00d0\3\2\2\2\u00d0\u00da\3"+ - "\2\2\2\u00d1\u00d2\7P\2\2\u00d2\u00d7\7W\2\2\u00d3\u00d5\t\7\2\2\u00d4"+ - "\u00d3\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\u00d6\3\2\2\2\u00d6\u00d8\5h"+ - "\65\2\u00d7\u00d4\3\2\2\2\u00d7\u00d8\3\2\2\2\u00d8\u00da\3\2\2\2\u00d9"+ - "t\3\2\2\2\u00d9u\3\2\2\2\u00d9\u0086\3\2\2\2\u00d9\u0095\3\2\2\2\u00d9"+ - "\u009b\3\2\2\2\u00d9\u00a2\3\2\2\2\u00d9\u00a7\3\2\2\2\u00d9\u00ac\3\2"+ - "\2\2\u00d9\u00ae\3\2\2\2\u00d9\u00c3\3\2\2\2\u00d9\u00d1\3\2\2\2\u00da"+ - "\7\3\2\2\2\u00db\u00dc\7[\2\2\u00dc\u00e1\5\34\17\2\u00dd\u00de\7\5\2"+ - "\2\u00de\u00e0\5\34\17\2\u00df\u00dd\3\2\2\2\u00e0\u00e3\3\2\2\2\u00e1"+ - "\u00df\3\2\2\2\u00e1\u00e2\3\2\2\2\u00e2\u00e5\3\2\2\2\u00e3\u00e1\3\2"+ - "\2\2\u00e4\u00db\3\2\2\2\u00e4\u00e5\3\2\2\2\u00e5\u00e6\3\2\2\2\u00e6"+ - "\u00e7\5\n\6\2\u00e7\t\3\2\2\2\u00e8\u00f3\5\16\b\2\u00e9\u00ea\7C\2\2"+ - "\u00ea\u00eb\7\17\2\2\u00eb\u00f0\5\20\t\2\u00ec\u00ed\7\5\2\2\u00ed\u00ef"+ - "\5\20\t\2\u00ee\u00ec\3\2\2\2\u00ef\u00f2\3\2\2\2\u00f0\u00ee\3\2\2\2"+ - "\u00f0\u00f1\3\2\2\2\u00f1\u00f4\3\2\2\2\u00f2\u00f0\3\2\2\2\u00f3\u00e9"+ - "\3\2\2\2\u00f3\u00f4\3\2\2\2\u00f4\u00f6\3\2\2\2\u00f5\u00f7\5\f\7\2\u00f6"+ - "\u00f5\3\2\2\2\u00f6\u00f7\3\2\2\2\u00f7\13\3\2\2\2\u00f8\u00f9\7\65\2"+ - "\2\u00f9\u00fe\t\b\2\2\u00fa\u00fb\7`\2\2\u00fb\u00fc\t\b\2\2\u00fc\u00fe"+ - "\7e\2\2\u00fd\u00f8\3\2\2\2\u00fd\u00fa\3\2\2\2\u00fe\r\3\2\2\2\u00ff"+ - "\u0105\5\22\n\2\u0100\u0101\7\3\2\2\u0101\u0102\5\n\6\2\u0102\u0103\7"+ - "\4\2\2\u0103\u0105\3\2\2\2\u0104\u00ff\3\2\2\2\u0104\u0100\3\2\2\2\u0105"+ - "\17\3\2\2\2\u0106\u0108\5,\27\2\u0107\u0109\t\t\2\2\u0108\u0107\3\2\2"+ - "\2\u0108\u0109\3\2\2\2\u0109\u010c\3\2\2\2\u010a\u010b\7?\2\2\u010b\u010d"+ - "\t\n\2\2\u010c\u010a\3\2\2\2\u010c\u010d\3\2\2\2\u010d\21\3\2\2\2\u010e"+ - "\u0110\7N\2\2\u010f\u0111\5\36\20\2\u0110\u010f\3\2\2\2\u0110\u0111\3"+ - "\2\2\2\u0111\u0112\3\2\2\2\u0112\u0117\5 \21\2\u0113\u0114\7\5\2\2\u0114"+ - "\u0116\5 \21\2\u0115\u0113\3\2\2\2\u0116\u0119\3\2\2\2\u0117\u0115\3\2"+ - "\2\2\u0117\u0118\3\2\2\2\u0118\u011b\3\2\2\2\u0119\u0117\3\2\2\2\u011a"+ - "\u011c\5\24\13\2\u011b\u011a\3\2\2\2\u011b\u011c\3\2\2\2\u011c\u011f\3"+ - "\2\2\2\u011d\u011e\7Z\2\2\u011e\u0120\5.\30\2\u011f\u011d\3\2\2\2\u011f"+ - "\u0120\3\2\2\2\u0120\u0124\3\2\2\2\u0121\u0122\7)\2\2\u0122\u0123\7\17"+ - "\2\2\u0123\u0125\5\26\f\2\u0124\u0121\3\2\2\2\u0124\u0125\3\2\2\2\u0125"+ - "\u0128\3\2\2\2\u0126\u0127\7*\2\2\u0127\u0129\5.\30\2\u0128\u0126\3\2"+ - "\2\2\u0128\u0129\3\2\2\2\u0129\23\3\2\2\2\u012a\u012b\7%\2\2\u012b\u0130"+ - "\5\"\22\2\u012c\u012d\7\5\2\2\u012d\u012f\5\"\22\2\u012e\u012c\3\2\2\2"+ - "\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130\u0131\3\2\2\2\u0131\25"+ - "\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0135\5\36\20\2\u0134\u0133\3\2\2\2"+ - "\u0134\u0135\3\2\2\2\u0135\u0136\3\2\2\2\u0136\u013b\5\30\r\2\u0137\u0138"+ - "\7\5\2\2\u0138\u013a\5\30\r\2\u0139\u0137\3\2\2\2\u013a\u013d\3\2\2\2"+ - "\u013b\u0139\3\2\2\2\u013b\u013c\3\2\2\2\u013c\27\3\2\2\2\u013d\u013b"+ - "\3\2\2\2\u013e\u013f\5\32\16\2\u013f\31\3\2\2\2\u0140\u0149\7\3\2\2\u0141"+ - "\u0146\5,\27\2\u0142\u0143\7\5\2\2\u0143\u0145\5,\27\2\u0144\u0142\3\2"+ - "\2\2\u0145\u0148\3\2\2\2\u0146\u0144\3\2\2\2\u0146\u0147\3\2\2\2\u0147"+ - "\u014a\3\2\2\2\u0148\u0146\3\2\2\2\u0149\u0141\3\2\2\2\u0149\u014a\3\2"+ - "\2\2\u014a\u014b\3\2\2\2\u014b\u014e\7\4\2\2\u014c\u014e\5,\27\2\u014d"+ - "\u0140\3\2\2\2\u014d\u014c\3\2\2\2\u014e\33\3\2\2\2\u014f\u0150\5`\61"+ - "\2\u0150\u0151\7\f\2\2\u0151\u0152\7\3\2\2\u0152\u0153\5\n\6\2\u0153\u0154"+ - "\7\4\2\2\u0154\35\3\2\2\2\u0155\u0156\t\13\2\2\u0156\37\3\2\2\2\u0157"+ - "\u015c\5,\27\2\u0158\u015a\7\f\2\2\u0159\u0158\3\2\2\2\u0159\u015a\3\2"+ - "\2\2\u015a\u015b\3\2\2\2\u015b\u015d\5`\61\2\u015c\u0159\3\2\2\2\u015c"+ + "\n!\3\"\3\"\3\"\3\"\3\"\3\"\3\"\3#\3#\3#\5#\u0268\n#\3#\3#\3#\5#\u026d"+ + "\n#\3#\5#\u0270\n#\5#\u0272\n#\3$\3$\3$\3$\3$\3$\3$\3%\3%\3%\3%\3%\5%"+ + "\u0280\n%\3&\3&\3&\3&\3&\3&\3&\3\'\3\'\3\'\3\'\3\'\5\'\u028e\n\'\3(\3"+ + "(\3(\5(\u0293\n(\3(\3(\3(\7(\u0298\n(\f(\16(\u029b\13(\5(\u029d\n(\3("+ + "\3(\3)\3)\3)\5)\u02a4\n)\3*\3*\3*\3*\3*\6*\u02ab\n*\r*\16*\u02ac\3*\3"+ + "*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\3*\5*\u02c0\n*\3+\3+\3,\3"+ + ",\3-\3-\5-\u02c8\n-\3-\3-\5-\u02cc\n-\3-\3-\3-\5-\u02d1\n-\3.\3.\3/\3"+ + "/\3\60\3\60\3\60\7\60\u02da\n\60\f\60\16\60\u02dd\13\60\3\60\3\60\3\61"+ + "\3\61\5\61\u02e3\n\61\3\62\3\62\3\62\5\62\u02e8\n\62\3\62\3\62\3\62\3"+ + "\62\5\62\u02ee\n\62\3\62\5\62\u02f1\n\62\3\63\3\63\5\63\u02f5\n\63\3\64"+ + "\3\64\3\64\5\64\u02fa\n\64\3\65\3\65\5\65\u02fe\n\65\3\66\3\66\3\67\3"+ + "\67\3\67\2\4.<8\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62"+ + "\64\668:<>@BDFHJLNPRTVXZ\\^`bdfhjl\2\22\b\2\7\7\t\t\36\36\66\66AAEE\4"+ + "\2((SS\4\2\t\tAA\4\2%%--\3\2\32\33\3\2mn\4\2\7\7vv\4\2\r\r\32\32\4\2#"+ + "#\62\62\4\2\7\7\34\34\3\2oq\3\2fl\4\2\"\"TT\7\2\27\30+,8;LM\\]\3\2tu\30"+ + "\2\b\t\22\23\27\27\31\31\36\36 #$&(++//\62\62\65\6688::AAEGILOPRSVWY"+ + "Y\\\\\u0361\2n\3\2\2\2\4q\3\2\2\2\6\u00d9\3\2\2\2\b\u00e4\3\2\2\2\n\u00e8"+ + "\3\2\2\2\f\u00fd\3\2\2\2\16\u0104\3\2\2\2\20\u0106\3\2\2\2\22\u010e\3"+ + "\2\2\2\24\u012a\3\2\2\2\26\u0134\3\2\2\2\30\u013e\3\2\2\2\32\u014d\3\2"+ + "\2\2\34\u014f\3\2\2\2\36\u0155\3\2\2\2 \u0157\3\2\2\2\"\u015e\3\2\2\2"+ + "$\u0170\3\2\2\2&\u0181\3\2\2\2(\u0191\3\2\2\2*\u01ac\3\2\2\2,\u01ae\3"+ + "\2\2\2.\u01cf\3\2\2\2\60\u01e0\3\2\2\2\62\u01e3\3\2\2\2\64\u0215\3\2\2"+ + "\2\66\u0217\3\2\2\28\u021a\3\2\2\2:\u0224\3\2\2\2<\u022a\3\2\2\2>\u024f"+ + "\3\2\2\2@\u025b\3\2\2\2B\u025d\3\2\2\2D\u0271\3\2\2\2F\u0273\3\2\2\2H"+ + "\u027f\3\2\2\2J\u0281\3\2\2\2L\u028d\3\2\2\2N\u028f\3\2\2\2P\u02a3\3\2"+ + "\2\2R\u02bf\3\2\2\2T\u02c1\3\2\2\2V\u02c3\3\2\2\2X\u02c5\3\2\2\2Z\u02d2"+ + "\3\2\2\2\\\u02d4\3\2\2\2^\u02db\3\2\2\2`\u02e2\3\2\2\2b\u02f0\3\2\2\2"+ + "d\u02f4\3\2\2\2f\u02f9\3\2\2\2h\u02fd\3\2\2\2j\u02ff\3\2\2\2l\u0301\3"+ + "\2\2\2no\5\6\4\2op\7\2\2\3p\3\3\2\2\2qr\5,\27\2rs\7\2\2\3s\5\3\2\2\2t"+ + "\u00da\5\b\5\2u\u0083\7 \2\2v\177\7\3\2\2wx\7G\2\2x~\t\2\2\2yz\7$\2\2"+ + "z~\t\3\2\2{|\7Y\2\2|~\5V,\2}w\3\2\2\2}y\3\2\2\2}{\3\2\2\2~\u0081\3\2\2"+ + "\2\177}\3\2\2\2\177\u0080\3\2\2\2\u0080\u0082\3\2\2\2\u0081\177\3\2\2"+ + "\2\u0082\u0084\7\4\2\2\u0083v\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0085"+ + "\3\2\2\2\u0085\u00da\5\6\4\2\u0086\u0092\7\31\2\2\u0087\u008e\7\3\2\2"+ + "\u0088\u0089\7G\2\2\u0089\u008d\t\4\2\2\u008a\u008b\7$\2\2\u008b\u008d"+ + "\t\3\2\2\u008c\u0088\3\2\2\2\u008c\u008a\3\2\2\2\u008d\u0090\3\2\2\2\u008e"+ + "\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2\2\2\u0090\u008e\3\2"+ + "\2\2\u0091\u0093\7\4\2\2\u0092\u0087\3\2\2\2\u0092\u0093\3\2\2\2\u0093"+ + "\u0094\3\2\2\2\u0094\u00da\5\6\4\2\u0095\u0096\7O\2\2\u0096\u0099\7R\2"+ + "\2\u0097\u009a\5\66\34\2\u0098\u009a\5b\62\2\u0099\u0097\3\2\2\2\u0099"+ + "\u0098\3\2\2\2\u0099\u009a\3\2\2\2\u009a\u00da\3\2\2\2\u009b\u009c\7O"+ + "\2\2\u009c\u009d\7\23\2\2\u009d\u00a0\t\5\2\2\u009e\u00a1\5\66\34\2\u009f"+ + "\u00a1\5b\62\2\u00a0\u009e\3\2\2\2\u00a0\u009f\3\2\2\2\u00a1\u00da\3\2"+ + "\2\2\u00a2\u00a5\t\6\2\2\u00a3\u00a6\5\66\34\2\u00a4\u00a6\5b\62\2\u00a5"+ + "\u00a3\3\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00da\3\2\2\2\u00a7\u00a8\7O"+ + "\2\2\u00a8\u00aa\7\'\2\2\u00a9\u00ab\5\66\34\2\u00aa\u00a9\3\2\2\2\u00aa"+ + "\u00ab\3\2\2\2\u00ab\u00da\3\2\2\2\u00ac\u00ad\7O\2\2\u00ad\u00da\7K\2"+ + "\2\u00ae\u00af\7P\2\2\u00af\u00b2\7R\2\2\u00b0\u00b1\7\21\2\2\u00b1\u00b3"+ + "\5\66\34\2\u00b2\u00b0\3\2\2\2\u00b2\u00b3\3\2\2\2\u00b3\u00b6\3\2\2\2"+ + "\u00b4\u00b7\5\66\34\2\u00b5\u00b7\5b\62\2\u00b6\u00b4\3\2\2\2\u00b6\u00b5"+ + "\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00c1\3\2\2\2\u00b8\u00b9\7V\2\2\u00b9"+ + "\u00be\5j\66\2\u00ba\u00bb\7\5\2\2\u00bb\u00bd\5j\66\2\u00bc\u00ba\3\2"+ + "\2\2\u00bd\u00c0\3\2\2\2\u00be\u00bc\3\2\2\2\u00be\u00bf\3\2\2\2\u00bf"+ + "\u00c2\3\2\2\2\u00c0\u00be\3\2\2\2\u00c1\u00b8\3\2\2\2\u00c1\u00c2\3\2"+ + "\2\2\u00c2\u00da\3\2\2\2\u00c3\u00c4\7P\2\2\u00c4\u00c7\7\23\2\2\u00c5"+ + "\u00c6\7\21\2\2\u00c6\u00c8\5j\66\2\u00c7\u00c5\3\2\2\2\u00c7\u00c8\3"+ + "\2\2\2\u00c8\u00cc\3\2\2\2\u00c9\u00ca\7Q\2\2\u00ca\u00cd\5\66\34\2\u00cb"+ + "\u00cd\5b\62\2\u00cc\u00c9\3\2\2\2\u00cc\u00cb\3\2\2\2\u00cc\u00cd\3\2"+ + "\2\2\u00cd\u00cf\3\2\2\2\u00ce\u00d0\5\66\34\2\u00cf\u00ce\3\2\2\2\u00cf"+ + "\u00d0\3\2\2\2\u00d0\u00da\3\2\2\2\u00d1\u00d2\7P\2\2\u00d2\u00d7\7W\2"+ + "\2\u00d3\u00d5\t\7\2\2\u00d4\u00d3\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\u00d6"+ + "\3\2\2\2\u00d6\u00d8\5h\65\2\u00d7\u00d4\3\2\2\2\u00d7\u00d8\3\2\2\2\u00d8"+ + "\u00da\3\2\2\2\u00d9t\3\2\2\2\u00d9u\3\2\2\2\u00d9\u0086\3\2\2\2\u00d9"+ + "\u0095\3\2\2\2\u00d9\u009b\3\2\2\2\u00d9\u00a2\3\2\2\2\u00d9\u00a7\3\2"+ + "\2\2\u00d9\u00ac\3\2\2\2\u00d9\u00ae\3\2\2\2\u00d9\u00c3\3\2\2\2\u00d9"+ + "\u00d1\3\2\2\2\u00da\7\3\2\2\2\u00db\u00dc\7[\2\2\u00dc\u00e1\5\34\17"+ + "\2\u00dd\u00de\7\5\2\2\u00de\u00e0\5\34\17\2\u00df\u00dd\3\2\2\2\u00e0"+ + "\u00e3\3\2\2\2\u00e1\u00df\3\2\2\2\u00e1\u00e2\3\2\2\2\u00e2\u00e5\3\2"+ + "\2\2\u00e3\u00e1\3\2\2\2\u00e4\u00db\3\2\2\2\u00e4\u00e5\3\2\2\2\u00e5"+ + "\u00e6\3\2\2\2\u00e6\u00e7\5\n\6\2\u00e7\t\3\2\2\2\u00e8\u00f3\5\16\b"+ + "\2\u00e9\u00ea\7C\2\2\u00ea\u00eb\7\17\2\2\u00eb\u00f0\5\20\t\2\u00ec"+ + "\u00ed\7\5\2\2\u00ed\u00ef\5\20\t\2\u00ee\u00ec\3\2\2\2\u00ef\u00f2\3"+ + "\2\2\2\u00f0\u00ee\3\2\2\2\u00f0\u00f1\3\2\2\2\u00f1\u00f4\3\2\2\2\u00f2"+ + "\u00f0\3\2\2\2\u00f3\u00e9\3\2\2\2\u00f3\u00f4\3\2\2\2\u00f4\u00f6\3\2"+ + "\2\2\u00f5\u00f7\5\f\7\2\u00f6\u00f5\3\2\2\2\u00f6\u00f7\3\2\2\2\u00f7"+ + "\13\3\2\2\2\u00f8\u00f9\7\65\2\2\u00f9\u00fe\t\b\2\2\u00fa\u00fb\7`\2"+ + "\2\u00fb\u00fc\t\b\2\2\u00fc\u00fe\7e\2\2\u00fd\u00f8\3\2\2\2\u00fd\u00fa"+ + "\3\2\2\2\u00fe\r\3\2\2\2\u00ff\u0105\5\22\n\2\u0100\u0101\7\3\2\2\u0101"+ + "\u0102\5\n\6\2\u0102\u0103\7\4\2\2\u0103\u0105\3\2\2\2\u0104\u00ff\3\2"+ + "\2\2\u0104\u0100\3\2\2\2\u0105\17\3\2\2\2\u0106\u0108\5,\27\2\u0107\u0109"+ + "\t\t\2\2\u0108\u0107\3\2\2\2\u0108\u0109\3\2\2\2\u0109\u010c\3\2\2\2\u010a"+ + "\u010b\7?\2\2\u010b\u010d\t\n\2\2\u010c\u010a\3\2\2\2\u010c\u010d\3\2"+ + "\2\2\u010d\21\3\2\2\2\u010e\u0110\7N\2\2\u010f\u0111\5\36\20\2\u0110\u010f"+ + "\3\2\2\2\u0110\u0111\3\2\2\2\u0111\u0112\3\2\2\2\u0112\u0117\5 \21\2\u0113"+ + "\u0114\7\5\2\2\u0114\u0116\5 \21\2\u0115\u0113\3\2\2\2\u0116\u0119\3\2"+ + "\2\2\u0117\u0115\3\2\2\2\u0117\u0118\3\2\2\2\u0118\u011b\3\2\2\2\u0119"+ + "\u0117\3\2\2\2\u011a\u011c\5\24\13\2\u011b\u011a\3\2\2\2\u011b\u011c\3"+ + "\2\2\2\u011c\u011f\3\2\2\2\u011d\u011e\7Z\2\2\u011e\u0120\5.\30\2\u011f"+ + "\u011d\3\2\2\2\u011f\u0120\3\2\2\2\u0120\u0124\3\2\2\2\u0121\u0122\7)"+ + "\2\2\u0122\u0123\7\17\2\2\u0123\u0125\5\26\f\2\u0124\u0121\3\2\2\2\u0124"+ + "\u0125\3\2\2\2\u0125\u0128\3\2\2\2\u0126\u0127\7*\2\2\u0127\u0129\5.\30"+ + "\2\u0128\u0126\3\2\2\2\u0128\u0129\3\2\2\2\u0129\23\3\2\2\2\u012a\u012b"+ + "\7%\2\2\u012b\u0130\5\"\22\2\u012c\u012d\7\5\2\2\u012d\u012f\5\"\22\2"+ + "\u012e\u012c\3\2\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130\u0131"+ + "\3\2\2\2\u0131\25\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0135\5\36\20\2\u0134"+ + "\u0133\3\2\2\2\u0134\u0135\3\2\2\2\u0135\u0136\3\2\2\2\u0136\u013b\5\30"+ + "\r\2\u0137\u0138\7\5\2\2\u0138\u013a\5\30\r\2\u0139\u0137\3\2\2\2\u013a"+ + "\u013d\3\2\2\2\u013b\u0139\3\2\2\2\u013b\u013c\3\2\2\2\u013c\27\3\2\2"+ + "\2\u013d\u013b\3\2\2\2\u013e\u013f\5\32\16\2\u013f\31\3\2\2\2\u0140\u0149"+ + "\7\3\2\2\u0141\u0146\5,\27\2\u0142\u0143\7\5\2\2\u0143\u0145\5,\27\2\u0144"+ + "\u0142\3\2\2\2\u0145\u0148\3\2\2\2\u0146\u0144\3\2\2\2\u0146\u0147\3\2"+ + "\2\2\u0147\u014a\3\2\2\2\u0148\u0146\3\2\2\2\u0149\u0141\3\2\2\2\u0149"+ + "\u014a\3\2\2\2\u014a\u014b\3\2\2\2\u014b\u014e\7\4\2\2\u014c\u014e\5,"+ + "\27\2\u014d\u0140\3\2\2\2\u014d\u014c\3\2\2\2\u014e\33\3\2\2\2\u014f\u0150"+ + "\5`\61\2\u0150\u0151\7\f\2\2\u0151\u0152\7\3\2\2\u0152\u0153\5\n\6\2\u0153"+ + "\u0154\7\4\2\2\u0154\35\3\2\2\2\u0155\u0156\t\13\2\2\u0156\37\3\2\2\2"+ + "\u0157\u015c\5,\27\2\u0158\u015a\7\f\2\2\u0159\u0158\3\2\2\2\u0159\u015a"+ + "\3\2\2\2\u015a\u015b\3\2\2\2\u015b\u015d\5`\61\2\u015c\u0159\3\2\2\2\u015c"+ "\u015d\3\2\2\2\u015d!\3\2\2\2\u015e\u0162\5*\26\2\u015f\u0161\5$\23\2"+ "\u0160\u015f\3\2\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162\u0163"+ "\3\2\2\2\u0163#\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u0166\5&\24\2\u0166"+ @@ -6598,65 +6616,67 @@ private boolean valueExpression_sempred(ValueExpressionContext _localctx, int pr "\2\u025b\u0252\3\2\2\2\u025b\u0256\3\2\2\2\u025b\u0257\3\2\2\2\u025cA"+ "\3\2\2\2\u025d\u025e\7\20\2\2\u025e\u025f\7\3\2\2\u025f\u0260\5,\27\2"+ "\u0260\u0261\7\f\2\2\u0261\u0262\5\\/\2\u0262\u0263\7\4\2\2\u0263C\3\2"+ - "\2\2\u0264\u026a\7\26\2\2\u0265\u0267\7\3\2\2\u0266\u0268\7v\2\2\u0267"+ - "\u0266\3\2\2\2\u0267\u0268\3\2\2\2\u0268\u0269\3\2\2\2\u0269\u026b\7\4"+ - "\2\2\u026a\u0265\3\2\2\2\u026a\u026b\3\2\2\2\u026bE\3\2\2\2\u026c\u026d"+ - "\7\24\2\2\u026d\u026e\7\3\2\2\u026e\u026f\5,\27\2\u026f\u0270\7\5\2\2"+ - "\u0270\u0271\5\\/\2\u0271\u0272\7\4\2\2\u0272G\3\2\2\2\u0273\u0279\5J"+ - "&\2\u0274\u0275\7_\2\2\u0275\u0276\5J&\2\u0276\u0277\7e\2\2\u0277\u0279"+ - "\3\2\2\2\u0278\u0273\3\2\2\2\u0278\u0274\3\2\2\2\u0279I\3\2\2\2\u027a"+ - "\u027b\7!\2\2\u027b\u027c\7\3\2\2\u027c\u027d\5`\61\2\u027d\u027e\7%\2"+ - "\2\u027e\u027f\5<\37\2\u027f\u0280\7\4\2\2\u0280K\3\2\2\2\u0281\u0287"+ - "\5N(\2\u0282\u0283\7_\2\2\u0283\u0284\5N(\2\u0284\u0285\7e\2\2\u0285\u0287"+ - "\3\2\2\2\u0286\u0281\3\2\2\2\u0286\u0282\3\2\2\2\u0287M\3\2\2\2\u0288"+ - "\u0289\5P)\2\u0289\u0295\7\3\2\2\u028a\u028c\5\36\20\2\u028b\u028a\3\2"+ - "\2\2\u028b\u028c\3\2\2\2\u028c\u028d\3\2\2\2\u028d\u0292\5,\27\2\u028e"+ - "\u028f\7\5\2\2\u028f\u0291\5,\27\2\u0290\u028e\3\2\2\2\u0291\u0294\3\2"+ - "\2\2\u0292\u0290\3\2\2\2\u0292\u0293\3\2\2\2\u0293\u0296\3\2\2\2\u0294"+ - "\u0292\3\2\2\2\u0295\u028b\3\2\2\2\u0295\u0296\3\2\2\2\u0296\u0297\3\2"+ - "\2\2\u0297\u0298\7\4\2\2\u0298O\3\2\2\2\u0299\u029d\7\63\2\2\u029a\u029d"+ - "\7H\2\2\u029b\u029d\5`\61\2\u029c\u0299\3\2\2\2\u029c\u029a\3\2\2\2\u029c"+ - "\u029b\3\2\2\2\u029dQ\3\2\2\2\u029e\u02b9\7>\2\2\u029f\u02b9\5X-\2\u02a0"+ - "\u02b9\5h\65\2\u02a1\u02b9\5V,\2\u02a2\u02a4\7u\2\2\u02a3\u02a2\3\2\2"+ - "\2\u02a4\u02a5\3\2\2\2\u02a5\u02a3\3\2\2\2\u02a5\u02a6\3\2\2\2\u02a6\u02b9"+ - "\3\2\2\2\u02a7\u02b9\7t\2\2\u02a8\u02a9\7a\2\2\u02a9\u02aa\5j\66\2\u02aa"+ - "\u02ab\7e\2\2\u02ab\u02b9\3\2\2\2\u02ac\u02ad\7b\2\2\u02ad\u02ae\5j\66"+ - "\2\u02ae\u02af\7e\2\2\u02af\u02b9\3\2\2\2\u02b0\u02b1\7c\2\2\u02b1\u02b2"+ - "\5j\66\2\u02b2\u02b3\7e\2\2\u02b3\u02b9\3\2\2\2\u02b4\u02b5\7d\2\2\u02b5"+ - "\u02b6\5j\66\2\u02b6\u02b7\7e\2\2\u02b7\u02b9\3\2\2\2\u02b8\u029e\3\2"+ - "\2\2\u02b8\u029f\3\2\2\2\u02b8\u02a0\3\2\2\2\u02b8\u02a1\3\2\2\2\u02b8"+ - "\u02a3\3\2\2\2\u02b8\u02a7\3\2\2\2\u02b8\u02a8\3\2\2\2\u02b8\u02ac\3\2"+ - "\2\2\u02b8\u02b0\3\2\2\2\u02b8\u02b4\3\2\2\2\u02b9S\3\2\2\2\u02ba\u02bb"+ - "\t\r\2\2\u02bbU\3\2\2\2\u02bc\u02bd\t\16\2\2\u02bdW\3\2\2\2\u02be\u02c0"+ - "\7/\2\2\u02bf\u02c1\t\7\2\2\u02c0\u02bf\3\2\2\2\u02c0\u02c1\3\2\2\2\u02c1"+ - "\u02c4\3\2\2\2\u02c2\u02c5\5h\65\2\u02c3\u02c5\5j\66\2\u02c4\u02c2\3\2"+ - "\2\2\u02c4\u02c3\3\2\2\2\u02c5\u02c6\3\2\2\2\u02c6\u02c9\5Z.\2\u02c7\u02c8"+ - "\7U\2\2\u02c8\u02ca\5Z.\2\u02c9\u02c7\3\2\2\2\u02c9\u02ca\3\2\2\2\u02ca"+ - "Y\3\2\2\2\u02cb\u02cc\t\17\2\2\u02cc[\3\2\2\2\u02cd\u02ce\5`\61\2\u02ce"+ - "]\3\2\2\2\u02cf\u02d0\5`\61\2\u02d0\u02d1\7s\2\2\u02d1\u02d3\3\2\2\2\u02d2"+ - "\u02cf\3\2\2\2\u02d3\u02d6\3\2\2\2\u02d4\u02d2\3\2\2\2\u02d4\u02d5\3\2"+ - "\2\2\u02d5\u02d7\3\2\2\2\u02d6\u02d4\3\2\2\2\u02d7\u02d8\5`\61\2\u02d8"+ - "_\3\2\2\2\u02d9\u02dc\5d\63\2\u02da\u02dc\5f\64\2\u02db\u02d9\3\2\2\2"+ - "\u02db\u02da\3\2\2\2\u02dca\3\2\2\2\u02dd\u02de\5`\61\2\u02de\u02df\7"+ - "\6\2\2\u02df\u02e1\3\2\2\2\u02e0\u02dd\3\2\2\2\u02e0\u02e1\3\2\2\2\u02e1"+ - "\u02e2\3\2\2\2\u02e2\u02ea\7z\2\2\u02e3\u02e4\5`\61\2\u02e4\u02e5\7\6"+ - "\2\2\u02e5\u02e7\3\2\2\2\u02e6\u02e3\3\2\2\2\u02e6\u02e7\3\2\2\2\u02e7"+ - "\u02e8\3\2\2\2\u02e8\u02ea\5`\61\2\u02e9\u02e0\3\2\2\2\u02e9\u02e6\3\2"+ - "\2\2\u02eac\3\2\2\2\u02eb\u02ee\7{\2\2\u02ec\u02ee\7|\2\2\u02ed\u02eb"+ - "\3\2\2\2\u02ed\u02ec\3\2\2\2\u02eee\3\2\2\2\u02ef\u02f3\7x\2\2\u02f0\u02f3"+ - "\5l\67\2\u02f1\u02f3\7y\2\2\u02f2\u02ef\3\2\2\2\u02f2\u02f0\3\2\2\2\u02f2"+ - "\u02f1\3\2\2\2\u02f3g\3\2\2\2\u02f4\u02f7\7w\2\2\u02f5\u02f7\7v\2\2\u02f6"+ - "\u02f4\3\2\2\2\u02f6\u02f5\3\2\2\2\u02f7i\3\2\2\2\u02f8\u02f9\t\20\2\2"+ - "\u02f9k\3\2\2\2\u02fa\u02fb\t\21\2\2\u02fbm\3\2\2\2h}\177\u0083\u008c"+ - "\u008e\u0092\u0099\u00a0\u00a5\u00aa\u00b2\u00b6\u00be\u00c1\u00c7\u00cc"+ - "\u00cf\u00d4\u00d7\u00d9\u00e1\u00e4\u00f0\u00f3\u00f6\u00fd\u0104\u0108"+ - "\u010c\u0110\u0117\u011b\u011f\u0124\u0128\u0130\u0134\u013b\u0146\u0149"+ - "\u014d\u0159\u015c\u0162\u0169\u0170\u0173\u0177\u017b\u017f\u0181\u018c"+ - "\u0191\u0195\u0198\u019e\u01a1\u01a7\u01aa\u01ac\u01cf\u01d7\u01d9\u01e0"+ - "\u01e5\u01e8\u01f0\u01f9\u01ff\u0207\u020c\u0212\u0215\u021c\u0224\u022a"+ - "\u0236\u0238\u0242\u024f\u025b\u0267\u026a\u0278\u0286\u028b\u0292\u0295"+ - "\u029c\u02a5\u02b8\u02c0\u02c4\u02c9\u02d4\u02db\u02e0\u02e6\u02e9\u02ed"+ - "\u02f2\u02f6"; + "\2\2\u0264\u0267\7\25\2\2\u0265\u0266\7\3\2\2\u0266\u0268\7\4\2\2\u0267"+ + "\u0265\3\2\2\2\u0267\u0268\3\2\2\2\u0268\u0272\3\2\2\2\u0269\u026f\7\26"+ + "\2\2\u026a\u026c\7\3\2\2\u026b\u026d\7v\2\2\u026c\u026b\3\2\2\2\u026c"+ + "\u026d\3\2\2\2\u026d\u026e\3\2\2\2\u026e\u0270\7\4\2\2\u026f\u026a\3\2"+ + "\2\2\u026f\u0270\3\2\2\2\u0270\u0272\3\2\2\2\u0271\u0264\3\2\2\2\u0271"+ + "\u0269\3\2\2\2\u0272E\3\2\2\2\u0273\u0274\7\24\2\2\u0274\u0275\7\3\2\2"+ + "\u0275\u0276\5,\27\2\u0276\u0277\7\5\2\2\u0277\u0278\5\\/\2\u0278\u0279"+ + "\7\4\2\2\u0279G\3\2\2\2\u027a\u0280\5J&\2\u027b\u027c\7_\2\2\u027c\u027d"+ + "\5J&\2\u027d\u027e\7e\2\2\u027e\u0280\3\2\2\2\u027f\u027a\3\2\2\2\u027f"+ + "\u027b\3\2\2\2\u0280I\3\2\2\2\u0281\u0282\7!\2\2\u0282\u0283\7\3\2\2\u0283"+ + "\u0284\5`\61\2\u0284\u0285\7%\2\2\u0285\u0286\5<\37\2\u0286\u0287\7\4"+ + "\2\2\u0287K\3\2\2\2\u0288\u028e\5N(\2\u0289\u028a\7_\2\2\u028a\u028b\5"+ + "N(\2\u028b\u028c\7e\2\2\u028c\u028e\3\2\2\2\u028d\u0288\3\2\2\2\u028d"+ + "\u0289\3\2\2\2\u028eM\3\2\2\2\u028f\u0290\5P)\2\u0290\u029c\7\3\2\2\u0291"+ + "\u0293\5\36\20\2\u0292\u0291\3\2\2\2\u0292\u0293\3\2\2\2\u0293\u0294\3"+ + "\2\2\2\u0294\u0299\5,\27\2\u0295\u0296\7\5\2\2\u0296\u0298\5,\27\2\u0297"+ + "\u0295\3\2\2\2\u0298\u029b\3\2\2\2\u0299\u0297\3\2\2\2\u0299\u029a\3\2"+ + "\2\2\u029a\u029d\3\2\2\2\u029b\u0299\3\2\2\2\u029c\u0292\3\2\2\2\u029c"+ + "\u029d\3\2\2\2\u029d\u029e\3\2\2\2\u029e\u029f\7\4\2\2\u029fO\3\2\2\2"+ + "\u02a0\u02a4\7\63\2\2\u02a1\u02a4\7H\2\2\u02a2\u02a4\5`\61\2\u02a3\u02a0"+ + "\3\2\2\2\u02a3\u02a1\3\2\2\2\u02a3\u02a2\3\2\2\2\u02a4Q\3\2\2\2\u02a5"+ + "\u02c0\7>\2\2\u02a6\u02c0\5X-\2\u02a7\u02c0\5h\65\2\u02a8\u02c0\5V,\2"+ + "\u02a9\u02ab\7u\2\2\u02aa\u02a9\3\2\2\2\u02ab\u02ac\3\2\2\2\u02ac\u02aa"+ + "\3\2\2\2\u02ac\u02ad\3\2\2\2\u02ad\u02c0\3\2\2\2\u02ae\u02c0\7t\2\2\u02af"+ + "\u02b0\7a\2\2\u02b0\u02b1\5j\66\2\u02b1\u02b2\7e\2\2\u02b2\u02c0\3\2\2"+ + "\2\u02b3\u02b4\7b\2\2\u02b4\u02b5\5j\66\2\u02b5\u02b6\7e\2\2\u02b6\u02c0"+ + "\3\2\2\2\u02b7\u02b8\7c\2\2\u02b8\u02b9\5j\66\2\u02b9\u02ba\7e\2\2\u02ba"+ + "\u02c0\3\2\2\2\u02bb\u02bc\7d\2\2\u02bc\u02bd\5j\66\2\u02bd\u02be\7e\2"+ + "\2\u02be\u02c0\3\2\2\2\u02bf\u02a5\3\2\2\2\u02bf\u02a6\3\2\2\2\u02bf\u02a7"+ + "\3\2\2\2\u02bf\u02a8\3\2\2\2\u02bf\u02aa\3\2\2\2\u02bf\u02ae\3\2\2\2\u02bf"+ + "\u02af\3\2\2\2\u02bf\u02b3\3\2\2\2\u02bf\u02b7\3\2\2\2\u02bf\u02bb\3\2"+ + "\2\2\u02c0S\3\2\2\2\u02c1\u02c2\t\r\2\2\u02c2U\3\2\2\2\u02c3\u02c4\t\16"+ + "\2\2\u02c4W\3\2\2\2\u02c5\u02c7\7/\2\2\u02c6\u02c8\t\7\2\2\u02c7\u02c6"+ + "\3\2\2\2\u02c7\u02c8\3\2\2\2\u02c8\u02cb\3\2\2\2\u02c9\u02cc\5h\65\2\u02ca"+ + "\u02cc\5j\66\2\u02cb\u02c9\3\2\2\2\u02cb\u02ca\3\2\2\2\u02cc\u02cd\3\2"+ + "\2\2\u02cd\u02d0\5Z.\2\u02ce\u02cf\7U\2\2\u02cf\u02d1\5Z.\2\u02d0\u02ce"+ + "\3\2\2\2\u02d0\u02d1\3\2\2\2\u02d1Y\3\2\2\2\u02d2\u02d3\t\17\2\2\u02d3"+ + "[\3\2\2\2\u02d4\u02d5\5`\61\2\u02d5]\3\2\2\2\u02d6\u02d7\5`\61\2\u02d7"+ + "\u02d8\7s\2\2\u02d8\u02da\3\2\2\2\u02d9\u02d6\3\2\2\2\u02da\u02dd\3\2"+ + "\2\2\u02db\u02d9\3\2\2\2\u02db\u02dc\3\2\2\2\u02dc\u02de\3\2\2\2\u02dd"+ + "\u02db\3\2\2\2\u02de\u02df\5`\61\2\u02df_\3\2\2\2\u02e0\u02e3\5d\63\2"+ + "\u02e1\u02e3\5f\64\2\u02e2\u02e0\3\2\2\2\u02e2\u02e1\3\2\2\2\u02e3a\3"+ + "\2\2\2\u02e4\u02e5\5`\61\2\u02e5\u02e6\7\6\2\2\u02e6\u02e8\3\2\2\2\u02e7"+ + "\u02e4\3\2\2\2\u02e7\u02e8\3\2\2\2\u02e8\u02e9\3\2\2\2\u02e9\u02f1\7z"+ + "\2\2\u02ea\u02eb\5`\61\2\u02eb\u02ec\7\6\2\2\u02ec\u02ee\3\2\2\2\u02ed"+ + "\u02ea\3\2\2\2\u02ed\u02ee\3\2\2\2\u02ee\u02ef\3\2\2\2\u02ef\u02f1\5`"+ + "\61\2\u02f0\u02e7\3\2\2\2\u02f0\u02ed\3\2\2\2\u02f1c\3\2\2\2\u02f2\u02f5"+ + "\7{\2\2\u02f3\u02f5\7|\2\2\u02f4\u02f2\3\2\2\2\u02f4\u02f3\3\2\2\2\u02f5"+ + "e\3\2\2\2\u02f6\u02fa\7x\2\2\u02f7\u02fa\5l\67\2\u02f8\u02fa\7y\2\2\u02f9"+ + "\u02f6\3\2\2\2\u02f9\u02f7\3\2\2\2\u02f9\u02f8\3\2\2\2\u02fag\3\2\2\2"+ + "\u02fb\u02fe\7w\2\2\u02fc\u02fe\7v\2\2\u02fd\u02fb\3\2\2\2\u02fd\u02fc"+ + "\3\2\2\2\u02fei\3\2\2\2\u02ff\u0300\t\20\2\2\u0300k\3\2\2\2\u0301\u0302"+ + "\t\21\2\2\u0302m\3\2\2\2j}\177\u0083\u008c\u008e\u0092\u0099\u00a0\u00a5"+ + "\u00aa\u00b2\u00b6\u00be\u00c1\u00c7\u00cc\u00cf\u00d4\u00d7\u00d9\u00e1"+ + "\u00e4\u00f0\u00f3\u00f6\u00fd\u0104\u0108\u010c\u0110\u0117\u011b\u011f"+ + "\u0124\u0128\u0130\u0134\u013b\u0146\u0149\u014d\u0159\u015c\u0162\u0169"+ + "\u0170\u0173\u0177\u017b\u017f\u0181\u018c\u0191\u0195\u0198\u019e\u01a1"+ + "\u01a7\u01aa\u01ac\u01cf\u01d7\u01d9\u01e0\u01e5\u01e8\u01f0\u01f9\u01ff"+ + "\u0207\u020c\u0212\u0215\u021c\u0224\u022a\u0236\u0238\u0242\u024f\u025b"+ + "\u0267\u026c\u026f\u0271\u027f\u028d\u0292\u0299\u029c\u02a3\u02ac\u02bf"+ + "\u02c7\u02cb\u02d0\u02db\u02e2\u02e7\u02ed\u02f0\u02f4\u02f9\u02fd"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTests.java new file mode 100644 index 0000000000000..5eaa9ccd6c268 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.Protocol; +import org.elasticsearch.xpack.sql.session.Configuration; +import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase; +import org.elasticsearch.xpack.sql.tree.Source; + +public class CurrentDateTests extends AbstractNodeTestCase { + + public static CurrentDate randomCurrentDate() { + return new CurrentDate(Source.EMPTY, new Configuration(randomZone(), Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null)); + } + + @Override + protected CurrentDate randomInstance() { + return randomCurrentDate(); + } + + @Override + protected CurrentDate copy(CurrentDate instance) { + return new CurrentDate(instance.source(), instance.configuration()); + } + + @Override + protected CurrentDate mutate(CurrentDate instance) { + return new CurrentDate(instance.source(), new Configuration(randomZone(), Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null)); + } + + @Override + public void testTransform() { + } + + @Override + public void testReplaceChildren() { + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java index c5cdb06724bf8..28b99c1863486 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java @@ -6,23 +6,61 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.Protocol; +import org.elasticsearch.xpack.sql.session.Configuration; +import org.elasticsearch.xpack.sql.tree.AbstractNodeTestCase; import java.time.ZonedDateTime; -public class CurrentDateTimeTests extends ESTestCase { +import static org.elasticsearch.xpack.sql.tree.Source.EMPTY; - public void testNanoPrecision() throws Exception { +public class CurrentDateTimeTests extends AbstractNodeTestCase { + + public static CurrentDateTime randomCurrentDateTime() { + return new CurrentDateTime(EMPTY, Literal.of(EMPTY, randomInt(10)), + new Configuration(randomZone(), Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null)); + } + + @Override + protected CurrentDateTime randomInstance() { + return randomCurrentDateTime(); + } + + @Override + protected CurrentDateTime copy(CurrentDateTime instance) { + return new CurrentDateTime(instance.source(), instance.precision(), instance.configuration()); + } + + @Override + protected CurrentDateTime mutate(CurrentDateTime instance) { + return new CurrentDateTime(instance.source(), Literal.of(EMPTY, randomInt(10)), + new Configuration(randomZone(), Protocol.FETCH_SIZE, + Protocol.REQUEST_TIMEOUT, Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null)); + } + + @Override + public void testTransform() { + } + + @Override + public void testReplaceChildren() { + } + + public void testNanoPrecision() { ZonedDateTime zdt = ZonedDateTime.parse("2018-01-23T12:34:45.123456789Z"); - assertEquals(000_000_000, CurrentDateTime.nanoPrecision(zdt, 0).getNano()); - assertEquals(100_000_000, CurrentDateTime.nanoPrecision(zdt, 1).getNano()); - assertEquals(120_000_000, CurrentDateTime.nanoPrecision(zdt, 2).getNano()); - assertEquals(123_000_000, CurrentDateTime.nanoPrecision(zdt, 3).getNano()); - assertEquals(123_400_000, CurrentDateTime.nanoPrecision(zdt, 4).getNano()); - assertEquals(123_450_000, CurrentDateTime.nanoPrecision(zdt, 5).getNano()); - assertEquals(123_456_000, CurrentDateTime.nanoPrecision(zdt, 6).getNano()); - assertEquals(123_456_700, CurrentDateTime.nanoPrecision(zdt, 7).getNano()); - assertEquals(123_456_780, CurrentDateTime.nanoPrecision(zdt, 8).getNano()); - assertEquals(123_456_789, CurrentDateTime.nanoPrecision(zdt, 9).getNano()); + assertEquals(000_000_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 0)).getNano()); + assertEquals(100_000_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 1)).getNano()); + assertEquals(120_000_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 2)).getNano()); + assertEquals(123_000_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 3)).getNano()); + assertEquals(123_400_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 4)).getNano()); + assertEquals(123_450_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 5)).getNano()); + assertEquals(123_456_000, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 6)).getNano()); + assertEquals(123_456_700, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 7)).getNano()); + assertEquals(123_456_780, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 8)).getNano()); + assertEquals(123_456_789, CurrentDateTime.nanoPrecision(zdt, Literal.of(EMPTY, 9)).getNano()); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java index 2926f380ec237..229d39705cd82 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java @@ -338,6 +338,22 @@ public void testConvertWithInvalidESDataType() { assertEquals("line 1:13: Invalid data type [INVALID] provided", ex.getMessage()); } + public void testCurrentDate() { + Expression expr = parser.createExpression("CURRENT_DATE"); + assertEquals(UnresolvedFunction.class, expr.getClass()); + UnresolvedFunction ur = (UnresolvedFunction) expr; + assertEquals("CURRENT_DATE", ur.sourceText()); + assertEquals(0, ur.children().size()); + } + + public void testCurrentDateWithParentheses() { + Expression expr = parser.createExpression("CURRENT_DATE( )"); + assertEquals(UnresolvedFunction.class, expr.getClass()); + UnresolvedFunction ur = (UnresolvedFunction) expr; + assertEquals("CURRENT_DATE( )", ur.sourceText()); + assertEquals(0, ur.children().size()); + } + public void testCurrentTimestamp() { Expression expr = parser.createExpression("CURRENT_TIMESTAMP"); assertEquals(UnresolvedFunction.class, expr.getClass()); @@ -373,4 +389,4 @@ public void testSourceFunction() throws Exception { Expression expr = parser.createExpression(s); assertEquals(s, expr.sourceText()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java index 03f1a032cdab2..f11519f0f973f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java @@ -92,8 +92,7 @@ */ public class NodeSubclassTests> extends ESTestCase { - - private static final List> CLASSES_WITH_MIN_TWO_CHILDREN = Arrays.> asList(IfNull.class, In.class, InPipe.class, + private static final List> CLASSES_WITH_MIN_TWO_CHILDREN = Arrays.asList(IfNull.class, In.class, InPipe.class, Percentile.class, Percentiles.class, PercentileRanks.class); private final Class subclass; @@ -138,9 +137,7 @@ public void testTransform() throws Exception { Type changedArgType = argTypes[changedArgOffset]; Object changedArgValue = randomValueOtherThan(nodeCtorArgs[changedArgOffset], () -> makeArg(changedArgType)); - B transformed = node.transformNodeProps(prop -> { - return Objects.equals(prop, originalArgValue) ? changedArgValue : prop; - }, Object.class); + B transformed = node.transformNodeProps(prop -> Objects.equals(prop, originalArgValue) ? changedArgValue : prop, Object.class); if (node.children().contains(originalArgValue) || node.children().equals(originalArgValue)) { if (node.children().equals(emptyList()) && originalArgValue.equals(emptyList())) { From 9dde53b4a9c8436ecbd5fa0d21d4977ce8847163 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 5 Feb 2019 10:04:19 -0700 Subject: [PATCH 09/25] Support unknown fields in ingest pipeline map configuration (#38429) Backport of #38352 We already support unknown objects in the list of pipelines, this changes the `PipelineConfiguration` to support fields other than just `id` and `config`. Relates to #36938 --- .../action/ingest/GetPipelineResponse.java | 19 +++++++----- .../ingest/PipelineConfiguration.java | 8 ++++- .../ingest/PipelineConfigurationTests.java | 31 +++++++++++++++++-- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java index 297a7f0efc1d2..fdbd241431394 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -109,13 +110,12 @@ public static GetPipelineResponse fromXContent(XContentParser parser) throws IOE while(parser.nextToken().equals(Token.FIELD_NAME)) { String pipelineId = parser.currentName(); parser.nextToken(); - XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent()); - contentBuilder.generator().copyCurrentStructure(parser); - PipelineConfiguration pipeline = - new PipelineConfiguration( - pipelineId, BytesReference.bytes(contentBuilder), contentBuilder.contentType() - ); - pipelines.add(pipeline); + try (XContentBuilder contentBuilder = XContentBuilder.builder(parser.contentType().xContent())) { + contentBuilder.generator().copyCurrentStructure(parser); + PipelineConfiguration pipeline = + new PipelineConfiguration(pipelineId, BytesReference.bytes(contentBuilder), contentBuilder.contentType()); + pipelines.add(pipeline); + } } ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); return new GetPipelineResponse(pipelines); @@ -148,6 +148,11 @@ public boolean equals(Object other) { } } + @Override + public String toString() { + return Strings.toString(this); + } + @Override public int hashCode() { int result = 1; diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index a2aa8e385e3f9..582bc8c46b455 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,7 +43,7 @@ */ public final class PipelineConfiguration extends AbstractDiffable implements ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("pipeline_config", Builder::new); + private static final ObjectParser PARSER = new ObjectParser<>("pipeline_config", true, Builder::new); static { PARSER.declareString(Builder::setId, new ParseField("id")); PARSER.declareField((parser, builder, aVoid) -> { @@ -130,6 +131,11 @@ public static Diff readDiffFrom(StreamInput in) throws IO return readDiffFrom(PipelineConfiguration::readFrom, in); } + @Override + public String toString() { + return Strings.toString(this); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(id); diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java index 7b134879cda45..eb1171f66a597 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java @@ -31,12 +31,13 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.function.Predicate; -public class PipelineConfigurationTests extends ESTestCase { +public class PipelineConfigurationTests extends AbstractXContentTestCase { public void testSerialization() throws IOException { PipelineConfiguration configuration = new PipelineConfiguration("1", @@ -68,4 +69,30 @@ public void testParser() throws IOException { assertEquals("{}", XContentHelper.convertToJson(parsed.getConfig(), false, parsed.getXContentType())); assertEquals("1", parsed.getId()); } + + @Override + protected PipelineConfiguration createTestInstance() { + BytesArray config; + if (randomBoolean()) { + config = new BytesArray("{}".getBytes(StandardCharsets.UTF_8)); + } else { + config = new BytesArray("{\"foo\": \"bar\"}".getBytes(StandardCharsets.UTF_8)); + } + return new PipelineConfiguration(randomAlphaOfLength(4), config, XContentType.JSON); + } + + @Override + protected PipelineConfiguration doParseInstance(XContentParser parser) throws IOException { + return PipelineConfiguration.getParser().parse(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> field.equals("config"); + } } From 3e0819a9112cf0de738f8d8e0fa0e22316c53916 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 18:06:12 +0100 Subject: [PATCH 10/25] Backport of #38411: `if_seq_no` and `if_primary_term` parameters aren't wired correctly in REST Client's CRUD API --- .../client/RequestConverters.java | 9 ++++ .../java/org/elasticsearch/client/CrudIT.java | 53 +++++++++++++++---- .../client/RequestConvertersTests.java | 39 +++++++++++--- .../rest-api-spec/test/bulk/80_cas.yml | 45 ++++++++++++++++ 4 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 5af5e9ed9d825..ee5948d64a78d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -109,6 +109,8 @@ static Request delete(DeleteRequest deleteRequest) { parameters.withTimeout(deleteRequest.timeout()); parameters.withVersion(deleteRequest.version()); parameters.withVersionType(deleteRequest.versionType()); + parameters.withIfSeqNo(deleteRequest.ifSeqNo()); + parameters.withIfPrimaryTerm(deleteRequest.ifPrimaryTerm()); parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); return request; @@ -193,6 +195,11 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { } } + if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) { + metadata.field("if_seq_no", action.ifSeqNo()); + metadata.field("if_primary_term", action.ifPrimaryTerm()); + } + if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { IndexRequest indexRequest = (IndexRequest) action; if (Strings.hasLength(indexRequest.getPipeline())) { @@ -309,6 +316,8 @@ static Request index(IndexRequest indexRequest) { parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); + parameters.withIfSeqNo(indexRequest.ifSeqNo()); + parameters.withIfPrimaryTerm(indexRequest.ifPrimaryTerm()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index e88ce713c686b..27e6221f0ec84 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.client.core.MultiTermVectorsResponse; import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -95,14 +96,20 @@ public class CrudIT extends ESRestHighLevelClientTestCase { public void testDelete() throws IOException { + highLevelClient().indices().create( + new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), + RequestOptions.DEFAULT); { // Testing deletion String docId = "id"; - highLevelClient().index( + IndexResponse indexResponse = highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); if (randomBoolean()) { - deleteRequest.version(1L); + deleteRequest.setIfSeqNo(indexResponse.getSeqNo()); + deleteRequest.setIfPrimaryTerm(indexResponse.getPrimaryTerm()); + } else { + deleteRequest.version(indexResponse.getVersion()); } DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, highLevelClient()::delete, highLevelClient()::deleteAsync); @@ -127,13 +134,26 @@ public void testDelete() throws IOException { String docId = "version_conflict"; highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); - DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).version(2); + DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); + final boolean seqNos = randomBoolean(); + if (seqNos) { + deleteRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + } else { + deleteRequest.version(2); + } + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, highLevelClient()::delete, highLevelClient()::deleteAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); - assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + - "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + if (seqNos) { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + + "version conflict, required seqNo [2], primary term [2]. current document has seqNo [3] and primary term [1]]", + exception.getMessage()); + } else { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + } assertEquals("index", exception.getMetadata("es.index").get(0)); } { @@ -453,18 +473,29 @@ public void testIndex() throws IOException { assertEquals("type", indexResponse.getType()); assertEquals("id", indexResponse.getId()); assertEquals(2L, indexResponse.getVersion()); + final boolean seqNosForConflict = randomBoolean(); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { IndexRequest wrongRequest = new IndexRequest("index", "type", "id"); wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); - wrongRequest.version(5L); + if (seqNosForConflict) { + wrongRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + } else { + wrongRequest.version(5); + } execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync, highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); - assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + - "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + if (seqNosForConflict) { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + + "version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]", + exception.getMessage()); + } else { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + + "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + } assertEquals("index", exception.getMetadata("es.index").get(0)); } { @@ -763,7 +794,8 @@ public void testBulk() throws IOException { if (opType == DocWriteRequest.OpType.INDEX) { IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType); if (erroneous) { - indexRequest.version(12L); + indexRequest.setIfSeqNo(12L); + indexRequest.setIfPrimaryTerm(12L); } bulkRequest.add(indexRequest); @@ -1075,7 +1107,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) if (opType == DocWriteRequest.OpType.INDEX) { IndexRequest indexRequest = new IndexRequest("index", "test", id).source(xContentType, "id", i); if (erroneous) { - indexRequest.version(12L); + indexRequest.setIfSeqNo(12L); + indexRequest.setIfPrimaryTerm(12L); } processor.add(indexRequest); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 78a16cbc1517c..4b15774568a6a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -53,11 +53,11 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; -import org.elasticsearch.client.core.MultiTermVectorsRequest; -import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestConverters.EndpointBuilder; import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.core.MultiTermVectorsRequest; +import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -216,6 +216,7 @@ public void testDelete() { setRandomRefreshPolicy(deleteRequest::setRefreshPolicy, expectedParams); setRandomVersion(deleteRequest, expectedParams); setRandomVersionType(deleteRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(deleteRequest, expectedParams); if (frequently()) { if (randomBoolean()) { @@ -545,6 +546,7 @@ public void testIndex() throws IOException { } else { setRandomVersion(indexRequest, expectedParams); setRandomVersionType(indexRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(indexRequest, expectedParams); } if (frequently()) { @@ -650,6 +652,7 @@ public void testUpdate() throws IOException { setRandomWaitForActiveShards(updateRequest::waitForActiveShards, ActiveShardCount.DEFAULT, expectedParams); setRandomVersion(updateRequest, expectedParams); setRandomVersionType(updateRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(updateRequest, new HashMap<>()); // if* params are passed in the body if (randomBoolean()) { int retryOnConflict = randomIntBetween(0, 5); updateRequest.retryOnConflict(retryOnConflict); @@ -680,6 +683,7 @@ public void testUpdate() throws IOException { assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert()); assertEquals(updateRequest.detectNoop(), parsedUpdateRequest.detectNoop()); assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource()); + assertIfSeqNoAndTerm(updateRequest, parsedUpdateRequest); assertEquals(updateRequest.script(), parsedUpdateRequest.script()); if (updateRequest.doc() != null) { assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType); @@ -693,6 +697,22 @@ public void testUpdate() throws IOException { } } + private static void assertIfSeqNoAndTerm(DocWriteRequestrequest, DocWriteRequest parsedRequest) { + assertEquals(request.ifSeqNo(), parsedRequest.ifSeqNo()); + assertEquals(request.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm()); + } + + private static void setRandomIfSeqNoAndTerm(DocWriteRequest request, Map expectedParams) { + if (randomBoolean()) { + final long seqNo = randomNonNegativeLong(); + request.setIfSeqNo(seqNo); + expectedParams.put("if_seq_no", Long.toString(seqNo)); + final long primaryTerm = randomLongBetween(1, 200); + request.setIfPrimaryTerm(primaryTerm); + expectedParams.put("if_primary_term", Long.toString(primaryTerm)); + } + } + public void testUpdateWithDifferentContentTypes() { IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { UpdateRequest updateRequest = new UpdateRequest(); @@ -767,10 +787,15 @@ public void testBulk() throws IOException { docWriteRequest.routing(randomAlphaOfLength(10)); } if (randomBoolean()) { - docWriteRequest.version(randomNonNegativeLong()); - } - if (randomBoolean()) { - docWriteRequest.versionType(randomFrom(VersionType.values())); + if (randomBoolean()) { + docWriteRequest.version(randomNonNegativeLong()); + } + if (randomBoolean()) { + docWriteRequest.versionType(randomFrom(VersionType.values())); + } + } else if (randomBoolean()) { + docWriteRequest.setIfSeqNo(randomNonNegativeLong()); + docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200)); } bulkRequest.add(docWriteRequest); } @@ -801,6 +826,8 @@ public void testBulk() throws IOException { assertEquals(originalRequest.parent(), parsedRequest.parent()); assertEquals(originalRequest.version(), parsedRequest.version()); assertEquals(originalRequest.versionType(), parsedRequest.versionType()); + assertEquals(originalRequest.ifSeqNo(), parsedRequest.ifSeqNo()); + assertEquals(originalRequest.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm()); DocWriteRequest.OpType opType = originalRequest.opType(); if (opType == DocWriteRequest.OpType.INDEX) { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml new file mode 100644 index 0000000000000..14237a1b9dc57 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml @@ -0,0 +1,45 @@ +--- +"Compare And Swap Sequence Numbers": + + - skip: + version: " - 6.5.99" + reason: cas operations with sequence numbers was added in 6.6 + + - do: + index: + index: test_1 + type: _doc + id: 1 + body: { foo: bar } + - match: { _version: 1} + - set: { _seq_no: seqno } + - set: { _primary_term: primary_term } + + - do: + bulk: + body: + - index: + _index: test_1 + _type: _doc + _id: 1 + if_seq_no: 10000 + if_primary_term: $primary_term + - foo: bar2 + + - match: { errors: true } + - match: { items.0.index.status: 409 } + - match: { items.0.index.error.type: version_conflict_engine_exception } + + - do: + bulk: + body: + - index: + _index: test_1 + _type: _doc + _id: 1 + if_seq_no: $seqno + if_primary_term: $primary_term + - foo: bar2 + + - match: { errors: false} + - match: { items.0.index.status: 200 } From 6d16374db5415ce66c22bcdc2e2db7fe63a36fec Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Tue, 5 Feb 2019 12:51:20 -0600 Subject: [PATCH 11/25] Update Rollup Caps to allow unknown fields (#38446) This commit ensures that the parts of rollup caps that can allow unknown fields will allow them. It also modifies the test such that we can use the features we need for disallowing fields in spots where they would not be allowed. Relates #36938 Backport of #38339 --- .../client/rollup/RollableIndexCaps.java | 2 +- .../client/rollup/RollupJobCaps.java | 26 +++++-------------- .../rollup/GetRollupCapsResponseTests.java | 10 +++++++ .../GetRollupIndexCapsResponseTests.java | 10 +++++++ .../rollup/RollupCapsResponseTestCase.java | 16 +++++++++--- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java index cf849e38dd0b4..8e0bea0996bbd 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java @@ -44,7 +44,7 @@ public class RollableIndexCaps implements ToXContentFragment { public static final Function> PARSER = indexName -> { @SuppressWarnings("unchecked") ConstructingObjectParser p - = new ConstructingObjectParser<>(indexName, + = new ConstructingObjectParser<>(indexName, true, a -> new RollableIndexCaps(indexName, (List) a[0])); p.declareObjectArray(ConstructingObjectParser.constructorArg(), RollupJobCaps.PARSER::apply, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java index 7ba1aaa4d7c2b..15161069f7338 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Function; +import java.util.stream.Collectors; /** * Represents the Rollup capabilities for a specific job on a single rollup index @@ -45,15 +45,12 @@ public class RollupJobCaps implements ToXContentObject { private static final ParseField FIELDS = new ParseField("fields"); private static final String NAME = "rollup_job_caps"; - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, a -> { @SuppressWarnings("unchecked") List> caps = (List>) a[3]; - if (caps.isEmpty()) { - return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], Collections.emptyMap()); - } - Map mapCaps = new HashMap<>(caps.size()); - caps.forEach(c -> mapCaps.put(c.v1(), c.v2())); + Map mapCaps = + new HashMap<>(caps.stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], mapCaps); }); @@ -140,16 +137,6 @@ public static class RollupFieldCaps implements ToXContentFragment { private static final String NAME = "rollup_field_caps"; private final List> aggs; - public static final Function> PARSER = fieldName -> { - @SuppressWarnings("unchecked") - ConstructingObjectParser parser - = new ConstructingObjectParser<>(NAME, a -> new RollupFieldCaps((List>) a[0])); - - parser.declareObjectArray(ConstructingObjectParser.constructorArg(), - (p, c) -> p.map(), new ParseField(fieldName)); - return parser; - }; - RollupFieldCaps(final List> aggs) { this.aggs = Collections.unmodifiableList(Objects.requireNonNull(aggs)); } @@ -170,13 +157,12 @@ public static RollupFieldCaps fromXContent(XContentParser parser) throws IOExcep List> aggs = new ArrayList<>(); if (parser.nextToken().equals(XContentParser.Token.START_ARRAY)) { while (parser.nextToken().equals(XContentParser.Token.START_OBJECT)) { - aggs.add(Collections.unmodifiableMap(parser.map())); + aggs.add(parser.map()); } } - return new RollupFieldCaps(Collections.unmodifiableList(aggs)); + return new RollupFieldCaps(aggs); } - @Override public boolean equals(Object other) { if (this == other) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java index a728b65cf64ce..a9c3a59faf5ae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Predicate; public class GetRollupCapsResponseTests extends RollupCapsResponseTestCase { @@ -40,6 +41,15 @@ protected void toXContent(GetRollupCapsResponse response, XContentBuilder builde builder.endObject(); } + @Override + protected Predicate randomFieldsExcludeFilter() { + return (field) -> + // base cannot have extra things in it + "".equals(field) + // the field list expects to be a nested object of a certain type + || field.contains("fields"); + } + @Override protected GetRollupCapsResponse fromXContent(XContentParser parser) throws IOException { return GetRollupCapsResponse.fromXContent(parser); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupIndexCapsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupIndexCapsResponseTests.java index afd0e54f92b1f..20e29aef0df64 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupIndexCapsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupIndexCapsResponseTests.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Predicate; public class GetRollupIndexCapsResponseTests extends RollupCapsResponseTestCase { @@ -40,6 +41,15 @@ protected void toXContent(GetRollupIndexCapsResponse response, XContentBuilder b builder.endObject(); } + @Override + protected Predicate randomFieldsExcludeFilter() { + return (field) -> + // base cannot have extra things in it + "".equals(field) + // the field list expects to be a nested object of a certain type + || field.contains("fields"); + } + @Override protected GetRollupIndexCapsResponse fromXContent(XContentParser parser) throws IOException { return GetRollupIndexCapsResponse.fromXContent(parser); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/RollupCapsResponseTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/RollupCapsResponseTestCase.java index 6d1c0359d172d..cdc4280dbff91 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/RollupCapsResponseTestCase.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/RollupCapsResponseTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.client.rollup.job.config.RollupJobConfig; import org.elasticsearch.client.rollup.job.config.RollupJobConfigTests; import org.elasticsearch.client.rollup.job.config.TermsGroupConfig; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -40,6 +41,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Collectors; import static java.util.Collections.singletonMap; @@ -55,15 +57,23 @@ abstract class RollupCapsResponseTestCase extends ESTestCase { protected abstract T fromXContent(XContentParser parser) throws IOException; + protected Predicate randomFieldsExcludeFilter() { + return field -> false; + } + + protected String[] shuffleFieldsExceptions() { + return Strings.EMPTY_ARRAY; + } + public void testFromXContent() throws IOException { xContentTester( this::createParser, this::createTestInstance, this::toXContent, this::fromXContent) - .supportsUnknownFields(false) - .randomFieldsExcludeFilter(field -> - field.endsWith("job_id")) + .supportsUnknownFields(true) + .randomFieldsExcludeFilter(randomFieldsExcludeFilter()) + .shuffleFieldsExceptions(shuffleFieldsExceptions()) .test(); } From 99192e7f0ca30be9b43fc26e38345ea4f70a28c8 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Tue, 5 Feb 2019 20:14:08 +0100 Subject: [PATCH 12/25] Deprecate maxRetryTimeout in RestClient and increase default value (#38425) This commit deprecates the `maxRetryTimeout` settings in the low-level REST client, and increases its default value from 30 seconds to 90 seconds. The goal of this is to have it set higher than the socket timeout so that users get as few listener timeouts as possible. Relates to #38085 --- .../org/elasticsearch/client/RestClientBuilder.java | 5 +++-- .../client/documentation/RestClientDocumentation.java | 3 +-- docs/java-rest/low-level/configuration.asciidoc | 3 +-- docs/java-rest/low-level/usage.asciidoc | 6 +++--- .../AzureStorageRepositoryClientYamlTestSuiteIT.java | 1 - .../upgrades/AbstractRollingTestCase.java | 1 - .../upgrades/UpgradeClusterClientYamlTestSuiteIT.java | 1 - .../org/elasticsearch/test/rest/ESRestTestCase.java | 6 ------ .../xpack/restart/FullClusterRestartIT.java | 1 - .../authc/kerberos/KerberosAuthenticationIT.java | 11 ++--------- .../upgrades/UpgradeClusterClientYamlTestSuiteIT.java | 1 - 11 files changed, 10 insertions(+), 29 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java index 84cc3ee1667b1..fcb8bddefc964 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java @@ -42,7 +42,7 @@ public final class RestClientBuilder { public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000; public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 30000; - public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS; + public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = 90000; public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10; public static final int DEFAULT_MAX_CONN_TOTAL = 30; @@ -105,9 +105,10 @@ public RestClientBuilder setFailureListener(RestClient.FailureListener failureLi /** * Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request. * {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified. - * + * @deprecated this setting is deprecated and will be removed in the future in favour of relying on socket and connect timeout * @throws IllegalArgumentException if {@code maxRetryTimeoutMillis} is not greater than 0 */ + @Deprecated public RestClientBuilder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) { if (maxRetryTimeoutMillis <= 0) { throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0"); diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index 7eae17d83cf2b..f4eaf4fa75b4a 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -305,8 +305,7 @@ public RequestConfig.Builder customizeRequestConfig( .setConnectTimeout(5000) .setSocketTimeout(60000); } - }) - .setMaxRetryTimeoutMillis(60000); + }); //end::rest-client-config-timeouts } { diff --git a/docs/java-rest/low-level/configuration.asciidoc b/docs/java-rest/low-level/configuration.asciidoc index b7da2b5ebccff..e284b52c67a67 100644 --- a/docs/java-rest/low-level/configuration.asciidoc +++ b/docs/java-rest/low-level/configuration.asciidoc @@ -18,8 +18,7 @@ https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/htt as an argument and has the same return type. The request config builder can be modified and then returned. In the following example we increase the connect timeout (defaults to 1 second) and the socket timeout (defaults to 30 -seconds). Also we adjust the max retry timeout accordingly (defaults to 30 -seconds too). +seconds). ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- diff --git a/docs/java-rest/low-level/usage.asciidoc b/docs/java-rest/low-level/usage.asciidoc index 3747314b6ecd3..f1cb63998d374 100644 --- a/docs/java-rest/low-level/usage.asciidoc +++ b/docs/java-rest/low-level/usage.asciidoc @@ -185,9 +185,9 @@ prevent having to specify them with each single request include-tagged::{doc-tests}/RestClientDocumentation.java[rest-client-init-max-retry-timeout] -------------------------------------------------- <1> Set the timeout that should be honoured in case multiple attempts are made -for the same request. The default value is 30 seconds, same as the default -socket timeout. In case the socket timeout is customized, the maximum retry -timeout should be adjusted accordingly +for the same request. The default value is 90 seconds. In case the socket +timeout is set to a higher value, the max retry timeout should be adjusted +accordingly. deprecated[6.7.0, max retry timeout will be removed in 7.0.0] ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java index 720f152265a61..9822da98fde70 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java +++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java @@ -41,7 +41,6 @@ public static Iterable parameters() throws Exception { protected Settings restClientSettings() { // Give more time to repository-azure to complete the snapshot operations return Settings.builder().put(super.restClientSettings()) - .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "60s") .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s") .build(); } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java index 10ee480075bd9..25f8ba36021e7 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java @@ -69,7 +69,6 @@ protected final Settings restClientSettings() { // increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards - .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s") .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") .build(); } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index cd01bcd9e0e19..4ef8a266da150 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -65,7 +65,6 @@ protected Settings restClientSettings() { // increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards - .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s") .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") .build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 3db525037b49c..86552e0a0fbfa 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -92,7 +92,6 @@ public abstract class ESRestTestCase extends ESTestCase { public static final String TRUSTSTORE_PATH = "truststore.path"; public static final String TRUSTSTORE_PASSWORD = "truststore.password"; - public static final String CLIENT_RETRY_TIMEOUT = "client.retry.timeout"; public static final String CLIENT_SOCKET_TIMEOUT = "client.socket.timeout"; public static final String CLIENT_PATH_PREFIX = "client.path.prefix"; @@ -729,11 +728,6 @@ protected static void configureClient(RestClientBuilder builder, Settings settin } builder.setDefaultHeaders(defaultHeaders); } - final String requestTimeoutString = settings.get(CLIENT_RETRY_TIMEOUT); - if (requestTimeoutString != null) { - final TimeValue maxRetryTimeout = TimeValue.parseTimeValue(requestTimeoutString, CLIENT_RETRY_TIMEOUT); - builder.setMaxRetryTimeoutMillis(Math.toIntExact(maxRetryTimeout.getMillis())); - } final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); if (socketTimeoutString != null) { final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 6d77d85b1ca7c..276cdaaacebf9 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -63,7 +63,6 @@ protected Settings restClientSettings() { // we increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards - .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s") .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") .build(); } diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java index 17eb3592d584a..be4eb0f787cf7 100644 --- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java +++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; +import javax.security.auth.login.LoginContext; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; @@ -33,8 +34,6 @@ import java.util.List; import java.util.Map; -import javax.security.auth.login.LoginContext; - import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.contains; @@ -148,13 +147,7 @@ private RestClient buildRestClientForKerberos(final SpnegoHttpClientConfigCallba return restClientBuilder.build(); } - private static void configureRestClientBuilder(final RestClientBuilder restClientBuilder, final Settings settings) - throws IOException { - final String requestTimeoutString = settings.get(CLIENT_RETRY_TIMEOUT); - if (requestTimeoutString != null) { - final TimeValue maxRetryTimeout = TimeValue.parseTimeValue(requestTimeoutString, CLIENT_RETRY_TIMEOUT); - restClientBuilder.setMaxRetryTimeoutMillis(Math.toIntExact(maxRetryTimeout.getMillis())); - } + private static void configureRestClientBuilder(final RestClientBuilder restClientBuilder, final Settings settings) { final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); if (socketTimeoutString != null) { final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index 3c7aca675d684..a147b97105b87 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -103,7 +103,6 @@ protected Settings restClientSettings() { // we increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards - .put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s") .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") .build(); } From 401c45c54d9e1409dc2e3d86fef1bc55c8293539 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 5 Feb 2019 14:42:30 -0500 Subject: [PATCH 13/25] Lift retention lease expiration to index shard (#38391) This commit lifts the control of when retention leases are expired to index shard. In this case, we move expiration to an explicit action rather than a side-effect of calling `ReplicationTracker#getRetentionLeases`. This explicit action is invoked on a timer. If any retention leases expire, then we hard sync the retention leases to the replicas. Otherwise, we proceed with a background sync. --- .../org/elasticsearch/index/IndexService.java | 22 +-- .../elasticsearch/index/IndexSettings.java | 5 + .../index/seqno/ReplicationTracker.java | 76 +++++----- .../elasticsearch/index/shard/IndexShard.java | 26 +++- ...ReplicationTrackerRetentionLeaseTests.java | 134 ++++-------------- .../index/seqno/RetentionLeaseSyncIT.java | 72 +++++++--- .../shard/IndexShardRetentionLeaseTests.java | 69 +++++---- 7 files changed, 192 insertions(+), 212 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index fee8d06c49873..4cf384516abfb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -120,7 +120,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust private volatile AsyncRefreshTask refreshTask; private volatile AsyncTranslogFSync fsyncTask; private volatile AsyncGlobalCheckpointTask globalCheckpointTask; - private volatile AsyncRetentionLeaseBackgroundSyncTask retentionLeaseBackgroundSyncTask; + private volatile AsyncRetentionLeaseSyncTask retentionLeaseSyncTask; // don't convert to Setting<> and register... we only set this in tests and register via a plugin private final String INDEX_TRANSLOG_RETENTION_CHECK_INTERVAL_SETTING = "index.translog.retention.check_interval"; @@ -197,7 +197,7 @@ public IndexService( this.refreshTask = new AsyncRefreshTask(this); this.trimTranslogTask = new AsyncTrimTranslogTask(this); this.globalCheckpointTask = new AsyncGlobalCheckpointTask(this); - this.retentionLeaseBackgroundSyncTask = new AsyncRetentionLeaseBackgroundSyncTask(this); + this.retentionLeaseSyncTask = new AsyncRetentionLeaseSyncTask(this); rescheduleFsyncTask(indexSettings.getTranslogDurability()); } @@ -288,7 +288,7 @@ public synchronized void close(final String reason, boolean delete) throws IOExc fsyncTask, trimTranslogTask, globalCheckpointTask, - retentionLeaseBackgroundSyncTask); + retentionLeaseSyncTask); } } } @@ -770,8 +770,8 @@ private void maybeSyncGlobalCheckpoints() { sync(is -> is.maybeSyncGlobalCheckpoint("background"), "global checkpoint"); } - private void backgroundSyncRetentionLeases() { - sync(IndexShard::backgroundSyncRetentionLeases, "retention lease"); + private void syncRetentionLeases() { + sync(IndexShard::syncRetentionLeases, "retention lease"); } private void sync(final Consumer sync, final String source) { @@ -794,11 +794,11 @@ private void sync(final Consumer sync, final String source) { && e instanceof IndexShardClosedException == false) { logger.warn( new ParameterizedMessage( - "{} failed to execute background {} sync", shard.shardId(), source), e); + "{} failed to execute {} sync", shard.shardId(), source), e); } }, ThreadPool.Names.SAME, - "background " + source + " sync"); + source + " sync"); } catch (final AlreadyClosedException | IndexShardClosedException e) { // the shard was closed concurrently, continue } @@ -939,15 +939,15 @@ public String toString() { } } - final class AsyncRetentionLeaseBackgroundSyncTask extends BaseAsyncTask { + final class AsyncRetentionLeaseSyncTask extends BaseAsyncTask { - AsyncRetentionLeaseBackgroundSyncTask(final IndexService indexService) { + AsyncRetentionLeaseSyncTask(final IndexService indexService) { super(indexService, RETENTION_LEASE_SYNC_INTERVAL_SETTING.get(indexService.getIndexSettings().getSettings())); } @Override protected void runInternal() { - indexService.backgroundSyncRetentionLeases(); + indexService.syncRetentionLeases(); } @Override @@ -957,7 +957,7 @@ protected String getThreadPool() { @Override public String toString() { - return "retention_lease_background_sync"; + return "retention_lease_sync"; } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 80b7acc1799b8..27b9b1687db78 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -358,6 +358,10 @@ public long getRetentionLeaseMillis() { return retentionLeaseMillis; } + private void setRetentionLeaseMillis(final TimeValue retentionLease) { + this.retentionLeaseMillis = retentionLease.millis(); + } + private volatile boolean warmerEnabled; private volatile int maxResultWindow; private volatile int maxInnerResultWindow; @@ -546,6 +550,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(DEFAULT_PIPELINE, this::setDefaultPipeline); scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING, this::setSoftDeleteRetentionOperations); scopedSettings.addSettingsUpdateConsumer(INDEX_SEARCH_THROTTLED, this::setSearchThrottled); + scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_LEASE_SETTING, this::setRetentionLeaseMillis); } private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) { diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java index 3b68dfa6addae..31f491d24cf9d 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -155,10 +156,10 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L private final LongSupplier currentTimeMillisSupplier; /** - * A callback when a new retention lease is created or an existing retention lease expires. In practice, this callback invokes the - * retention lease sync action, to sync retention leases to replicas. + * A callback when a new retention lease is created. In practice, this callback invokes the retention lease sync action, to sync + * retention leases to replicas. */ - private final BiConsumer> onSyncRetentionLeases; + private final BiConsumer> onAddRetentionLease; /** * This set contains allocation IDs for which there is a thread actively waiting for the local checkpoint to advance to at least the @@ -177,43 +178,42 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L private RetentionLeases retentionLeases = RetentionLeases.EMPTY; /** - * Get all non-expired retention leases tracked on this shard. Note that only the primary shard calculates which leases are expired, - * and if any have expired, syncs the retention leases to any replicas. + * Get all retention leases tracked on this shard. * * @return the retention leases */ public RetentionLeases getRetentionLeases() { - final boolean wasPrimaryMode; - final RetentionLeases nonExpiredRetentionLeases; - synchronized (this) { - if (primaryMode) { - // the primary calculates the non-expired retention leases and syncs them to replicas - final long currentTimeMillis = currentTimeMillisSupplier.getAsLong(); - final long retentionLeaseMillis = indexSettings.getRetentionLeaseMillis(); - final Map> partitionByExpiration = retentionLeases - .leases() - .stream() - .collect(Collectors.groupingBy(lease -> currentTimeMillis - lease.timestamp() > retentionLeaseMillis)); - if (partitionByExpiration.get(true) == null) { - // early out as no retention leases have expired - return retentionLeases; - } - final Collection nonExpiredLeases = - partitionByExpiration.get(false) != null ? partitionByExpiration.get(false) : Collections.emptyList(); - retentionLeases = new RetentionLeases(operationPrimaryTerm, retentionLeases.version() + 1, nonExpiredLeases); - } - /* - * At this point, we were either in primary mode and have updated the non-expired retention leases into the tracking map, or - * we were in replica mode and merely need to copy the existing retention leases since a replica does not calculate the - * non-expired retention leases, instead receiving them on syncs from the primary. - */ - wasPrimaryMode = primaryMode; - nonExpiredRetentionLeases = retentionLeases; + return getRetentionLeases(false).v2(); + } + + /** + * If the expire leases parameter is false, gets all retention leases tracked on this shard and otherwise first calculates + * expiration of existing retention leases, and then gets all non-expired retention leases tracked on this shard. Note that only the + * primary shard calculates which leases are expired, and if any have expired, syncs the retention leases to any replicas. If the + * expire leases parameter is true, this replication tracker must be in primary mode. + * + * @return a tuple indicating whether or not any retention leases were expired, and the non-expired retention leases + */ + public synchronized Tuple getRetentionLeases(final boolean expireLeases) { + if (expireLeases == false) { + return Tuple.tuple(false, retentionLeases); } - if (wasPrimaryMode) { - onSyncRetentionLeases.accept(nonExpiredRetentionLeases, ActionListener.wrap(() -> {})); + assert primaryMode; + // the primary calculates the non-expired retention leases and syncs them to replicas + final long currentTimeMillis = currentTimeMillisSupplier.getAsLong(); + final long retentionLeaseMillis = indexSettings.getRetentionLeaseMillis(); + final Map> partitionByExpiration = retentionLeases + .leases() + .stream() + .collect(Collectors.groupingBy(lease -> currentTimeMillis - lease.timestamp() > retentionLeaseMillis)); + if (partitionByExpiration.get(true) == null) { + // early out as no retention leases have expired + return Tuple.tuple(false, retentionLeases); } - return nonExpiredRetentionLeases; + final Collection nonExpiredLeases = + partitionByExpiration.get(false) != null ? partitionByExpiration.get(false) : Collections.emptyList(); + retentionLeases = new RetentionLeases(operationPrimaryTerm, retentionLeases.version() + 1, nonExpiredLeases); + return Tuple.tuple(true, retentionLeases); } /** @@ -246,7 +246,7 @@ public RetentionLease addRetentionLease( Stream.concat(retentionLeases.leases().stream(), Stream.of(retentionLease)).collect(Collectors.toList())); currentRetentionLeases = retentionLeases; } - onSyncRetentionLeases.accept(currentRetentionLeases, listener); + onAddRetentionLease.accept(currentRetentionLeases, listener); return retentionLease; } @@ -563,7 +563,7 @@ private static long inSyncCheckpointStates( * @param indexSettings the index settings * @param operationPrimaryTerm the current primary term * @param globalCheckpoint the last known global checkpoint for this shard, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} - * @param onSyncRetentionLeases a callback when a new retention lease is created or an existing retention lease expires + * @param onAddRetentionLease a callback when a new retention lease is created or an existing retention lease expires */ public ReplicationTracker( final ShardId shardId, @@ -573,7 +573,7 @@ public ReplicationTracker( final long globalCheckpoint, final LongConsumer onGlobalCheckpointUpdated, final LongSupplier currentTimeMillisSupplier, - final BiConsumer> onSyncRetentionLeases) { + final BiConsumer> onAddRetentionLease) { super(shardId, indexSettings); assert globalCheckpoint >= SequenceNumbers.UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint; this.shardAllocationId = allocationId; @@ -585,7 +585,7 @@ public ReplicationTracker( checkpoints.put(allocationId, new CheckpointState(SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint, false, false)); this.onGlobalCheckpointUpdated = Objects.requireNonNull(onGlobalCheckpointUpdated); this.currentTimeMillisSupplier = Objects.requireNonNull(currentTimeMillisSupplier); - this.onSyncRetentionLeases = Objects.requireNonNull(onSyncRetentionLeases); + this.onAddRetentionLease = Objects.requireNonNull(onAddRetentionLease); this.pendingInSync = new HashSet<>(); this.routingTable = null; this.replicationGroup = null; diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 2c1ffd0dccc62..fe9c326ace0d2 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1929,13 +1929,26 @@ public void addGlobalCheckpointListener( } /** - * Get all non-expired retention leases tracked on this shard. + * Get all retention leases tracked on this shard. * * @return the retention leases */ public RetentionLeases getRetentionLeases() { + return getRetentionLeases(false).v2(); + } + + /** + * If the expire leases parameter is false, gets all retention leases tracked on this shard and otherwise first calculates + * expiration of existing retention leases, and then gets all non-expired retention leases tracked on this shard. Note that only the + * primary shard calculates which leases are expired, and if any have expired, syncs the retention leases to any replicas. If the + * expire leases parameter is true, this replication tracker must be in primary mode. + * + * @return a tuple indicating whether or not any retention leases were expired, and the non-expired retention leases + */ + public Tuple getRetentionLeases(final boolean expireLeases) { + assert expireLeases == false || assertPrimaryMode(); verifyNotClosed(); - return replicationTracker.getRetentionLeases(); + return replicationTracker.getRetentionLeases(expireLeases); } public RetentionLeaseStats getRetentionLeaseStats() { @@ -1993,10 +2006,15 @@ public void updateRetentionLeasesOnReplica(final RetentionLeases retentionLeases /** * Syncs the current retention leases to all replicas. */ - public void backgroundSyncRetentionLeases() { + public void syncRetentionLeases() { assert assertPrimaryMode(); verifyNotClosed(); - retentionLeaseSyncer.backgroundSync(shardId, getRetentionLeases()); + final Tuple retentionLeases = getRetentionLeases(true); + if (retentionLeases.v1()) { + retentionLeaseSyncer.sync(shardId, retentionLeases.v2(), ActionListener.wrap(() -> {})); + } else { + retentionLeaseSyncer.backgroundSync(shardId, retentionLeases.v2()); + } } /** diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java index 520344489adf9..bb526a3470873 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java @@ -37,7 +37,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.LongSupplier; import java.util.stream.Collectors; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -46,7 +45,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.lessThanOrEqualTo; public class ReplicationTrackerRetentionLeaseTests extends ReplicationTrackerTestCase { @@ -78,7 +76,7 @@ public void testAddOrRenewRetentionLease() { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); replicationTracker.addRetentionLease( Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); - assertRetentionLeases(replicationTracker, i + 1, minimumRetainingSequenceNumbers, () -> 0L, primaryTerm, 1 + i, true); + assertRetentionLeases(replicationTracker, i + 1, minimumRetainingSequenceNumbers, primaryTerm, 1 + i, true, false); } for (int i = 0; i < length; i++) { @@ -88,7 +86,7 @@ public void testAddOrRenewRetentionLease() { } minimumRetainingSequenceNumbers[i] = randomLongBetween(minimumRetainingSequenceNumbers[i], Long.MAX_VALUE); replicationTracker.renewRetentionLease(Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i); - assertRetentionLeases(replicationTracker, length, minimumRetainingSequenceNumbers, () -> 0L, primaryTerm, 1 + length + i, true); + assertRetentionLeases(replicationTracker, length, minimumRetainingSequenceNumbers, primaryTerm, 1 + length + i, true, false); } } @@ -193,7 +191,7 @@ private void runExpirationTest(final boolean primaryMode) { assertThat(retentionLeases.leases(), hasSize(1)); final RetentionLease retentionLease = retentionLeases.leases().iterator().next(); assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get())); - assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 1, primaryMode); + assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, primaryTerm, 1, primaryMode, false); } // renew the lease @@ -215,108 +213,20 @@ private void runExpirationTest(final boolean primaryMode) { assertThat(retentionLeases.leases(), hasSize(1)); final RetentionLease retentionLease = retentionLeases.leases().iterator().next(); assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get())); - assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 2, primaryMode); + assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, primaryTerm, 2, primaryMode, false); } // now force the lease to expire currentTimeMillis.set(currentTimeMillis.get() + randomLongBetween(retentionLeaseMillis, Long.MAX_VALUE - currentTimeMillis.get())); if (primaryMode) { - assertRetentionLeases(replicationTracker, 0, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 3, true); + assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, primaryTerm, 2, true, false); + assertRetentionLeases(replicationTracker, 0, new long[0], primaryTerm, 3, true, true); } else { // leases do not expire on replicas until synced from the primary - assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 2, false); + assertRetentionLeases(replicationTracker, 1, retainingSequenceNumbers, primaryTerm, 2, false, false); } } - public void testRetentionLeaseExpirationCausesRetentionLeaseSync() { - final AllocationId allocationId = AllocationId.newInitializing(); - final AtomicLong currentTimeMillis = new AtomicLong(randomLongBetween(0, 1024)); - final long retentionLeaseMillis = randomLongBetween(1, TimeValue.timeValueHours(12).millis()); - final Settings settings = Settings - .builder() - .put( - IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_SETTING.getKey(), - TimeValue.timeValueMillis(retentionLeaseMillis)) - .build(); - final Map> retentionLeases = new HashMap<>(); - final AtomicBoolean invoked = new AtomicBoolean(); - final AtomicReference reference = new AtomicReference<>(); - final ReplicationTracker replicationTracker = new ReplicationTracker( - new ShardId("test", "_na", 0), - allocationId.getId(), - IndexSettingsModule.newIndexSettings("test", settings), - randomNonNegativeLong(), - UNASSIGNED_SEQ_NO, - value -> {}, - currentTimeMillis::get, - (leases, listener) -> { - // we do not want to hold a lock on the replication tracker in the callback! - assertFalse(Thread.holdsLock(reference.get())); - invoked.set(true); - assertThat( - leases.leases() - .stream() - .collect(Collectors.toMap(RetentionLease::id, ReplicationTrackerRetentionLeaseTests::toTuple)), - equalTo(retentionLeases)); - }); - reference.set(replicationTracker); - replicationTracker.updateFromMaster( - randomNonNegativeLong(), - Collections.singleton(allocationId.getId()), - routingTable(Collections.emptySet(), allocationId), - Collections.emptySet()); - replicationTracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED); - - final int length = randomIntBetween(0, 8); - long version = 0; - for (int i = 0; i < length; i++) { - final String id = randomAlphaOfLength(8); - final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); - retentionLeases.put(id, Tuple.tuple(retainingSequenceNumber, currentTimeMillis.get())); - replicationTracker.addRetentionLease(id, retainingSequenceNumber, "test", ActionListener.wrap(() -> {})); - version++; - assertThat(replicationTracker.getRetentionLeases().version(), equalTo(version)); - // assert that the new retention lease callback was invoked - assertTrue(invoked.get()); - - // reset the invocation marker so that we can assert the callback was not invoked when renewing the lease - invoked.set(false); - currentTimeMillis.set(1 + currentTimeMillis.get()); - retentionLeases.put(id, Tuple.tuple(retainingSequenceNumber, currentTimeMillis.get())); - replicationTracker.renewRetentionLease(id, retainingSequenceNumber, "test"); - version++; - assertThat(replicationTracker.getRetentionLeases().version(), equalTo(version)); - - // reset the invocation marker so that we can assert the callback was invoked if any leases are expired - assertFalse(invoked.get()); - // randomly expire some leases - final long currentTimeMillisIncrement = randomLongBetween(0, Long.MAX_VALUE - currentTimeMillis.get()); - // calculate the expired leases and update our tracking map - final List expiredIds = retentionLeases.entrySet() - .stream() - .filter(r -> currentTimeMillis.get() + currentTimeMillisIncrement > r.getValue().v2() + retentionLeaseMillis) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - expiredIds.forEach(retentionLeases::remove); - if (expiredIds.isEmpty() == false) { - version++; - } - currentTimeMillis.set(currentTimeMillis.get() + currentTimeMillisIncrement); - // getting the leases has the side effect of calculating which leases are expired and invoking the sync callback - final RetentionLeases current = replicationTracker.getRetentionLeases(); - assertThat(current.version(), equalTo(version)); - // the current leases should equal our tracking map - assertThat( - current.leases() - .stream() - .collect(Collectors.toMap(RetentionLease::id, ReplicationTrackerRetentionLeaseTests::toTuple)), - equalTo(retentionLeases)); - // the callback should only be invoked if there were expired leases - assertThat(invoked.get(), equalTo(expiredIds.isEmpty() == false)); - } - assertThat(replicationTracker.getRetentionLeases().version(), equalTo(version)); - } - public void testReplicaIgnoresOlderRetentionLeasesVersion() { final AllocationId allocationId = AllocationId.newInitializing(); final ReplicationTracker replicationTracker = new ReplicationTracker( @@ -370,19 +280,29 @@ public void testReplicaIgnoresOlderRetentionLeasesVersion() { } } - private static Tuple toTuple(final RetentionLease retentionLease) { - return Tuple.tuple(retentionLease.retainingSequenceNumber(), retentionLease.timestamp()); - } - private void assertRetentionLeases( final ReplicationTracker replicationTracker, final int size, final long[] minimumRetainingSequenceNumbers, - final LongSupplier currentTimeMillisSupplier, final long primaryTerm, final long version, - final boolean primaryMode) { - final RetentionLeases retentionLeases = replicationTracker.getRetentionLeases(); + final boolean primaryMode, + final boolean expireLeases) { + assertTrue(expireLeases == false || primaryMode); + final RetentionLeases retentionLeases; + if (expireLeases == false) { + if (randomBoolean()) { + retentionLeases = replicationTracker.getRetentionLeases(); + } else { + final Tuple tuple = replicationTracker.getRetentionLeases(false); + assertFalse(tuple.v1()); + retentionLeases = tuple.v2(); + } + } else { + final Tuple tuple = replicationTracker.getRetentionLeases(true); + assertTrue(tuple.v1()); + retentionLeases = tuple.v2(); + } assertThat(retentionLeases.primaryTerm(), equalTo(primaryTerm)); assertThat(retentionLeases.version(), equalTo(version)); final Map idToRetentionLease = new HashMap<>(); @@ -395,12 +315,6 @@ private void assertRetentionLeases( assertThat(idToRetentionLease.keySet(), hasItem(Integer.toString(i))); final RetentionLease retentionLease = idToRetentionLease.get(Integer.toString(i)); assertThat(retentionLease.retainingSequenceNumber(), equalTo(minimumRetainingSequenceNumbers[i])); - if (primaryMode) { - // retention leases can be expired on replicas, so we can only assert on primaries here - assertThat( - currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), - lessThanOrEqualTo(replicationTracker.indexSettings().getRetentionLeaseMillis())); - } assertThat(retentionLease.source(), equalTo("test-" + i)); } } diff --git a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java index fffd29531c960..249fc4d580072 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java @@ -20,33 +20,58 @@ package org.elasticsearch.index.seqno; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) public class RetentionLeaseSyncIT extends ESIntegTestCase { + public static final class RetentionLeaseSyncIntervalSettingPlugin extends Plugin { + + @Override + public List> getSettings() { + return Collections.singletonList(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING); + } + + } + + @Override + protected Collection> nodePlugins() { + return Stream.concat( + super.nodePlugins().stream(), + Stream.of(RetentionLeaseBackgroundSyncIT.RetentionLeaseSyncIntervalSettingPlugin.class)) + .collect(Collectors.toList()); + } + public void testRetentionLeasesSyncedOnAdd() throws Exception { final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2); internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas); @@ -100,7 +125,6 @@ public void testRetentionLeasesSyncedOnAdd() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37963") public void testRetentionLeasesSyncOnExpiration() throws Exception { final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2); internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas); @@ -111,7 +135,7 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { .put("index.number_of_shards", 1) .put("index.number_of_replicas", numberOfReplicas) .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_SETTING.getKey(), retentionLeaseTimeToLive) + .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(1)) .build(); createIndex("index", settings); ensureGreen("index"); @@ -123,6 +147,17 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { // we will add multiple retention leases, wait for some to expire, and assert a consistent view between the primary and the replicas final int length = randomIntBetween(1, 8); for (int i = 0; i < length; i++) { + // update the index for retention leases to live a long time + final AcknowledgedResponse longTtlResponse = client().admin() + .indices() + .prepareUpdateSettings("index") + .setSettings( + Settings.builder() + .putNull(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_SETTING.getKey()) + .build()) + .get(); + assertTrue(longTtlResponse.isAcknowledged()); + final String id = randomAlphaOfLength(8); final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE); final String source = randomAlphaOfLength(8); @@ -139,19 +174,26 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { final IndexShard replica = internalCluster() .getInstance(IndicesService.class, replicaShardNodeName) .getShardOrNull(new ShardId(resolveIndex("index"), 0)); - assertThat(replica.getRetentionLeases().leases(), hasItem(currentRetentionLease)); + assertThat(replica.getRetentionLeases().leases(), anyOf(empty(), contains(currentRetentionLease))); } - // sleep long enough that *possibly* the current retention lease has expired, and certainly that any previous have + // update the index for retention leases to short a long time, to force expiration + final AcknowledgedResponse shortTtlResponse = client().admin() + .indices() + .prepareUpdateSettings("index") + .setSettings( + Settings.builder() + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_SETTING.getKey(), retentionLeaseTimeToLive) + .build()) + .get(); + assertTrue(shortTtlResponse.isAcknowledged()); + + // sleep long enough that the current retention lease has expired final long later = System.nanoTime(); Thread.sleep(Math.max(0, retentionLeaseTimeToLive.millis() - TimeUnit.NANOSECONDS.toMillis(later - now))); - final RetentionLeases currentRetentionLeases = primary.getRetentionLeases(); - assertThat(currentRetentionLeases.leases(), anyOf(empty(), contains(currentRetentionLease))); + assertBusy(() -> assertThat(primary.getRetentionLeases().leases(), empty())); - /* - * Check that expiration of retention leases has been synced to all replicas. We have to assert busy since syncing happens in - * the background. - */ + // now that all retention leases are expired should have been synced to all replicas assertBusy(() -> { for (final ShardRouting replicaShard : clusterService().state().routingTable().index("index").shard(0).replicaShards()) { final String replicaShardNodeId = replicaShard.currentNodeId(); @@ -159,13 +201,7 @@ public void testRetentionLeasesSyncOnExpiration() throws Exception { final IndexShard replica = internalCluster() .getInstance(IndicesService.class, replicaShardNodeName) .getShardOrNull(new ShardId(resolveIndex("index"), 0)); - if (currentRetentionLeases.leases().isEmpty()) { - assertThat(replica.getRetentionLeases().leases(), empty()); - } else { - assertThat( - replica.getRetentionLeases().leases(), - contains(currentRetentionLeases.leases().toArray(new RetentionLease[0]))); - } + assertThat(replica.getRetentionLeases().leases(), empty()); } }); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java index 75d8d7e8e2679..cc64fc6f8b2de 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRoutingHelper; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.IndexSettings; @@ -43,14 +44,12 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.LongSupplier; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -85,13 +84,20 @@ public void testAddOrRenewRetentionLease() throws IOException { indexShard.addRetentionLease( Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); assertRetentionLeases( - indexShard, i + 1, minimumRetainingSequenceNumbers, () -> 0L, primaryTerm, 1 + i, true); + indexShard, i + 1, minimumRetainingSequenceNumbers, primaryTerm, 1 + i, true, false); } for (int i = 0; i < length; i++) { minimumRetainingSequenceNumbers[i] = randomLongBetween(minimumRetainingSequenceNumbers[i], Long.MAX_VALUE); indexShard.renewRetentionLease(Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i); - assertRetentionLeases(indexShard, length, minimumRetainingSequenceNumbers, () -> 0L, primaryTerm, 1 + length + i, true); + assertRetentionLeases( + indexShard, + length, + minimumRetainingSequenceNumbers, + primaryTerm, + 1 + length + i, + true, + false); } } finally { closeShards(indexShard); @@ -121,8 +127,7 @@ private void runExpirationTest(final boolean primary) throws IOException { final long[] retainingSequenceNumbers = new long[1]; retainingSequenceNumbers[0] = randomLongBetween(0, Long.MAX_VALUE); if (primary) { - indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> { - })); + indexShard.addRetentionLease("0", retainingSequenceNumbers[0], "test-0", ActionListener.wrap(() -> {})); } else { final RetentionLeases retentionLeases = new RetentionLeases( primaryTerm, @@ -137,7 +142,7 @@ private void runExpirationTest(final boolean primary) throws IOException { assertThat(retentionLeases.leases(), hasSize(1)); final RetentionLease retentionLease = retentionLeases.leases().iterator().next(); assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get())); - assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 1, primary); + assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, 1, primary, false); } // renew the lease @@ -159,16 +164,17 @@ private void runExpirationTest(final boolean primary) throws IOException { assertThat(retentionLeases.leases(), hasSize(1)); final RetentionLease retentionLease = retentionLeases.leases().iterator().next(); assertThat(retentionLease.timestamp(), equalTo(currentTimeMillis.get())); - assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 2, primary); + assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, 2, primary, false); } // now force the lease to expire currentTimeMillis.set( currentTimeMillis.get() + randomLongBetween(retentionLeaseMillis, Long.MAX_VALUE - currentTimeMillis.get())); if (primary) { - assertRetentionLeases(indexShard, 0, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 3, true); + assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, 2, true, false); + assertRetentionLeases(indexShard, 0, new long[0], primaryTerm, 3, true, true); } else { - assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, currentTimeMillis::get, primaryTerm, 2, false); + assertRetentionLeases(indexShard, 1, retainingSequenceNumbers, primaryTerm, 2, false, false); } } finally { closeShards(indexShard); @@ -191,8 +197,7 @@ public void testCommit() throws IOException { minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE); currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(randomNonNegativeLong())); indexShard.addRetentionLease( - Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> { - })); + Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {})); } currentTimeMillis.set(TimeUnit.NANOSECONDS.toMillis(Long.MAX_VALUE)); @@ -250,13 +255,10 @@ public void testRetentionLeaseStats() throws IOException { final RetentionLeaseStats stats = indexShard.getRetentionLeaseStats(); assertRetentionLeases( stats.retentionLeases(), - indexShard.indexSettings().getRetentionLeaseMillis(), length, minimumRetainingSequenceNumbers, - () -> 0L, length == 0 ? RetentionLeases.EMPTY.primaryTerm() : indexShard.getOperationPrimaryTerm(), - length, - true); + length); } finally { closeShards(indexShard); } @@ -266,30 +268,39 @@ private void assertRetentionLeases( final IndexShard indexShard, final int size, final long[] minimumRetainingSequenceNumbers, - final LongSupplier currentTimeMillisSupplier, final long primaryTerm, final long version, - final boolean primary) { + final boolean primary, + final boolean expireLeases) { + assertTrue(expireLeases == false || primary); + final RetentionLeases retentionLeases; + if (expireLeases == false) { + if (randomBoolean()) { + retentionLeases = indexShard.getRetentionLeases(); + } else { + final Tuple tuple = indexShard.getRetentionLeases(false); + assertFalse(tuple.v1()); + retentionLeases = tuple.v2(); + } + } else { + final Tuple tuple = indexShard.getRetentionLeases(true); + assertTrue(tuple.v1()); + retentionLeases = tuple.v2(); + } assertRetentionLeases( - indexShard.getEngine().config().retentionLeasesSupplier().get(), - indexShard.indexSettings().getRetentionLeaseMillis(), + retentionLeases, size, minimumRetainingSequenceNumbers, - currentTimeMillisSupplier, primaryTerm, - version, - primary); + version); } private void assertRetentionLeases( final RetentionLeases retentionLeases, - final long retentionLeaseMillis, final int size, final long[] minimumRetainingSequenceNumbers, - final LongSupplier currentTimeMillisSupplier, final long primaryTerm, - final long version, - final boolean primary) { + final long version) { assertThat(retentionLeases.primaryTerm(), equalTo(primaryTerm)); assertThat(retentionLeases.version(), equalTo(version)); final Map idToRetentionLease = new HashMap<>(); @@ -302,10 +313,6 @@ private void assertRetentionLeases( assertThat(idToRetentionLease.keySet(), hasItem(Integer.toString(i))); final RetentionLease retentionLease = idToRetentionLease.get(Integer.toString(i)); assertThat(retentionLease.retainingSequenceNumber(), equalTo(minimumRetainingSequenceNumbers[i])); - if (primary) { - // retention leases can be expired on replicas, so we can only assert on primaries here - assertThat(currentTimeMillisSupplier.getAsLong() - retentionLease.timestamp(), lessThanOrEqualTo(retentionLeaseMillis)); - } assertThat(retentionLease.source(), equalTo("test-" + i)); } } From d0ac828428217760ac9344c6a6f2155965e7dc18 Mon Sep 17 00:00:00 2001 From: Henning Andersen <33268011+henningandersen@users.noreply.github.com> Date: Tue, 5 Feb 2019 21:30:29 +0100 Subject: [PATCH 14/25] Bubble-up exceptions from scheduler (#38441) Instead of logging warnings we now rethrow exceptions thrown inside scheduled/submitted tasks. This will still log them as warnings in production but has the added benefit that if they are thrown during unit/integration test runs, the test will be flagged as an error. Fixed NPE in GlobalCheckPointListeners that caused CCR tests (IndexFollowingIT and likely others) to fail. This is a continuation of #38014 Backports #38317 --- .../threadpool/EvilThreadPoolTests.java | 115 ++++++++++-------- .../shard/GlobalCheckpointListeners.java | 2 +- .../elasticsearch/threadpool/Scheduler.java | 19 ++- .../elasticsearch/threadpool/ThreadPool.java | 12 +- .../threadpool/SchedulerTests.java | 37 ------ 5 files changed, 82 insertions(+), 103 deletions(-) diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java index 86f2ba332790f..f6c2c616d4008 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/threadpool/EvilThreadPoolTests.java @@ -19,11 +19,7 @@ package org.elasticsearch.threadpool; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LogEvent; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -31,20 +27,21 @@ import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockLogAppender; import org.junit.After; import org.junit.Before; import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Consumer; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; @@ -260,6 +257,50 @@ public void testExecutionExceptionOnScheduler() throws InterruptedException { } } + @SuppressForbidden(reason = "this tests that the deprecated method still works") + public void testDeprecatedSchedule() throws ExecutionException, InterruptedException { + verifyDeprecatedSchedule(((threadPool, runnable) + -> threadPool.schedule(TimeValue.timeValueMillis(randomInt(10)), ThreadPool.Names.SAME, runnable))); + } + + public void testThreadPoolScheduleDeprecated() throws ExecutionException, InterruptedException { + verifyDeprecatedSchedule(((threadPool, runnable) + -> threadPool.scheduleDeprecated(TimeValue.timeValueMillis(randomInt(10)), ThreadPool.Names.SAME, runnable))); + } + + private void verifyDeprecatedSchedule(BiFunction> scheduleFunction) throws InterruptedException, ExecutionException { + Thread.UncaughtExceptionHandler originalHandler = Thread.getDefaultUncaughtExceptionHandler(); + CountDownLatch missingExceptions = new CountDownLatch(1); + Thread.setDefaultUncaughtExceptionHandler((t, e) -> { + missingExceptions.countDown(); + }); + try { + ThreadPool threadPool = new TestThreadPool("test"); + CountDownLatch missingExecutions = new CountDownLatch(1); + try { + scheduleFunction.apply(threadPool, missingExecutions::countDown) + .get(); + assertEquals(0, missingExecutions.getCount()); + + ExecutionException exception = expectThrows(ExecutionException.class, + "schedule(...).get() must throw exception from runnable", + () -> scheduleFunction.apply(threadPool, + () -> { + throw new IllegalArgumentException("FAIL"); + } + ).get()); + + assertEquals(IllegalArgumentException.class, exception.getCause().getClass()); + assertTrue(missingExceptions.await(30, TimeUnit.SECONDS)); + } finally { + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + } finally { + Thread.setDefaultUncaughtExceptionHandler(originalHandler); + } + } + private Runnable delayMillis(Runnable r, int ms) { return () -> { try { @@ -369,60 +410,26 @@ private void runExecutionTest( final CountDownLatch supplierLatch = new CountDownLatch(1); - Runnable job = () -> { - try { - runnable.run(); - } finally { - supplierLatch.countDown(); - } - }; - - // snoop on logging to also handle the cases where exceptions are simply logged in Scheduler. - final Logger schedulerLogger = LogManager.getLogger(Scheduler.SafeScheduledThreadPoolExecutor.class); - final MockLogAppender appender = new MockLogAppender(); - appender.addExpectation( - new MockLogAppender.LoggingExpectation() { - @Override - public void match(LogEvent event) { - if (event.getLevel() == Level.WARN) { - assertThat("no other warnings than those expected", - event.getMessage().getFormattedMessage(), - equalTo("uncaught exception in scheduled thread [" + Thread.currentThread().getName() + "]")); - assertTrue(expectThrowable); - assertNotNull(event.getThrown()); - assertTrue("only one message allowed", throwableReference.compareAndSet(null, event.getThrown())); - uncaughtExceptionHandlerLatch.countDown(); - } - } - - @Override - public void assertMatched() { + try { + runner.accept(() -> { + try { + runnable.run(); + } finally { + supplierLatch.countDown(); } }); + } catch (Throwable t) { + consumer.accept(Optional.of(t)); + return; + } - appender.start(); - Loggers.addAppender(schedulerLogger, appender); - try { - try { - runner.accept(job); - } catch (Throwable t) { - consumer.accept(Optional.of(t)); - return; - } - - supplierLatch.await(); + supplierLatch.await(); - if (expectThrowable) { - uncaughtExceptionHandlerLatch.await(); - } - } finally { - Loggers.removeAppender(schedulerLogger, appender); - appender.stop(); + if (expectThrowable) { + uncaughtExceptionHandlerLatch.await(); } consumer.accept(Optional.ofNullable(throwableReference.get())); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); } finally { Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/GlobalCheckpointListeners.java b/server/src/main/java/org/elasticsearch/index/shard/GlobalCheckpointListeners.java index eb9e36eeec0a8..d45a77ddb22ac 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/GlobalCheckpointListeners.java +++ b/server/src/main/java/org/elasticsearch/index/shard/GlobalCheckpointListeners.java @@ -135,7 +135,7 @@ synchronized void add(final long waitingForGlobalCheckpoint, final GlobalCheckpo * before we could be cancelled by the notification. In this case, our listener here would * not be in the map and we should not fire the timeout logic. */ - removed = listeners.remove(listener).v2() != null; + removed = listeners.remove(listener) != null; } if (removed) { final TimeoutException e = new TimeoutException(timeout.getStringRep()); diff --git a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java index 34a6efe82038d..450354e44dbbb 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java +++ b/server/src/main/java/org/elasticsearch/threadpool/Scheduler.java @@ -21,7 +21,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -47,8 +47,8 @@ public interface Scheduler { /** * Create a scheduler that can be used client side. Server side, please use ThreadPool.schedule instead. * - * Notice that if any scheduled jobs fail with an exception, they will be logged as a warning. This includes jobs started - * using execute, submit and schedule. + * Notice that if any scheduled jobs fail with an exception, these will bubble up to the uncaught exception handler where they will + * be logged as a warning. This includes jobs started using execute, submit and schedule. * @param settings the settings to use * @return executor */ @@ -272,7 +272,8 @@ public void onAfter() { } /** - * This subclass ensures to properly bubble up Throwable instances of type Error and logs exceptions thrown in submitted/scheduled tasks + * This subclass ensures to properly bubble up Throwable instances of both type Error and Exception thrown in submitted/scheduled + * tasks to the uncaught exception handler */ class SafeScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor { private static final Logger logger = LogManager.getLogger(SafeScheduledThreadPoolExecutor.class); @@ -294,12 +295,10 @@ public SafeScheduledThreadPoolExecutor(int corePoolSize) { @Override protected void afterExecute(Runnable r, Throwable t) { - Throwable exception = EsExecutors.rethrowErrors(r); - if (exception != null) { - logger.warn(() -> - new ParameterizedMessage("uncaught exception in scheduled thread [{}]", Thread.currentThread().getName()), - exception); - } + if (t != null) return; + // Scheduler only allows Runnable's so we expect no checked exceptions here. If anyone uses submit directly on `this`, we + // accept the wrapped exception in the output. + ExceptionsHelper.reThrowIfNotNull(EsExecutors.rethrowErrors(r)); } } } diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index d22f0166ab921..30db1ac52726f 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.Counter; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; @@ -497,7 +498,16 @@ class ThreadedRunnable implements Runnable { @Override public void run() { - executor.execute(runnable); + try { + executor.execute(runnable); + } catch (EsRejectedExecutionException e) { + if (e.isExecutorShutdown()) { + logger.debug(new ParameterizedMessage("could not schedule execution of [{}] on [{}] as executor is shut down", + runnable, executor), e); + } else { + throw e; + } + } } @Override diff --git a/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java b/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java index ef7371693c3e1..186f9e86b1e76 100644 --- a/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java +++ b/server/src/test/java/org/elasticsearch/threadpool/SchedulerTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.threadpool; -import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -29,12 +28,9 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -157,37 +153,4 @@ public void testScheduledOnScheduler() throws InterruptedException { Scheduler.terminate(executor, 10, TimeUnit.SECONDS); } } - - @SuppressForbidden(reason = "this tests that the deprecated method still works") - public void testDeprecatedSchedule() throws ExecutionException, InterruptedException { - verifyDeprecatedSchedule(((threadPool, runnable) - -> threadPool.schedule(TimeValue.timeValueMillis(randomInt(10)), ThreadPool.Names.SAME, runnable))); - } - - public void testThreadPoolScheduleDeprecated() throws ExecutionException, InterruptedException { - verifyDeprecatedSchedule(((threadPool, runnable) - -> threadPool.scheduleDeprecated(TimeValue.timeValueMillis(randomInt(10)), ThreadPool.Names.SAME, runnable))); - } - - private void verifyDeprecatedSchedule(BiFunction> scheduleFunction) throws InterruptedException, ExecutionException { - ThreadPool threadPool = new TestThreadPool("test"); - CountDownLatch missingExecutions = new CountDownLatch(1); - try { - scheduleFunction.apply(threadPool, missingExecutions::countDown) - .get(); - assertEquals(0, missingExecutions.getCount()); - - ExecutionException exception = expectThrows(ExecutionException.class, - "schedule(...).get() must throw exception from runnable", - () -> scheduleFunction.apply(threadPool, - () -> { throw new IllegalArgumentException("FAIL"); } - ).get()); - - assertEquals(IllegalArgumentException.class, exception.getCause().getClass()); - } finally { - ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); - } - } - } From 5512574ab4a71ff5112fb1a7fbbb7e553eb9ccc5 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 5 Feb 2019 14:03:44 -0700 Subject: [PATCH 15/25] Deprecation check for No Master Block setting (#38383) `discovery.zen.no_master_block` is deprecated in favor of `cluster.no_master_block` in 7.0, but is a dynamic setting so we need both node- and cluster-level checks for it. --- .../deprecation/ClusterDeprecationChecks.java | 14 +++++++++++++ .../xpack/deprecation/DeprecationChecks.java | 2 ++ .../deprecation/NodeDeprecationChecks.java | 13 ++++++++++++ .../ClusterDeprecationChecksTests.java | 20 +++++++++++++++++++ .../NodeDeprecationChecksTests.java | 12 +++++++++++ 5 files changed, 61 insertions(+) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java index 60ba41c06fb5f..8142622b1f0ba 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -18,6 +18,8 @@ import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.discovery.DiscoverySettings.NO_MASTER_BLOCK_SETTING; + public class ClusterDeprecationChecks { static DeprecationIssue checkShardLimit(ClusterState state) { @@ -37,6 +39,18 @@ static DeprecationIssue checkShardLimit(ClusterState state) { return null; } + static DeprecationIssue checkNoMasterBlock(ClusterState state) { + if (state.metaData().settings().hasValue(NO_MASTER_BLOCK_SETTING.getKey())) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Master block setting will be renamed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "_new_name_for_literal_no_master_block_literal_setting", + "The setting discovery.zen.no_master_block will be renamed to cluster.no_master_block in 7.0. " + + "Please unset discovery.zen.no_master_block and set cluster.no_master_block after upgrading to 7.0."); + } + return null; + } + static DeprecationIssue checkClusterName(ClusterState state) { String clusterName = state.getClusterName().value(); if (clusterName.contains(":")) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 6434c8103f7cb..42e0abfcdb2bf 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -35,12 +35,14 @@ private DeprecationChecks() { Collections.unmodifiableList(Arrays.asList( ClusterDeprecationChecks::checkUserAgentPipelines, ClusterDeprecationChecks::checkShardLimit, + ClusterDeprecationChecks::checkNoMasterBlock, ClusterDeprecationChecks::checkClusterName )); static List> NODE_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( NodeDeprecationChecks::httpEnabledSettingRemoved, + NodeDeprecationChecks::noMasterBlockRenamed, NodeDeprecationChecks::auditLogPrefixSettingsCheck, NodeDeprecationChecks::indexThreadPoolCheck, NodeDeprecationChecks::bulkThreadPoolCheck, diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 780391e4a4244..0ba932b5094dd 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -24,6 +24,7 @@ import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_TYPE_SETTING; +import static org.elasticsearch.discovery.DiscoverySettings.NO_MASTER_BLOCK_SETTING; import static org.elasticsearch.discovery.zen.SettingsBasedHostsProvider.DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING; import static org.elasticsearch.xpack.core.XPackSettings.SECURITY_ENABLED; import static org.elasticsearch.xpack.core.XPackSettings.TRANSPORT_SSL_ENABLED; @@ -44,6 +45,18 @@ static DeprecationIssue httpEnabledSettingRemoved(Settings nodeSettings, Plugins return null; } + static DeprecationIssue noMasterBlockRenamed(Settings nodeSettings, PluginsAndModules plugins) { + if (nodeSettings.hasValue(NO_MASTER_BLOCK_SETTING.getKey())) { + return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "Master block setting renamed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "_new_name_for_literal_no_master_block_literal_setting", + "The setting discovery.zen.no_master_block will be renamed to cluster.no_master_block in 7.0. " + + "Please unset discovery.zen.no_master_block and set cluster.no_master_block after upgrading to 7.0."); + } + return null; + } + static DeprecationIssue auditLogPrefixSettingsCheck(Settings nodeSettings, PluginsAndModules plugins) { if (nodeSettings.getByPrefix("xpack.security.audit.logfile.prefix").isEmpty() == false) { return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java index 09bc2b991fa8a..007c3edd0a85e 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -24,6 +24,7 @@ import java.util.List; import static java.util.Collections.singletonList; +import static org.elasticsearch.discovery.DiscoverySettings.NO_MASTER_BLOCK_SETTING; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; public class ClusterDeprecationChecksTests extends ESTestCase { @@ -45,6 +46,25 @@ public void testCheckClusterName() { assertTrue(noIssues.isEmpty()); } + public void testCheckNoMasterBlock() { + MetaData metaData = MetaData.builder() + .persistentSettings(Settings.builder() + .put(NO_MASTER_BLOCK_SETTING.getKey(), randomFrom("all", "write")) + .build()) + .build(); + ClusterState state = ClusterState.builder(new ClusterName("test")) + .metaData(metaData) + .build(); + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "Master block setting will be renamed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "_new_name_for_literal_no_master_block_literal_setting", + "The setting discovery.zen.no_master_block will be renamed to cluster.no_master_block in 7.0. " + + "Please unset discovery.zen.no_master_block and set cluster.no_master_block after upgrading to 7.0."); + List issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(state)); + assertEquals(singletonList(expected), issues); + } + public void testCheckShardLimit() { int shardsPerNode = randomIntBetween(2, 10000); int nodeCount = randomIntBetween(1, 10); diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 88da4e5252f96..35d7a11799664 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -30,6 +30,7 @@ import static org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_TYPE_SETTING; +import static org.elasticsearch.discovery.DiscoverySettings.NO_MASTER_BLOCK_SETTING; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; import static org.elasticsearch.xpack.deprecation.DeprecationChecks.NODE_SETTINGS_CHECKS; @@ -84,6 +85,17 @@ public void testHttpEnabledCheck() { assertSettingsAndIssue("http.enabled", Boolean.toString(randomBoolean()), expected); } + public void testNoMasterBlockRenamed() { + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, + "Master block setting renamed", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" + + "_new_name_for_literal_no_master_block_literal_setting", + "The setting discovery.zen.no_master_block will be renamed to cluster.no_master_block in 7.0. " + + "Please unset discovery.zen.no_master_block and set cluster.no_master_block after upgrading to 7.0."); + + assertSettingsAndIssue(NO_MASTER_BLOCK_SETTING.getKey(), randomFrom("all", "write"), expected); + } + public void testAuditLoggingPrefixSettingsCheck() { DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Audit log node info settings renamed", From 571cf521fc3009db7bd74c4ab7c7eff408962f10 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 5 Feb 2019 22:04:32 +0100 Subject: [PATCH 16/25] Clean up duplicate follow config parameter code (#37688) (#38443) Introduced FollowParameters class that put follow, resume follow, put auto follow pattern requests and follow info response classes reuse. The FollowParameters class had the fields, getters etc. for the common parameters that all these APIs have. Also binary and xcontent serialization / parsing is handled by this class. The follow, resume follow, put auto follow pattern request classes originally used optional non primitive fields, so FollowParameters has that too and the follow info api can handle that now too. Also the followerIndex field can in production only be specified via the url path. If it is also specified via the request body then it must have the same value as is specified in the url path. This option only existed to xcontent testing. However the AbstractSerializingTestCase base class now also supports createXContextTestInstance() to provide a different test instance when testing xcontent, so allowing followerIndex to be specified via the request body is no longer needed. By moving the followerIndex field from Body to ResumeFollowAction.Request class and not allowing the followerIndex field to be specified via the request body the Body class is redundant and can be removed. The ResumeFollowAction.Request class can then directly use the FollowParameters class. For consistency I also removed the ability to specified followerIndex in the put follow api and the name in put auto follow pattern api via the request body. --- .../client/ccr/PutFollowRequest.java | 2 - .../client/ccr/ResumeFollowRequest.java | 3 - .../client/ccr/PutFollowRequestTests.java | 5 +- .../client/ccr/ResumeFollowRequestTests.java | 8 +- .../ccr/action/AutoFollowCoordinator.java | 26 +- .../xpack/ccr/action/Pre67PutFollow.java | 13 +- .../ccr/action/TransportFollowInfoAction.java | 25 +- .../TransportPutAutoFollowPatternAction.java | 23 +- .../ccr/action/TransportPutFollowAction.java | 21 +- .../action/TransportResumeFollowAction.java | 53 +-- .../elasticsearch/xpack/CcrIntegTestCase.java | 8 +- .../xpack/CcrSingleNodeTestCase.java | 8 +- .../elasticsearch/xpack/ccr/AutoFollowIT.java | 69 ++-- .../xpack/ccr/FollowerFailOverIT.java | 30 +- .../xpack/ccr/IndexFollowingIT.java | 14 +- .../xpack/ccr/LocalIndexFollowingIT.java | 4 +- .../action/AutoFollowCoordinatorTests.java | 6 +- .../ccr/action/FollowInfoResponseTests.java | 79 +---- .../PutAutoFollowPatternRequestTests.java | 54 +-- .../action/PutFollowActionRequestTests.java | 21 +- .../ResumeFollowActionRequestTests.java | 48 ++- .../core/ccr/action/FollowInfoAction.java | 158 +-------- .../core/ccr/action/FollowParameters.java | 314 ++++++++++++++++++ .../action/PutAutoFollowPatternAction.java | 305 ++++------------- .../core/ccr/action/PutFollowAction.java | 142 +++----- .../core/ccr/action/ResumeFollowAction.java | 297 ++--------------- 26 files changed, 714 insertions(+), 1022 deletions(-) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowParameters.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java index 8307b04bd7087..9c9e3f92b8173 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java @@ -32,7 +32,6 @@ public final class PutFollowRequest extends FollowConfig implements Validatable, static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster"); static final ParseField LEADER_INDEX_FIELD = new ParseField("leader_index"); - static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index"); private final String remoteCluster; private final String leaderIndex; @@ -55,7 +54,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex); - builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); toXContentFragment(builder, params); builder.endObject(); return builder; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ResumeFollowRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ResumeFollowRequest.java index d9ceb666afd2f..972f327134749 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ResumeFollowRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ResumeFollowRequest.java @@ -26,8 +26,6 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.client.ccr.PutFollowRequest.FOLLOWER_INDEX_FIELD; - public final class ResumeFollowRequest extends FollowConfig implements Validatable, ToXContentObject { private final String followerIndex; @@ -39,7 +37,6 @@ public ResumeFollowRequest(String followerIndex) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); toXContentFragment(builder, params); builder.endObject(); return builder; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java index 35353ce4a96f9..1f6a3d9f0ac28 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java @@ -31,12 +31,11 @@ public class PutFollowRequestTests extends AbstractXContentTestCase { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("test_parser", - true, (args) -> new PutFollowRequest((String) args[0], (String) args[1], (String) args[2])); + true, (args) -> new PutFollowRequest((String) args[0], (String) args[1], "followerIndex")); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD); PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.LEADER_INDEX_FIELD); - PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.FOLLOWER_INDEX_FIELD); PARSER.declareInt(PutFollowRequest::setMaxReadRequestOperationCount, PutFollowRequest.MAX_READ_REQUEST_OPERATION_COUNT); PARSER.declareField( PutFollowRequest::setMaxReadRequestSize, @@ -82,7 +81,7 @@ protected boolean supportsUnknownFields() { @Override protected PutFollowRequest createTestInstance() { PutFollowRequest putFollowRequest = - new PutFollowRequest(randomAlphaOfLength(4), randomAlphaOfLength(4), randomAlphaOfLength(4)); + new PutFollowRequest(randomAlphaOfLength(4), randomAlphaOfLength(4), "followerIndex"); if (randomBoolean()) { putFollowRequest.setMaxOutstandingReadRequests(randomIntBetween(0, Integer.MAX_VALUE)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/ResumeFollowRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/ResumeFollowRequestTests.java index 3f00891331839..d5d2b7e25539f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/ResumeFollowRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/ResumeFollowRequestTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; @@ -30,11 +29,10 @@ public class ResumeFollowRequestTests extends AbstractXContentTestCase { - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("test_parser", - true, (args) -> new ResumeFollowRequest((String) args[0])); + private static final ObjectParser PARSER = new ObjectParser<>("test_parser", + true, () -> new ResumeFollowRequest("followerIndex")); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.FOLLOWER_INDEX_FIELD); PARSER.declareInt(ResumeFollowRequest::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT); PARSER.declareField( ResumeFollowRequest::setMaxReadRequestSize, @@ -79,7 +77,7 @@ protected boolean supportsUnknownFields() { @Override protected ResumeFollowRequest createTestInstance() { - ResumeFollowRequest resumeFollowRequest = new ResumeFollowRequest(randomAlphaOfLength(4)); + ResumeFollowRequest resumeFollowRequest = new ResumeFollowRequest("followerIndex"); if (randomBoolean()) { resumeFollowRequest.setMaxOutstandingReadRequests(randomIntBetween(0, Integer.MAX_VALUE)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index 4ce212b158761..34350767008e4 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -40,7 +40,6 @@ import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; import org.elasticsearch.xpack.core.ccr.AutoFollowStats; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; -import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.util.ArrayList; import java.util.Collections; @@ -511,23 +510,20 @@ private void followLeaderIndex(String autoFollowPattenName, final String leaderIndexName = indexToFollow.getName(); final String followIndexName = getFollowerIndexName(pattern, leaderIndexName); - ResumeFollowAction.Request followRequest = new ResumeFollowAction.Request(); - followRequest.setFollowerIndex(followIndexName); - followRequest.setMaxReadRequestOperationCount(pattern.getMaxReadRequestOperationCount()); - followRequest.setMaxReadRequestSize(pattern.getMaxReadRequestSize()); - followRequest.setMaxOutstandingReadRequests(pattern.getMaxOutstandingReadRequests()); - followRequest.setMaxWriteRequestOperationCount(pattern.getMaxWriteRequestOperationCount()); - followRequest.setMaxWriteRequestSize(pattern.getMaxWriteRequestSize()); - followRequest.setMaxOutstandingWriteRequests(pattern.getMaxOutstandingWriteRequests()); - followRequest.setMaxWriteBufferCount(pattern.getMaxWriteBufferCount()); - followRequest.setMaxWriteBufferSize(pattern.getMaxWriteBufferSize()); - followRequest.setMaxRetryDelay(pattern.getMaxRetryDelay()); - followRequest.setReadPollTimeout(pattern.getPollTimeout()); - PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster(remoteCluster); request.setLeaderIndex(indexToFollow.getName()); - request.setFollowRequest(followRequest); + request.setFollowerIndex(followIndexName); + request.getParameters().setMaxReadRequestOperationCount(pattern.getMaxReadRequestOperationCount()); + request.getParameters().setMaxReadRequestSize(pattern.getMaxReadRequestSize()); + request.getParameters().setMaxOutstandingReadRequests(pattern.getMaxOutstandingReadRequests()); + request.getParameters().setMaxWriteRequestOperationCount(pattern.getMaxWriteRequestOperationCount()); + request.getParameters().setMaxWriteRequestSize(pattern.getMaxWriteRequestSize()); + request.getParameters().setMaxOutstandingWriteRequests(pattern.getMaxOutstandingWriteRequests()); + request.getParameters().setMaxWriteBufferCount(pattern.getMaxWriteBufferCount()); + request.getParameters().setMaxWriteBufferSize(pattern.getMaxWriteBufferSize()); + request.getParameters().setMaxRetryDelay(pattern.getMaxRetryDelay()); + request.getParameters().setReadPollTimeout(pattern.getPollTimeout()); // Execute if the create and follow api call succeeds: Runnable successHandler = () -> { diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/Pre67PutFollow.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/Pre67PutFollow.java index a0796624de50d..c070d8eb5a65c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/Pre67PutFollow.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/Pre67PutFollow.java @@ -72,7 +72,7 @@ protected Boolean newResponse(final boolean acknowledged) { @Override public ClusterState execute(final ClusterState currentState) throws Exception { - String followIndex = request.getFollowRequest().getFollowerIndex(); + String followIndex = request.getFollowerIndex(); IndexMetaData currentIndex = currentState.metaData().index(followIndex); if (currentIndex != null) { throw new ResourceAlreadyExistsException(currentIndex.getIndex()); @@ -112,10 +112,10 @@ public ClusterState execute(final ClusterState currentState) throws Exception { ClusterState updatedState = builder.build(); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(updatedState.routingTable()) - .addAsNew(updatedState.metaData().index(request.getFollowRequest().getFollowerIndex())); + .addAsNew(updatedState.metaData().index(request.getFollowerIndex())); updatedState = allocationService.reroute( ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(), - "follow index [" + request.getFollowRequest().getFollowerIndex() + "] created"); + "follow index [" + request.getFollowerIndex() + "] created"); logger.info("[{}] creating index, cause [ccr_create_and_follow], shards [{}]/[{}]", followIndex, followIMD.getNumberOfShards(), followIMD.getNumberOfReplicas()); @@ -126,10 +126,13 @@ public ClusterState execute(final ClusterState currentState) throws Exception { } private void initiateFollowing(final PutFollowAction.Request request, final ActionListener listener) { - activeShardsObserver.waitForActiveShards(new String[]{request.getFollowRequest().getFollowerIndex()}, + activeShardsObserver.waitForActiveShards(new String[]{request.getFollowerIndex()}, ActiveShardCount.DEFAULT, request.timeout(), result -> { if (result) { - client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( + ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request(); + resumeFollowRequest.setFollowerIndex(request.getFollowerIndex()); + resumeFollowRequest.setParameters(request.getParameters()); + client.execute(ResumeFollowAction.INSTANCE, resumeFollowRequest, ActionListener.wrap( r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), listener::onFailure )); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java index 09d2c31c4c0b3..c8b44e6192820 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java @@ -23,7 +23,7 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction; -import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowParameters; +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.Status; @@ -99,18 +99,17 @@ static List getFollowInfos(List concreteFollowerIndices, C String leaderIndex = ccrCustomData.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_NAME_KEY); if (result.isPresent()) { ShardFollowTask params = result.get(); - FollowParameters followParameters = new FollowParameters( - params.getMaxReadRequestOperationCount(), - params.getMaxReadRequestSize(), - params.getMaxOutstandingReadRequests(), - params.getMaxWriteRequestOperationCount(), - params.getMaxWriteRequestSize(), - params.getMaxOutstandingWriteRequests(), - params.getMaxWriteBufferCount(), - params.getMaxWriteBufferSize(), - params.getMaxRetryDelay(), - params.getReadPollTimeout() - ); + FollowParameters followParameters = new FollowParameters(); + followParameters.setMaxOutstandingReadRequests(params.getMaxOutstandingReadRequests()); + followParameters.setMaxOutstandingWriteRequests(params.getMaxOutstandingWriteRequests()); + followParameters.setMaxReadRequestOperationCount(params.getMaxReadRequestOperationCount()); + followParameters.setMaxWriteRequestOperationCount(params.getMaxWriteRequestOperationCount()); + followParameters.setMaxReadRequestSize(params.getMaxReadRequestSize()); + followParameters.setMaxWriteRequestSize(params.getMaxWriteRequestSize()); + followParameters.setMaxWriteBufferCount(params.getMaxWriteBufferCount()); + followParameters.setMaxWriteBufferSize(params.getMaxWriteBufferSize()); + followParameters.setMaxRetryDelay(params.getMaxRetryDelay()); + followParameters.setReadPollTimeout(params.getReadPollTimeout()); followerInfos.add(new FollowerInfo(followerIndex, remoteCluster, leaderIndex, Status.ACTIVE, followParameters)); } else { followerInfos.add(new FollowerInfo(followerIndex, remoteCluster, leaderIndex, Status.PAUSED, null)); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index 97f10ac8bfa9a..379cb518a12c0 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -149,8 +149,7 @@ static ClusterState innerPut(PutAutoFollowPatternAction.Request request, markExistingIndicesAsAutoFollowedForNewPatterns(request.getLeaderIndexPatterns(), remoteClusterState.metaData(), previousPattern, followedIndexUUIDs); } else { - markExistingIndicesAsAutoFollowed(request.getLeaderIndexPatterns(), remoteClusterState.metaData(), - followedIndexUUIDs); + markExistingIndicesAsAutoFollowed(request.getLeaderIndexPatterns(), remoteClusterState.metaData(), followedIndexUUIDs); } if (filteredHeaders != null) { @@ -161,16 +160,16 @@ static ClusterState innerPut(PutAutoFollowPatternAction.Request request, request.getRemoteCluster(), request.getLeaderIndexPatterns(), request.getFollowIndexNamePattern(), - request.getMaxReadRequestOperationCount(), - request.getMaxReadRequestSize(), - request.getMaxConcurrentReadBatches(), - request.getMaxWriteRequestOperationCount(), - request.getMaxWriteRequestSize(), - request.getMaxConcurrentWriteBatches(), - request.getMaxWriteBufferCount(), - request.getMaxWriteBufferSize(), - request.getMaxRetryDelay(), - request.getReadPollTimeout()); + request.getParameters().getMaxReadRequestOperationCount(), + request.getParameters().getMaxReadRequestSize(), + request.getParameters().getMaxOutstandingReadRequests(), + request.getParameters().getMaxWriteRequestOperationCount(), + request.getParameters().getMaxWriteRequestSize(), + request.getParameters().getMaxOutstandingWriteRequests(), + request.getParameters().getMaxWriteBufferCount(), + request.getParameters().getMaxWriteBufferSize(), + request.getParameters().getMaxRetryDelay(), + request.getParameters().getReadPollTimeout()); patterns.put(request.getName(), autoFollowPattern); ClusterState.Builder newState = ClusterState.builder(localState); newState.metaData(MetaData.builder(localState.getMetaData()) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 515da70719fd7..96285021d6d95 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.ccr.repository.CcrRepository; +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; @@ -136,8 +137,8 @@ private void createFollowerIndex( return; } if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(leaderIndexMetaData.getSettings()) == false) { - listener.onFailure( - new IllegalArgumentException("leader index [" + request.getLeaderIndex() + "] does not have soft deletes enabled. " + + listener.onFailure(new IllegalArgumentException("leader index [" + request.getLeaderIndex() + + "] does not have soft deletes enabled. " + "soft deletes must be enabled when the index is created by setting " + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey() + " to true")); return; @@ -148,17 +149,17 @@ private void createFollowerIndex( if (pre67CompatibilityMode) { logger.debug("Pre-6.7 nodes present in local/remote cluster. Cannot bootstrap from remote. Creating empty follower index " + - "[{}] and initiating following [{}, {}].", request.getFollowRequest().getFollowerIndex(), request.getRemoteCluster(), + "[{}] and initiating following [{}, {}].", request.getFollowerIndex(), request.getRemoteCluster(), request.getLeaderIndex()); pre67PutFollow.doPre67PutFollow(request, leaderIndexMetaData, historyUUID, listener); } else { final Settings.Builder settingsBuilder = Settings.builder() - .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowRequest().getFollowerIndex()) + .put(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, request.getFollowerIndex()) .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); final String leaderClusterRepoName = CcrRepository.NAME_PREFIX + request.getRemoteCluster(); final RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) .indices(request.getLeaderIndex()).indicesOptions(request.indicesOptions()).renamePattern("^(.*)$") - .renameReplacement(request.getFollowRequest().getFollowerIndex()).masterNodeTimeout(request.masterNodeTimeout()) + .renameReplacement(request.getFollowerIndex()).masterNodeTimeout(request.masterNodeTimeout()) .indexSettings(settingsBuilder); final Client clientWithHeaders = CcrLicenseChecker.wrapClient(this.client, threadPool.getThreadContext().getHeaders()); @@ -239,10 +240,14 @@ private void initiateFollowing( final PutFollowAction.Request request, final ActionListener listener) { assert request.waitForActiveShards() != ActiveShardCount.DEFAULT : "PutFollowAction does not support DEFAULT."; - activeShardsObserver.waitForActiveShards(new String[]{request.getFollowRequest().getFollowerIndex()}, + activeShardsObserver.waitForActiveShards(new String[]{request.getFollowerIndex()}, request.waitForActiveShards(), request.timeout(), result -> { if (result) { - client.execute(ResumeFollowAction.INSTANCE, request.getFollowRequest(), ActionListener.wrap( + FollowParameters parameters = request.getParameters(); + ResumeFollowAction.Request resumeFollowRequest = new ResumeFollowAction.Request(); + resumeFollowRequest.setFollowerIndex(request.getFollowerIndex()); + resumeFollowRequest.setParameters(new FollowParameters(parameters)); + client.execute(ResumeFollowAction.INSTANCE, resumeFollowRequest, ActionListener.wrap( r -> listener.onResponse(new PutFollowAction.Response(true, true, r.isAcknowledged())), listener::onFailure )); @@ -254,6 +259,6 @@ private void initiateFollowing( @Override protected ClusterBlockException checkBlock(final PutFollowAction.Request request, final ClusterState state) { - return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getFollowRequest().getFollowerIndex()); + return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getFollowerIndex()); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 67d5d7ffd74d3..409cc96b6ee12 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -15,8 +15,8 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; @@ -176,8 +177,7 @@ void start( for (int shardId = 0; shardId < numShards; shardId++) { String taskId = followIndexMetadata.getIndexUUID() + "-" + shardId; - - final ShardFollowTask shardFollowTask = createShardFollowTask(shardId, clusterNameAlias, request, + final ShardFollowTask shardFollowTask = createShardFollowTask(shardId, clusterNameAlias, request.getParameters(), leaderIndexMetadata, followIndexMetadata, filteredHeaders); persistentTasksService.sendStartRequest(taskId, ShardFollowTask.NAME, shardFollowTask, handler.getActionListener(shardId)); } @@ -189,6 +189,8 @@ static void validate( final IndexMetaData followIndex, final String[] leaderIndexHistoryUUID, final MapperService followerMapperService) { + FollowParameters parameters = request.getParameters(); + Map ccrIndexMetadata = followIndex.getCustomData(Ccr.CCR_CUSTOM_METADATA_KEY); if (ccrIndexMetadata == null) { throw new IllegalArgumentException("follow index ["+ followIndex.getIndex().getName() + "] does not have ccr metadata"); @@ -196,8 +198,8 @@ static void validate( String leaderIndexUUID = leaderIndex.getIndex().getUUID(); String recordedLeaderIndexUUID = ccrIndexMetadata.get(Ccr.CCR_CUSTOM_METADATA_LEADER_INDEX_UUID_KEY); if (leaderIndexUUID.equals(recordedLeaderIndexUUID) == false) { - throw new IllegalArgumentException("follow index [" + request.getFollowerIndex() + "] should reference [" + leaderIndexUUID + - "] as leader index but instead reference [" + recordedLeaderIndexUUID + "] as leader index"); + throw new IllegalArgumentException("follow index [" + request.getFollowerIndex() + "] should reference [" + + leaderIndexUUID + "] as leader index but instead reference [" + recordedLeaderIndexUUID + "] as leader index"); } String[] recordedHistoryUUIDs = extractLeaderShardHistoryUUIDs(ccrIndexMetadata); @@ -218,7 +220,8 @@ static void validate( + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey() + " to true"); } if (followIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) == false) { - throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() + "] does not have soft deletes enabled"); + throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() + + "] does not have soft deletes enabled"); } if (leaderIndex.getNumberOfShards() != followIndex.getNumberOfShards()) { throw new IllegalArgumentException("leader index primary shards [" + leaderIndex.getNumberOfShards() + @@ -250,69 +253,69 @@ static void validate( private static ShardFollowTask createShardFollowTask( int shardId, String clusterAliasName, - ResumeFollowAction.Request request, + FollowParameters parameters, IndexMetaData leaderIndexMetadata, IndexMetaData followIndexMetadata, Map filteredHeaders ) { int maxReadRequestOperationCount; - if (request.getMaxReadRequestOperationCount() != null) { - maxReadRequestOperationCount = request.getMaxReadRequestOperationCount(); + if (parameters.getMaxReadRequestOperationCount() != null) { + maxReadRequestOperationCount = parameters.getMaxReadRequestOperationCount(); } else { maxReadRequestOperationCount = DEFAULT_MAX_READ_REQUEST_OPERATION_COUNT; } ByteSizeValue maxReadRequestSize; - if (request.getMaxReadRequestSize() != null) { - maxReadRequestSize = request.getMaxReadRequestSize(); + if (parameters.getMaxReadRequestSize() != null) { + maxReadRequestSize = parameters.getMaxReadRequestSize(); } else { maxReadRequestSize = DEFAULT_MAX_READ_REQUEST_SIZE; } int maxOutstandingReadRequests; - if (request.getMaxOutstandingReadRequests() != null){ - maxOutstandingReadRequests = request.getMaxOutstandingReadRequests(); + if (parameters.getMaxOutstandingReadRequests() != null){ + maxOutstandingReadRequests = parameters.getMaxOutstandingReadRequests(); } else { maxOutstandingReadRequests = DEFAULT_MAX_OUTSTANDING_READ_REQUESTS; } final int maxWriteRequestOperationCount; - if (request.getMaxWriteRequestOperationCount() != null) { - maxWriteRequestOperationCount = request.getMaxWriteRequestOperationCount(); + if (parameters.getMaxWriteRequestOperationCount() != null) { + maxWriteRequestOperationCount = parameters.getMaxWriteRequestOperationCount(); } else { maxWriteRequestOperationCount = DEFAULT_MAX_WRITE_REQUEST_OPERATION_COUNT; } final ByteSizeValue maxWriteRequestSize; - if (request.getMaxWriteRequestSize() != null) { - maxWriteRequestSize = request.getMaxWriteRequestSize(); + if (parameters.getMaxWriteRequestSize() != null) { + maxWriteRequestSize = parameters.getMaxWriteRequestSize(); } else { maxWriteRequestSize = DEFAULT_MAX_WRITE_REQUEST_SIZE; } int maxOutstandingWriteRequests; - if (request.getMaxOutstandingWriteRequests() != null) { - maxOutstandingWriteRequests = request.getMaxOutstandingWriteRequests(); + if (parameters.getMaxOutstandingWriteRequests() != null) { + maxOutstandingWriteRequests = parameters.getMaxOutstandingWriteRequests(); } else { maxOutstandingWriteRequests = DEFAULT_MAX_OUTSTANDING_WRITE_REQUESTS; } int maxWriteBufferCount; - if (request.getMaxWriteBufferCount() != null) { - maxWriteBufferCount = request.getMaxWriteBufferCount(); + if (parameters.getMaxWriteBufferCount() != null) { + maxWriteBufferCount = parameters.getMaxWriteBufferCount(); } else { maxWriteBufferCount = DEFAULT_MAX_WRITE_BUFFER_COUNT; } ByteSizeValue maxWriteBufferSize; - if (request.getMaxWriteBufferSize() != null) { - maxWriteBufferSize = request.getMaxWriteBufferSize(); + if (parameters.getMaxWriteBufferSize() != null) { + maxWriteBufferSize = parameters.getMaxWriteBufferSize(); } else { maxWriteBufferSize = DEFAULT_MAX_WRITE_BUFFER_SIZE; } - TimeValue maxRetryDelay = request.getMaxRetryDelay() == null ? DEFAULT_MAX_RETRY_DELAY : request.getMaxRetryDelay(); - TimeValue readPollTimeout = request.getReadPollTimeout() == null ? DEFAULT_READ_POLL_TIMEOUT : request.getReadPollTimeout(); + TimeValue maxRetryDelay = parameters.getMaxRetryDelay() == null ? DEFAULT_MAX_RETRY_DELAY : parameters.getMaxRetryDelay(); + TimeValue readPollTimeout = parameters.getReadPollTimeout() == null ? DEFAULT_READ_POLL_TIMEOUT : parameters.getReadPollTimeout(); return new ShardFollowTask( clusterAliasName, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 1ed67edbe3962..a09e62f016983 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -429,7 +429,9 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster("leader_cluster"); request.setLeaderIndex(leaderIndex); - request.setFollowRequest(resumeFollow(followerIndex)); + request.setFollowerIndex(followerIndex); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10)); request.waitForActiveShards(waitForActiveShards); return request; } @@ -437,8 +439,8 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo public static ResumeFollowAction.Request resumeFollow(String followerIndex) { ResumeFollowAction.Request request = new ResumeFollowAction.Request(); request.setFollowerIndex(followerIndex); - request.setMaxRetryDelay(TimeValue.timeValueMillis(10)); - request.setReadPollTimeout(TimeValue.timeValueMillis(10)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10)); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java index 48531c7d28f9a..c2760aa5efd6b 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java @@ -89,8 +89,8 @@ protected AutoFollowStats getAutoFollowStats() { protected ResumeFollowAction.Request getResumeFollowRequest(String followerIndex) { ResumeFollowAction.Request request = new ResumeFollowAction.Request(); request.setFollowerIndex(followerIndex); - request.setMaxRetryDelay(TimeValue.timeValueMillis(1)); - request.setReadPollTimeout(TimeValue.timeValueMillis(1)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(1)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1)); return request; } @@ -98,7 +98,9 @@ protected PutFollowAction.Request getPutFollowRequest(String leaderIndex, String PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster("local"); request.setLeaderIndex(leaderIndex); - request.setFollowRequest(getResumeFollowRequest(followerIndex)); + request.setFollowerIndex(followerIndex); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(1)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(1)); request.waitForActiveShards(ActiveShardCount.ONE); return request; } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index 4025f647cb2a6..f12dcea4af9b5 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction; -import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowParameters; +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowerInfo; import org.elasticsearch.xpack.core.ccr.action.PutAutoFollowPatternAction; @@ -186,41 +186,42 @@ public void testAutoFollowParameterAreDelegated() throws Exception { // Enabling auto following: PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); - request.setName("my-pattern"); request.setRemoteCluster("leader_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); // Need to set this, because following an index in the same cluster request.setFollowIndexNamePattern("copy-{{leader_index}}"); if (randomBoolean()) { - request.setMaxWriteBufferCount(randomIntBetween(0, Integer.MAX_VALUE)); + request.getParameters().setMaxWriteBufferCount(randomIntBetween(0, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxConcurrentReadBatches(randomIntBetween(0, Integer.MAX_VALUE)); + request.getParameters().setMaxOutstandingReadRequests(randomIntBetween(0, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxConcurrentWriteBatches(randomIntBetween(0, Integer.MAX_VALUE)); + request.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(0, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxReadRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); + request.getParameters().setMaxReadRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); + request.getParameters().setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); } if (randomBoolean()) { - request.setMaxRetryDelay(TimeValue.timeValueMillis(500)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(500)); } if (randomBoolean()) { - request.setReadPollTimeout(TimeValue.timeValueMillis(500)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(500)); } if (randomBoolean()) { - request.setMaxWriteRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); + request.getParameters().setMaxWriteRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); + request.getParameters().setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); } if (randomBoolean()) { - request.setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong())); + request.getParameters().setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong())); } + + request.setName("my-pattern"); assertTrue(followerClient().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged()); createLeaderIndex("logs-201901", leaderIndexSettings); @@ -242,35 +243,39 @@ public void testAutoFollowParameterAreDelegated() throws Exception { FollowParameters followParameters = followerInfo.getParameters(); assertThat(followParameters, notNullValue()); - if (request.getMaxWriteBufferCount() != null) { - assertThat(followParameters.getMaxWriteBufferCount(), equalTo(request.getMaxWriteBufferCount())); + if (request.getParameters().getMaxWriteBufferCount() != null) { + assertThat(followParameters.getMaxWriteBufferCount(), equalTo(request.getParameters().getMaxWriteBufferCount())); } - if (request.getMaxWriteBufferSize() != null) { - assertThat(followParameters.getMaxWriteBufferSize(), equalTo(request.getMaxWriteBufferSize())); + if (request.getParameters().getMaxWriteBufferSize() != null) { + assertThat(followParameters.getMaxWriteBufferSize(), equalTo(request.getParameters().getMaxWriteBufferSize())); } - if (request.getMaxConcurrentReadBatches() != null) { - assertThat(followParameters.getMaxOutstandingReadRequests(), equalTo(request.getMaxConcurrentReadBatches())); + if (request.getParameters().getMaxOutstandingReadRequests() != null) { + assertThat(followParameters.getMaxOutstandingReadRequests(), + equalTo(request.getParameters().getMaxOutstandingReadRequests())); } - if (request.getMaxConcurrentWriteBatches() != null) { - assertThat(followParameters.getMaxOutstandingWriteRequests(), equalTo(request.getMaxConcurrentWriteBatches())); + if (request.getParameters().getMaxOutstandingWriteRequests() != null) { + assertThat(followParameters.getMaxOutstandingWriteRequests(), + equalTo(request.getParameters().getMaxOutstandingWriteRequests())); } - if (request.getMaxReadRequestOperationCount() != null) { - assertThat(followParameters.getMaxReadRequestOperationCount(), equalTo(request.getMaxReadRequestOperationCount())); + if (request.getParameters().getMaxReadRequestOperationCount() != null) { + assertThat(followParameters.getMaxReadRequestOperationCount(), + equalTo(request.getParameters().getMaxReadRequestOperationCount())); } - if (request.getMaxReadRequestSize() != null) { - assertThat(followParameters.getMaxReadRequestSize(), equalTo(request.getMaxReadRequestSize())); + if (request.getParameters().getMaxReadRequestSize() != null) { + assertThat(followParameters.getMaxReadRequestSize(), equalTo(request.getParameters().getMaxReadRequestSize())); } - if (request.getMaxRetryDelay() != null) { - assertThat(followParameters.getMaxRetryDelay(), equalTo(request.getMaxRetryDelay())); + if (request.getParameters().getMaxRetryDelay() != null) { + assertThat(followParameters.getMaxRetryDelay(), equalTo(request.getParameters().getMaxRetryDelay())); } - if (request.getReadPollTimeout() != null) { - assertThat(followParameters.getReadPollTimeout(), equalTo(request.getReadPollTimeout())); + if (request.getParameters().getReadPollTimeout() != null) { + assertThat(followParameters.getReadPollTimeout(), equalTo(request.getParameters().getReadPollTimeout())); } - if (request.getMaxWriteRequestOperationCount() != null) { - assertThat(followParameters.getMaxWriteRequestOperationCount(), equalTo(request.getMaxWriteRequestOperationCount())); + if (request.getParameters().getMaxWriteRequestOperationCount() != null) { + assertThat(followParameters.getMaxWriteRequestOperationCount(), + equalTo(request.getParameters().getMaxWriteRequestOperationCount())); } - if (request.getMaxWriteRequestSize() != null) { - assertThat(followParameters.getMaxWriteRequestSize(), equalTo(request.getMaxWriteRequestSize())); + if (request.getParameters().getMaxWriteRequestSize() != null) { + assertThat(followParameters.getMaxWriteRequestSize(), equalTo(request.getParameters().getMaxWriteRequestSize())); } }); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 925b9311e810d..415a90c3956f7 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -90,13 +90,13 @@ public void testFailOverOnFollower() throws Exception { } availableDocs.release(between(100, 200)); PutFollowAction.Request follow = putFollow("leader-index", "follower-index"); - follow.getFollowRequest().setMaxReadRequestOperationCount(randomIntBetween(32, 2048)); - follow.getFollowRequest().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); - follow.getFollowRequest().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); - follow.getFollowRequest().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048)); - follow.getFollowRequest().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); - follow.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); - logger.info("--> follow params {}", Strings.toString(follow.getFollowRequest())); + follow.getParameters().setMaxReadRequestOperationCount(randomIntBetween(32, 2048)); + follow.getParameters().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); + follow.getParameters().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); + follow.getParameters().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048)); + follow.getParameters().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); + follow.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); + logger.info("--> follow request {}", Strings.toString(follow)); followerClient().execute(PutFollowAction.INSTANCE, follow).get(); disableDelayedAllocation("follower-index"); ensureFollowerGreen("follower-index"); @@ -151,17 +151,17 @@ public void testFollowIndexAndCloseNode() throws Exception { thread.start(); PutFollowAction.Request followRequest = putFollow("index1", "index2"); - followRequest.getFollowRequest().setMaxReadRequestOperationCount(randomIntBetween(32, 2048)); - followRequest.getFollowRequest().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); - followRequest.getFollowRequest().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); - followRequest.getFollowRequest().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048)); - followRequest.getFollowRequest().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); - followRequest.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); + followRequest.getParameters().setMaxReadRequestOperationCount(randomIntBetween(32, 2048)); + followRequest.getParameters().setMaxReadRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); + followRequest.getParameters().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); + followRequest.getParameters().setMaxWriteRequestOperationCount(randomIntBetween(32, 2048)); + followRequest.getParameters().setMaxWriteRequestSize(new ByteSizeValue(randomIntBetween(1, 4096), ByteSizeUnit.KB)); + followRequest.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); disableDelayedAllocation("index2"); - logger.info("--> follow params {}", Strings.toString(followRequest.getFollowRequest())); + logger.info("--> follow request {}", Strings.toString(followRequest)); - int maxOpsPerRead = followRequest.getFollowRequest().getMaxReadRequestOperationCount(); + int maxOpsPerRead = followRequest.getParameters().getMaxReadRequestOperationCount(); int maxNumDocsReplicated = Math.min(between(50, 500), between(maxOpsPerRead, maxOpsPerRead * 10)); availableDocs.release(maxNumDocsReplicated / 2 + 1); atLeastDocsIndexed(followerClient(), "index2", maxNumDocsReplicated / 3); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 6fa5eec3ccaaa..acd380b88e498 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -180,7 +180,7 @@ public void testFollowIndex() throws Exception { } pauseFollow("index2"); - followerClient().execute(ResumeFollowAction.INSTANCE, followRequest.getFollowRequest()).get(); + followerClient().execute(ResumeFollowAction.INSTANCE, resumeFollow("index2")).get(); final int secondBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); for (int i = firstBatchNumDocs; i < firstBatchNumDocs + secondBatchNumDocs; i++) { @@ -445,10 +445,10 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) atLeastDocsIndexed(leaderClient(), "index1", numDocsIndexed / 3); PutFollowAction.Request followRequest = putFollow("index1", "index2"); - followRequest.getFollowRequest().setMaxReadRequestOperationCount(maxOpsPerRead); - followRequest.getFollowRequest().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); - followRequest.getFollowRequest().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); - followRequest.getFollowRequest().setMaxWriteBufferCount(randomIntBetween(1024, 10240)); + followRequest.getParameters().setMaxReadRequestOperationCount(maxOpsPerRead); + followRequest.getParameters().setMaxOutstandingReadRequests(randomIntBetween(1, 10)); + followRequest.getParameters().setMaxOutstandingWriteRequests(randomIntBetween(1, 10)); + followRequest.getParameters().setMaxWriteBufferCount(randomIntBetween(1024, 10240)); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); availableDocs.release(numDocsIndexed * 2 + bulkSize); atLeastDocsIndexed(leaderClient(), "index1", numDocsIndexed); @@ -543,7 +543,7 @@ public void testFollowIndexMaxOperationSizeInBytes() throws Exception { } PutFollowAction.Request followRequest = putFollow("index1", "index2"); - followRequest.getFollowRequest().setMaxReadRequestSize(new ByteSizeValue(1, ByteSizeUnit.BYTES)); + followRequest.getParameters().setMaxReadRequestSize(new ByteSizeValue(1, ByteSizeUnit.BYTES)); followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); final Map firstBatchNumDocsPerShard = new HashMap<>(); @@ -1012,7 +1012,7 @@ public void testIndexFallBehind() throws Exception { forceMergeRequest.maxNumSegments(1); leaderClient().admin().indices().forceMerge(forceMergeRequest).actionGet(); - followerClient().execute(ResumeFollowAction.INSTANCE, followRequest.getFollowRequest()).get(); + followerClient().execute(ResumeFollowAction.INSTANCE, resumeFollow("index2")).get(); assertBusy(() -> { List statuses = getFollowTaskStatuses("index2"); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java index 9d8e0146bfc51..45109bc3aa3fe 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java @@ -84,7 +84,7 @@ public void testDoNotCreateFollowerIfLeaderDoesNotHaveSoftDeletes() throws Excep followRequest.setFollowerIndex("follower-index"); PutFollowAction.Request putFollowRequest = getPutFollowRequest("leader", "follower"); putFollowRequest.setLeaderIndex("leader-index"); - putFollowRequest.setFollowRequest(followRequest); + putFollowRequest.setFollowerIndex("follower-index"); IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> client().execute(PutFollowAction.INSTANCE, putFollowRequest).actionGet()); assertThat(error.getMessage(), equalTo("leader index [leader-index] does not have soft deletes enabled. " + @@ -99,7 +99,7 @@ public void testRemoveRemoteConnection() throws Exception { request.setRemoteCluster("local"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); request.setFollowIndexNamePattern("copy-{{leader_index}}"); - request.setReadPollTimeout(TimeValue.timeValueMillis(10)); + request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10)); assertTrue(client().execute(PutAutoFollowPatternAction.INSTANCE, request).actionGet().isAcknowledged()); long previousNumberOfSuccessfulFollowedIndices = getAutoFollowStats().getNumberOfSuccessfulFollowIndices(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 4d4603d022f7d..2037c7faaa7b4 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -109,7 +109,7 @@ void createAndFollow(Map headers, assertThat(headers, equalTo(autoFollowHeaders.get("remote"))); assertThat(followRequest.getRemoteCluster(), equalTo("remote")); assertThat(followRequest.getLeaderIndex(), equalTo("logs-20190101")); - assertThat(followRequest.getFollowRequest().getFollowerIndex(), equalTo("logs-20190101")); + assertThat(followRequest.getFollowerIndex(), equalTo("logs-20190101")); successHandler.run(); } @@ -227,7 +227,7 @@ void createAndFollow(Map headers, Consumer failureHandler) { assertThat(followRequest.getRemoteCluster(), equalTo("remote")); assertThat(followRequest.getLeaderIndex(), equalTo("logs-20190101")); - assertThat(followRequest.getFollowRequest().getFollowerIndex(), equalTo("logs-20190101")); + assertThat(followRequest.getFollowerIndex(), equalTo("logs-20190101")); successHandler.run(); } @@ -284,7 +284,7 @@ void createAndFollow(Map headers, Consumer failureHandler) { assertThat(followRequest.getRemoteCluster(), equalTo("remote")); assertThat(followRequest.getLeaderIndex(), equalTo("logs-20190101")); - assertThat(followRequest.getFollowRequest().getFollowerIndex(), equalTo("logs-20190101")); + assertThat(followRequest.getFollowerIndex(), equalTo("logs-20190101")); failureHandler.accept(failure); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java index d21098506a121..eceb37819d187 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java @@ -20,61 +20,13 @@ import java.util.List; import static org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FOLLOWER_INDICES_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.FollowParameters; + +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import static org.elasticsearch.xpack.core.ccr.action.FollowInfoAction.Response.Status; public class FollowInfoResponseTests extends AbstractSerializingTestCase { - static final ConstructingObjectParser PARAMETERS_PARSER = new ConstructingObjectParser<>( - "parameters_parser", - args -> { - return new FollowParameters( - (Integer) args[0], - (ByteSizeValue) args[1], - (Integer) args[2], - (Integer) args[3], - (ByteSizeValue) args[4], - (Integer) args[5], - (Integer) args[6], - (ByteSizeValue) args[7], - (TimeValue) args[8], - (TimeValue) args[9] - ); - }); - - static { - PARAMETERS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTask.MAX_READ_REQUEST_OPERATION_COUNT); - PARAMETERS_PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), ShardFollowTask.MAX_READ_REQUEST_SIZE.getPreferredName()), - ShardFollowTask.MAX_READ_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARAMETERS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTask.MAX_OUTSTANDING_READ_REQUESTS); - PARAMETERS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTask.MAX_WRITE_REQUEST_OPERATION_COUNT); - PARAMETERS_PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), ShardFollowTask.MAX_WRITE_REQUEST_SIZE.getPreferredName()), - ShardFollowTask.MAX_WRITE_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARAMETERS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTask.MAX_OUTSTANDING_WRITE_REQUESTS); - PARAMETERS_PARSER.declareInt(ConstructingObjectParser.constructorArg(), ShardFollowTask.MAX_WRITE_BUFFER_COUNT); - PARAMETERS_PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), ShardFollowTask.MAX_WRITE_BUFFER_SIZE.getPreferredName()), - ShardFollowTask.MAX_WRITE_BUFFER_SIZE, - ObjectParser.ValueType.STRING); - PARAMETERS_PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> TimeValue.parseTimeValue(p.text(), ShardFollowTask.MAX_RETRY_DELAY.getPreferredName()), - ShardFollowTask.MAX_RETRY_DELAY, - ObjectParser.ValueType.STRING); - PARAMETERS_PARSER.declareField( - ConstructingObjectParser.constructorArg(), - (p, c) -> TimeValue.parseTimeValue(p.text(), ShardFollowTask.READ_POLL_TIMEOUT.getPreferredName()), - ShardFollowTask.READ_POLL_TIMEOUT, - ObjectParser.ValueType.STRING); - } - + static final ObjectParser PARAMETERS_PARSER = new ObjectParser<>("parameters_parser", FollowParameters::new); static final ConstructingObjectParser INFO_PARSER = new ConstructingObjectParser<>( "info_parser", args -> { @@ -88,6 +40,8 @@ public class FollowInfoResponseTests extends AbstractSerializingTestCase instanceReader() protected PutAutoFollowPatternAction.Request createTestInstance() { PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); request.setName(randomAlphaOfLength(4)); + request.setRemoteCluster(randomAlphaOfLength(4)); request.setLeaderIndexPatterns(Arrays.asList(generateRandomStringArray(4, 4, false))); if (randomBoolean()) { request.setFollowIndexNamePattern(randomAlphaOfLength(4)); } + ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); + return request; + } + + @Override + protected PutAutoFollowPatternAction.Request createXContextTestInstance(XContentType xContentType) { + // follower index parameter is not part of the request body and is provided in the url path. + // So this field cannot be used for creating a test instance for xcontent testing. + PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + request.setRemoteCluster(randomAlphaOfLength(4)); + request.setLeaderIndexPatterns(Arrays.asList(generateRandomStringArray(4, 4, false))); if (randomBoolean()) { - request.setReadPollTimeout(TimeValue.timeValueMillis(500)); - } - if (randomBoolean()) { - request.setMaxRetryDelay(TimeValue.timeValueMillis(500)); - } - if (randomBoolean()) { - request.setMaxWriteRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); - } - if (randomBoolean()) { - request.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong())); - } - if (randomBoolean()) { - request.setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong())); - } - if (randomBoolean()) { - request.setMaxReadRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE)); - } - if (randomBoolean()) { - request.setMaxConcurrentReadBatches(randomIntBetween(0, Integer.MAX_VALUE)); - } - if (randomBoolean()) { - request.setMaxConcurrentWriteBatches(randomIntBetween(0, Integer.MAX_VALUE)); - } - if (randomBoolean()) { - request.setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); - } - if (randomBoolean()) { - request.setMaxWriteBufferCount(randomIntBetween(0, Integer.MAX_VALUE)); - } - if (randomBoolean()) { - request.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong())); + request.setFollowIndexNamePattern(randomAlphaOfLength(4)); } + ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); return request; } @@ -109,17 +91,17 @@ public void testValidate() { validationException = request.validate(); assertThat(validationException, nullValue()); - request.setMaxRetryDelay(TimeValue.ZERO); + request.getParameters().setMaxRetryDelay(TimeValue.ZERO); validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be positive but was [0ms]")); - request.setMaxRetryDelay(TimeValue.timeValueMinutes(10)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMinutes(10)); validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be less than [5m] but was [10m]")); - request.setMaxRetryDelay(TimeValue.timeValueMinutes(1)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMinutes(1)); validationException = request.validate(); assertThat(validationException, nullValue()); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java index d32a773ebe218..02b5eca08fa4a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; @@ -22,16 +23,32 @@ protected Writeable.Reader instanceReader() { @Override protected PutFollowAction.Request createTestInstance() { + PutFollowAction.Request request = new PutFollowAction.Request(); + request.setFollowerIndex(randomAlphaOfLength(4)); + request.waitForActiveShards(randomFrom(ActiveShardCount.DEFAULT, ActiveShardCount.NONE, ActiveShardCount.ONE, + ActiveShardCount.ALL)); + + request.setRemoteCluster(randomAlphaOfLength(4)); + request.setLeaderIndex(randomAlphaOfLength(4)); + ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); + return request; + } + + @Override + protected PutFollowAction.Request createXContextTestInstance(XContentType xContentType) { + // follower index parameter and wait for active shards params are not part of the request body and + // are provided in the url path. So these fields cannot be used for creating a test instance for xcontent testing. PutFollowAction.Request request = new PutFollowAction.Request(); request.setRemoteCluster(randomAlphaOfLength(4)); request.setLeaderIndex(randomAlphaOfLength(4)); - request.setFollowRequest(ResumeFollowActionRequestTests.createTestRequest()); + request.setFollowerIndex("followerIndex"); + ResumeFollowActionRequestTests.generateFollowParameters(request.getParameters()); return request; } @Override protected PutFollowAction.Request doParseInstance(XContentParser parser) throws IOException { - return PutFollowAction.Request.fromXContent(parser, null, ActiveShardCount.DEFAULT); + return PutFollowAction.Request.fromXContent(parser, "followerIndex", ActiveShardCount.DEFAULT); } @Override diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java index 3d3e869f53e8a..53efac70a7dc0 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java @@ -11,7 +11,9 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.ccr.action.FollowParameters; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; @@ -29,7 +31,20 @@ protected Writeable.Reader instanceReader() { @Override protected ResumeFollowAction.Request createTestInstance() { - return createTestRequest(); + ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + request.setFollowerIndex(randomAlphaOfLength(4)); + + generateFollowParameters(request.getParameters()); + return request; + } + + @Override + protected ResumeFollowAction.Request createXContextTestInstance(XContentType type) { + // follower index parameter is not part of the request body and is provided in the url path. + // So this field cannot be used for creating a test instance for xcontent testing. + ResumeFollowAction.Request request = new ResumeFollowAction.Request(); + generateFollowParameters(request.getParameters()); + return request; } @Override @@ -42,57 +57,54 @@ protected boolean supportsUnknownFields() { return false; } - static ResumeFollowAction.Request createTestRequest() { - ResumeFollowAction.Request request = new ResumeFollowAction.Request(); - request.setFollowerIndex(randomAlphaOfLength(4)); + static void generateFollowParameters(FollowParameters followParameters) { if (randomBoolean()) { - request.setMaxReadRequestOperationCount(randomIntBetween(1, Integer.MAX_VALUE)); + followParameters.setMaxReadRequestOperationCount(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxOutstandingReadRequests(randomIntBetween(1, Integer.MAX_VALUE)); + followParameters.setMaxOutstandingReadRequests(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxOutstandingWriteRequests(randomIntBetween(1, Integer.MAX_VALUE)); + followParameters.setMaxOutstandingWriteRequests(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); + followParameters.setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); } if (randomBoolean()) { - request.setMaxWriteBufferCount(randomIntBetween(1, Integer.MAX_VALUE)); + followParameters.setMaxWriteBufferCount(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxWriteRequestOperationCount(randomIntBetween(1, Integer.MAX_VALUE)); + followParameters.setMaxWriteRequestOperationCount(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - request.setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong())); + followParameters.setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong())); } if (randomBoolean()) { - request.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); + followParameters.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES)); } if (randomBoolean()) { - request.setMaxRetryDelay(TimeValue.timeValueMillis(500)); + followParameters.setMaxRetryDelay(TimeValue.timeValueMillis(500)); } if (randomBoolean()) { - request.setReadPollTimeout(TimeValue.timeValueMillis(500)); + followParameters.setReadPollTimeout(TimeValue.timeValueMillis(500)); } - return request; } public void testValidate() { ResumeFollowAction.Request request = new ResumeFollowAction.Request(); request.setFollowerIndex("index2"); - request.setMaxRetryDelay(TimeValue.ZERO); + request.getParameters().setMaxRetryDelay(TimeValue.ZERO); ActionRequestValidationException validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be positive but was [0ms]")); - request.setMaxRetryDelay(TimeValue.timeValueMinutes(10)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMinutes(10)); validationException = request.validate(); assertThat(validationException, notNullValue()); assertThat(validationException.getMessage(), containsString("[max_retry_delay] must be less than [5m] but was [10m]")); - request.setMaxRetryDelay(TimeValue.timeValueMinutes(1)); + request.getParameters().setMaxRetryDelay(TimeValue.timeValueMinutes(1)); validationException = request.validate(); assertThat(validationException, nullValue()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java index 7ea7975c05f22..2f523883ce0d9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java @@ -16,8 +16,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -26,17 +24,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_READ_REQUESTS; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_WRITE_REQUESTS; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_RETRY_DELAY_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_REQUEST_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.READ_POLL_TIMEOUT; - public class FollowInfoAction extends Action { public static final String NAME = "cluster:monitor/ccr/follow_info"; @@ -204,7 +191,7 @@ public FollowParameters getParameters() { remoteCluster = in.readString(); leaderIndex = in.readString(); status = Status.fromString(in.readString()); - parameters = in.readOptionalWriteable(FollowParameters::new); + parameters = in.readOptionalWriteable(innerIn -> new FollowParameters(in)); } @Override @@ -226,16 +213,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (parameters != null) { builder.startObject(PARAMETERS_FIELD.getPreferredName()); { - builder.field(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName(), parameters.maxReadRequestOperationCount); - builder.field(MAX_READ_REQUEST_SIZE.getPreferredName(), parameters.maxReadRequestSize.getStringRep()); - builder.field(MAX_OUTSTANDING_READ_REQUESTS.getPreferredName(), parameters.maxOutstandingReadRequests); - builder.field(MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName(), parameters.maxWriteRequestOperationCount); - builder.field(MAX_WRITE_REQUEST_SIZE.getPreferredName(), parameters.maxWriteRequestSize.getStringRep()); - builder.field(MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName(), parameters.maxOutstandingWriteRequests); - builder.field(MAX_WRITE_BUFFER_COUNT.getPreferredName(), parameters.maxWriteBufferCount); - builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), parameters.maxWriteBufferSize.getStringRep()); - builder.field(MAX_RETRY_DELAY_FIELD.getPreferredName(), parameters.maxRetryDelay.getStringRep()); - builder.field(READ_POLL_TIMEOUT.getPreferredName(), parameters.readPollTimeout.getStringRep()); + parameters.toXContentFragment(builder); } builder.endObject(); } @@ -265,138 +243,6 @@ public String toString() { } } - public static class FollowParameters implements Writeable { - - private final int maxReadRequestOperationCount; - private final ByteSizeValue maxReadRequestSize; - private final int maxOutstandingReadRequests; - private final int maxWriteRequestOperationCount; - private final ByteSizeValue maxWriteRequestSize; - private final int maxOutstandingWriteRequests; - private final int maxWriteBufferCount; - private final ByteSizeValue maxWriteBufferSize; - private final TimeValue maxRetryDelay; - private final TimeValue readPollTimeout; - - public FollowParameters(int maxReadRequestOperationCount, - ByteSizeValue maxReadRequestSize, int maxOutstandingReadRequests, - int maxWriteRequestOperationCount, ByteSizeValue maxWriteRequestSize, - int maxOutstandingWriteRequests, int maxWriteBufferCount, - ByteSizeValue maxWriteBufferSize, TimeValue maxRetryDelay, TimeValue readPollTimeout) { - this.maxReadRequestOperationCount = maxReadRequestOperationCount; - this.maxReadRequestSize = maxReadRequestSize; - this.maxOutstandingReadRequests = maxOutstandingReadRequests; - this.maxWriteRequestOperationCount = maxWriteRequestOperationCount; - this.maxWriteRequestSize = maxWriteRequestSize; - this.maxOutstandingWriteRequests = maxOutstandingWriteRequests; - this.maxWriteBufferCount = maxWriteBufferCount; - this.maxWriteBufferSize = maxWriteBufferSize; - this.maxRetryDelay = maxRetryDelay; - this.readPollTimeout = readPollTimeout; - } - - public int getMaxReadRequestOperationCount() { - return maxReadRequestOperationCount; - } - - public ByteSizeValue getMaxReadRequestSize() { - return maxReadRequestSize; - } - - public int getMaxOutstandingReadRequests() { - return maxOutstandingReadRequests; - } - - public int getMaxWriteRequestOperationCount() { - return maxWriteRequestOperationCount; - } - - public ByteSizeValue getMaxWriteRequestSize() { - return maxWriteRequestSize; - } - - public int getMaxOutstandingWriteRequests() { - return maxOutstandingWriteRequests; - } - - public int getMaxWriteBufferCount() { - return maxWriteBufferCount; - } - - public ByteSizeValue getMaxWriteBufferSize() { - return maxWriteBufferSize; - } - - public TimeValue getMaxRetryDelay() { - return maxRetryDelay; - } - - public TimeValue getReadPollTimeout() { - return readPollTimeout; - } - - FollowParameters(StreamInput in) throws IOException { - this.maxReadRequestOperationCount = in.readVInt(); - this.maxReadRequestSize = new ByteSizeValue(in); - this.maxOutstandingReadRequests = in.readVInt(); - this.maxWriteRequestOperationCount = in.readVInt(); - this.maxWriteRequestSize = new ByteSizeValue(in); - this.maxOutstandingWriteRequests = in.readVInt(); - this.maxWriteBufferCount = in.readVInt(); - this.maxWriteBufferSize = new ByteSizeValue(in); - this.maxRetryDelay = in.readTimeValue(); - this.readPollTimeout = in.readTimeValue(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(maxReadRequestOperationCount); - maxReadRequestSize.writeTo(out); - out.writeVInt(maxOutstandingReadRequests); - out.writeVLong(maxWriteRequestOperationCount); - maxWriteRequestSize.writeTo(out); - out.writeVInt(maxOutstandingWriteRequests); - out.writeVInt(maxWriteBufferCount); - maxWriteBufferSize.writeTo(out); - out.writeTimeValue(maxRetryDelay); - out.writeTimeValue(readPollTimeout); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FollowParameters that = (FollowParameters) o; - return maxReadRequestOperationCount == that.maxReadRequestOperationCount && - maxOutstandingReadRequests == that.maxOutstandingReadRequests && - maxWriteRequestOperationCount == that.maxWriteRequestOperationCount && - maxOutstandingWriteRequests == that.maxOutstandingWriteRequests && - maxWriteBufferCount == that.maxWriteBufferCount && - Objects.equals(maxReadRequestSize, that.maxReadRequestSize) && - Objects.equals(maxWriteRequestSize, that.maxWriteRequestSize) && - Objects.equals(maxWriteBufferSize, that.maxWriteBufferSize) && - Objects.equals(maxRetryDelay, that.maxRetryDelay) && - Objects.equals(readPollTimeout, that.readPollTimeout); - } - - @Override - public int hashCode() { - return Objects.hash( - maxReadRequestOperationCount, - maxReadRequestSize, - maxOutstandingReadRequests, - maxWriteRequestOperationCount, - maxWriteRequestSize, - maxOutstandingWriteRequests, - maxWriteBufferCount, - maxWriteBufferSize, - maxRetryDelay, - readPollTimeout - ); - } - - } - public enum Status { ACTIVE("active"), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowParameters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowParameters.java new file mode 100644 index 0000000000000..001a79323ab38 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowParameters.java @@ -0,0 +1,314 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ccr.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.AbstractObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class FollowParameters implements Writeable { + + static final TimeValue RETRY_DELAY_MAX = TimeValue.timeValueMinutes(5); + + static final ParseField MAX_READ_REQUEST_OPERATION_COUNT = new ParseField("max_read_request_operation_count"); + static final ParseField MAX_WRITE_REQUEST_OPERATION_COUNT = new ParseField("max_write_request_operation_count"); + static final ParseField MAX_OUTSTANDING_READ_REQUESTS = new ParseField("max_outstanding_read_requests"); + static final ParseField MAX_OUTSTANDING_WRITE_REQUESTS = new ParseField("max_outstanding_write_requests"); + static final ParseField MAX_READ_REQUEST_SIZE = new ParseField("max_read_request_size"); + static final ParseField MAX_WRITE_REQUEST_SIZE = new ParseField("max_write_request_size"); + static final ParseField MAX_WRITE_BUFFER_COUNT = new ParseField("max_write_buffer_count"); + static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size"); + static final ParseField MAX_RETRY_DELAY = new ParseField("max_retry_delay"); + static final ParseField READ_POLL_TIMEOUT = new ParseField("read_poll_timeout"); + + Integer maxReadRequestOperationCount; + Integer maxWriteRequestOperationCount; + Integer maxOutstandingReadRequests; + Integer maxOutstandingWriteRequests; + ByteSizeValue maxReadRequestSize; + ByteSizeValue maxWriteRequestSize; + Integer maxWriteBufferCount; + ByteSizeValue maxWriteBufferSize; + TimeValue maxRetryDelay; + TimeValue readPollTimeout; + + public FollowParameters() { + } + + public FollowParameters(FollowParameters source) { + this.maxReadRequestOperationCount = source.maxReadRequestOperationCount; + this.maxWriteRequestOperationCount = source.maxWriteRequestOperationCount; + this.maxOutstandingReadRequests = source.maxOutstandingReadRequests; + this.maxOutstandingWriteRequests = source.maxOutstandingWriteRequests; + this.maxReadRequestSize = source.maxReadRequestSize; + this.maxWriteRequestSize = source.maxWriteRequestSize; + this.maxWriteBufferCount = source.maxWriteBufferCount; + this.maxWriteBufferSize = source.maxWriteBufferSize; + this.maxRetryDelay = source.maxRetryDelay; + this.readPollTimeout = source.readPollTimeout; + } + + public Integer getMaxReadRequestOperationCount() { + return maxReadRequestOperationCount; + } + + public void setMaxReadRequestOperationCount(Integer maxReadRequestOperationCount) { + this.maxReadRequestOperationCount = maxReadRequestOperationCount; + } + + public ByteSizeValue getMaxReadRequestSize() { + return maxReadRequestSize; + } + + public void setMaxReadRequestSize(ByteSizeValue maxReadRequestSize) { + this.maxReadRequestSize = maxReadRequestSize; + } + + public Integer getMaxOutstandingReadRequests() { + return maxOutstandingReadRequests; + } + + public void setMaxOutstandingReadRequests(Integer maxOutstandingReadRequests) { + this.maxOutstandingReadRequests = maxOutstandingReadRequests; + } + + public Integer getMaxWriteRequestOperationCount() { + return maxWriteRequestOperationCount; + } + + public void setMaxWriteRequestOperationCount(Integer maxWriteRequestOperationCount) { + this.maxWriteRequestOperationCount = maxWriteRequestOperationCount; + } + + public ByteSizeValue getMaxWriteRequestSize() { + return maxWriteRequestSize; + } + + public void setMaxWriteRequestSize(ByteSizeValue maxWriteRequestSize) { + this.maxWriteRequestSize = maxWriteRequestSize; + } + + public Integer getMaxOutstandingWriteRequests() { + return maxOutstandingWriteRequests; + } + + public void setMaxOutstandingWriteRequests(Integer maxOutstandingWriteRequests) { + this.maxOutstandingWriteRequests = maxOutstandingWriteRequests; + } + + public Integer getMaxWriteBufferCount() { + return maxWriteBufferCount; + } + + public void setMaxWriteBufferCount(Integer maxWriteBufferCount) { + this.maxWriteBufferCount = maxWriteBufferCount; + } + + public ByteSizeValue getMaxWriteBufferSize() { + return maxWriteBufferSize; + } + + public void setMaxWriteBufferSize(ByteSizeValue maxWriteBufferSize) { + this.maxWriteBufferSize = maxWriteBufferSize; + } + + public TimeValue getMaxRetryDelay() { + return maxRetryDelay; + } + + public void setMaxRetryDelay(TimeValue maxRetryDelay) { + this.maxRetryDelay = maxRetryDelay; + } + + public TimeValue getReadPollTimeout() { + return readPollTimeout; + } + + public void setReadPollTimeout(TimeValue readPollTimeout) { + this.readPollTimeout = readPollTimeout; + } + + public ActionRequestValidationException validate() { + ActionRequestValidationException e = null; + + if (maxReadRequestOperationCount != null && maxReadRequestOperationCount < 1) { + e = addValidationError(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e); + } + if (maxReadRequestSize != null && maxReadRequestSize.compareTo(ByteSizeValue.ZERO) <= 0) { + e = addValidationError(MAX_READ_REQUEST_SIZE.getPreferredName() + " must be larger than 0", e); + } + if (maxOutstandingReadRequests != null && maxOutstandingReadRequests < 1) { + e = addValidationError(MAX_OUTSTANDING_READ_REQUESTS.getPreferredName() + " must be larger than 0", e); + } + if (maxWriteRequestOperationCount != null && maxWriteRequestOperationCount < 1) { + e = addValidationError(MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e); + } + if (maxWriteRequestSize != null && maxWriteRequestSize.compareTo(ByteSizeValue.ZERO) <= 0) { + e = addValidationError(MAX_WRITE_REQUEST_SIZE.getPreferredName() + " must be larger than 0", e); + } + if (maxOutstandingWriteRequests != null && maxOutstandingWriteRequests < 1) { + e = addValidationError(MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName() + " must be larger than 0", e); + } + if (maxWriteBufferCount != null && maxWriteBufferCount < 1) { + e = addValidationError(MAX_WRITE_BUFFER_COUNT.getPreferredName() + " must be larger than 0", e); + } + if (maxWriteBufferSize != null && maxWriteBufferSize.compareTo(ByteSizeValue.ZERO) <= 0) { + e = addValidationError(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0", e); + } + if (maxRetryDelay != null && maxRetryDelay.millis() <= 0) { + String message = "[" + MAX_RETRY_DELAY.getPreferredName() + "] must be positive but was [" + + maxRetryDelay.getStringRep() + "]"; + e = addValidationError(message, e); + } + if (maxRetryDelay != null && maxRetryDelay.millis() > RETRY_DELAY_MAX.millis()) { + String message = "[" + MAX_RETRY_DELAY.getPreferredName() + "] must be less than [" + RETRY_DELAY_MAX.getStringRep() + + "] but was [" + maxRetryDelay.getStringRep() + "]"; + e = addValidationError(message, e); + } + + return e; + } + + FollowParameters(StreamInput in) throws IOException { + fromStreamInput(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalVInt(maxReadRequestOperationCount); + out.writeOptionalVInt(maxOutstandingReadRequests); + out.writeOptionalWriteable(maxReadRequestSize); + out.writeOptionalVInt(maxWriteRequestOperationCount); + out.writeOptionalWriteable(maxWriteRequestSize); + out.writeOptionalVInt(maxOutstandingWriteRequests); + out.writeOptionalVInt(maxWriteBufferCount); + out.writeOptionalWriteable(maxWriteBufferSize); + out.writeOptionalTimeValue(maxRetryDelay); + out.writeOptionalTimeValue(readPollTimeout); + } + + void fromStreamInput(StreamInput in) throws IOException { + maxReadRequestOperationCount = in.readOptionalVInt(); + maxOutstandingReadRequests = in.readOptionalVInt(); + maxReadRequestSize = in.readOptionalWriteable(ByteSizeValue::new); + maxWriteRequestOperationCount = in.readOptionalVInt(); + maxWriteRequestSize = in.readOptionalWriteable(ByteSizeValue::new); + maxOutstandingWriteRequests = in.readOptionalVInt(); + maxWriteBufferCount = in.readOptionalVInt(); + maxWriteBufferSize = in.readOptionalWriteable(ByteSizeValue::new); + maxRetryDelay = in.readOptionalTimeValue(); + readPollTimeout = in.readOptionalTimeValue(); + } + + XContentBuilder toXContentFragment(final XContentBuilder builder) throws IOException { + if (maxReadRequestOperationCount != null) { + builder.field(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName(), maxReadRequestOperationCount); + } + if (maxWriteRequestOperationCount != null) { + builder.field(MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName(), maxWriteRequestOperationCount); + } + if (maxOutstandingReadRequests != null) { + builder.field(MAX_OUTSTANDING_READ_REQUESTS.getPreferredName(), maxOutstandingReadRequests); + } + if (maxOutstandingWriteRequests != null) { + builder.field(MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName(), maxOutstandingWriteRequests); + } + if (maxReadRequestSize != null) { + builder.field(MAX_READ_REQUEST_SIZE.getPreferredName(), maxReadRequestSize.getStringRep()); + } + if (maxWriteRequestSize != null) { + builder.field(MAX_WRITE_REQUEST_SIZE.getPreferredName(), maxWriteRequestSize.getStringRep()); + } + if (maxWriteBufferCount != null) { + builder.field(MAX_WRITE_BUFFER_COUNT.getPreferredName(), maxWriteBufferCount); + } + if (maxWriteBufferSize != null) { + builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize.getStringRep()); + } + if (maxRetryDelay != null) { + builder.field(MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay.getStringRep()); + } + if (readPollTimeout != null) { + builder.field(READ_POLL_TIMEOUT.getPreferredName(), readPollTimeout.getStringRep()); + } + return builder; + } + + public static

void initParser(AbstractObjectParser parser) { + parser.declareInt(FollowParameters::setMaxReadRequestOperationCount, MAX_READ_REQUEST_OPERATION_COUNT); + parser.declareInt(FollowParameters::setMaxWriteRequestOperationCount, MAX_WRITE_REQUEST_OPERATION_COUNT); + parser.declareInt(FollowParameters::setMaxOutstandingReadRequests, MAX_OUTSTANDING_READ_REQUESTS); + parser.declareInt(FollowParameters::setMaxOutstandingWriteRequests, MAX_OUTSTANDING_WRITE_REQUESTS); + parser.declareField( + FollowParameters::setMaxReadRequestSize, + (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_READ_REQUEST_SIZE.getPreferredName()), + AutoFollowMetadata.AutoFollowPattern.MAX_READ_REQUEST_SIZE, + ObjectParser.ValueType.STRING); + parser.declareField( + FollowParameters::setMaxWriteRequestSize, + (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_REQUEST_SIZE.getPreferredName()), + AutoFollowMetadata.AutoFollowPattern.MAX_WRITE_REQUEST_SIZE, + ObjectParser.ValueType.STRING); + parser.declareInt(FollowParameters::setMaxWriteBufferCount, MAX_WRITE_BUFFER_COUNT); + parser.declareField( + FollowParameters::setMaxWriteBufferSize, + (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_BUFFER_SIZE.getPreferredName()), + MAX_WRITE_BUFFER_SIZE, + ObjectParser.ValueType.STRING); + parser.declareField(FollowParameters::setMaxRetryDelay, + (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY.getPreferredName()), + MAX_RETRY_DELAY, ObjectParser.ValueType.STRING); + parser.declareField(FollowParameters::setReadPollTimeout, + (p, c) -> TimeValue.parseTimeValue(p.text(), READ_POLL_TIMEOUT.getPreferredName()), + READ_POLL_TIMEOUT, ObjectParser.ValueType.STRING); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o instanceof FollowParameters == false) return false; + FollowParameters that = (FollowParameters) o; + return Objects.equals(maxReadRequestOperationCount, that.maxReadRequestOperationCount) && + Objects.equals(maxWriteRequestOperationCount, that.maxWriteRequestOperationCount) && + Objects.equals(maxOutstandingReadRequests, that.maxOutstandingReadRequests) && + Objects.equals(maxOutstandingWriteRequests, that.maxOutstandingWriteRequests) && + Objects.equals(maxReadRequestSize, that.maxReadRequestSize) && + Objects.equals(maxWriteRequestSize, that.maxWriteRequestSize) && + Objects.equals(maxWriteBufferCount, that.maxWriteBufferCount) && + Objects.equals(maxWriteBufferSize, that.maxWriteBufferSize) && + Objects.equals(maxRetryDelay, that.maxRetryDelay) && + Objects.equals(readPollTimeout, that.readPollTimeout); + } + + @Override + public int hashCode() { + return Objects.hash( + maxReadRequestOperationCount, + maxWriteRequestOperationCount, + maxOutstandingReadRequests, + maxOutstandingWriteRequests, + maxReadRequestSize, + maxWriteRequestSize, + maxWriteBufferCount, + maxWriteBufferSize, + maxRetryDelay, + readPollTimeout + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java index bc5a564e4df76..1e199b6fb3cc6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java @@ -5,17 +5,16 @@ */ package org.elasticsearch.xpack.core.ccr.action; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -35,6 +34,7 @@ public class PutAutoFollowPatternAction public static final String NAME = "cluster:admin/xpack/ccr/auto_follow_pattern/put"; public static final PutAutoFollowPatternAction INSTANCE = new PutAutoFollowPatternAction(); + private static final int MAX_NAME_BYTES = 255; private PutAutoFollowPatternAction() { super(NAME); @@ -52,54 +52,27 @@ public RequestBuilder newRequestBuilder(ElasticsearchClient client) { public static class Request extends AcknowledgedRequest implements ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("put_auto_follow_pattern_request", Request::new); - private static final ParseField NAME_FIELD = new ParseField("name"); - private static final int MAX_NAME_BYTES = 255; + // Note that Request should be the Value class here for this parser with a 'parameters' field that maps to + // PutAutoFollowPatternParameters class. But since two minor version are already released with duplicate + // follow parameters in several APIs, PutAutoFollowPatternParameters is now the Value class here. + private static final ObjectParser PARSER = + new ObjectParser<>("put_auto_follow_pattern_request", PutAutoFollowPatternParameters::new); static { - PARSER.declareString(Request::setName, NAME_FIELD); - PARSER.declareString(Request::setRemoteCluster, REMOTE_CLUSTER_FIELD); - PARSER.declareStringArray(Request::setLeaderIndexPatterns, AutoFollowPattern.LEADER_PATTERNS_FIELD); - PARSER.declareString(Request::setFollowIndexNamePattern, AutoFollowPattern.FOLLOW_PATTERN_FIELD); - PARSER.declareInt(Request::setMaxReadRequestOperationCount, AutoFollowPattern.MAX_READ_REQUEST_OPERATION_COUNT); - PARSER.declareField( - Request::setMaxReadRequestSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), AutoFollowPattern.MAX_READ_REQUEST_SIZE.getPreferredName()), - AutoFollowPattern.MAX_READ_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt(Request::setMaxConcurrentReadBatches, AutoFollowPattern.MAX_OUTSTANDING_READ_REQUESTS); - PARSER.declareInt(Request::setMaxWriteRequestOperationCount, AutoFollowPattern.MAX_WRITE_REQUEST_OPERATION_COUNT); - PARSER.declareField( - Request::setMaxWriteRequestSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), AutoFollowPattern.MAX_WRITE_REQUEST_SIZE.getPreferredName()), - AutoFollowPattern.MAX_WRITE_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt(Request::setMaxConcurrentWriteBatches, AutoFollowPattern.MAX_OUTSTANDING_WRITE_REQUESTS); - PARSER.declareInt(Request::setMaxWriteBufferCount, AutoFollowPattern.MAX_WRITE_BUFFER_COUNT); - PARSER.declareField( - Request::setMaxWriteBufferSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), AutoFollowPattern.MAX_WRITE_BUFFER_SIZE.getPreferredName()), - AutoFollowPattern.MAX_WRITE_BUFFER_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareField(Request::setMaxRetryDelay, - (p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName()), - AutoFollowPattern.MAX_RETRY_DELAY, ObjectParser.ValueType.STRING); - PARSER.declareField(Request::setReadPollTimeout, - (p, c) -> TimeValue.parseTimeValue(p.text(), AutoFollowPattern.READ_POLL_TIMEOUT.getPreferredName()), - AutoFollowPattern.READ_POLL_TIMEOUT, ObjectParser.ValueType.STRING); + PARSER.declareString((params, value) -> params.remoteCluster = value, REMOTE_CLUSTER_FIELD); + PARSER.declareStringArray((params, value) -> params.leaderIndexPatterns = value, AutoFollowPattern.LEADER_PATTERNS_FIELD); + PARSER.declareString((params, value) -> params.followIndexNamePattern = value, AutoFollowPattern.FOLLOW_PATTERN_FIELD); + FollowParameters.initParser(PARSER); } public static Request fromXContent(XContentParser parser, String name) throws IOException { - Request request = PARSER.parse(parser, null); - if (name != null) { - if (request.name == null) { - request.name = name; - } else { - if (request.name.equals(name) == false) { - throw new IllegalArgumentException("provided name is not equal"); - } - } - } + PutAutoFollowPatternParameters parameters = PARSER.parse(parser, null); + Request request = new Request(); + request.setName(name); + request.setRemoteCluster(parameters.remoteCluster); + request.setLeaderIndexPatterns(parameters.leaderIndexPatterns); + request.setFollowIndexNamePattern(parameters.followIndexNamePattern); + request.setParameters(parameters); return request; } @@ -107,40 +80,28 @@ public static Request fromXContent(XContentParser parser, String name) throws IO private String remoteCluster; private List leaderIndexPatterns; private String followIndexNamePattern; - - private Integer maxReadRequestOperationCount; - private ByteSizeValue maxReadRequestSize; - private Integer maxConcurrentReadBatches; - private Integer maxWriteRequestOperationCount; - private ByteSizeValue maxWriteRequestSize; - private Integer maxConcurrentWriteBatches; - private Integer maxWriteBufferCount; - private ByteSizeValue maxWriteBufferSize; - private TimeValue maxRetryDelay; - private TimeValue readPollTimeout; + private FollowParameters parameters = new FollowParameters(); public Request() { } @Override public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = null; + ActionRequestValidationException validationException = parameters.validate(); if (name == null) { - validationException = addValidationError("[" + NAME_FIELD.getPreferredName() + "] is missing", validationException); + validationException = addValidationError("[name] is missing", validationException); } if (name != null) { if (name.contains(",")) { - validationException = addValidationError("[" + NAME_FIELD.getPreferredName() + "] name must not contain a ','", - validationException); + validationException = addValidationError("[name] name must not contain a ','", validationException); } if (name.startsWith("_")) { - validationException = addValidationError("[" + NAME_FIELD.getPreferredName() + "] name must not start with '_'", - validationException); + validationException = addValidationError("[name] name must not start with '_'", validationException); } int byteCount = name.getBytes(StandardCharsets.UTF_8).length; if (byteCount > MAX_NAME_BYTES) { - validationException = addValidationError("[" + NAME_FIELD.getPreferredName() + "] name is too long (" + - byteCount + " > " + MAX_NAME_BYTES + ")", validationException); + validationException = addValidationError("[name] name is too long (" + byteCount + " > " + MAX_NAME_BYTES + ")", + validationException); } } if (remoteCluster == null) { @@ -151,19 +112,6 @@ public ActionRequestValidationException validate() { validationException = addValidationError("[" + AutoFollowPattern.LEADER_PATTERNS_FIELD.getPreferredName() + "] is missing", validationException); } - if (maxRetryDelay != null) { - if (maxRetryDelay.millis() <= 0) { - String message = "[" + AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName() + "] must be positive but was [" + - maxRetryDelay.getStringRep() + "]"; - validationException = addValidationError(message, validationException); - } - if (maxRetryDelay.millis() > ResumeFollowAction.MAX_RETRY_DELAY.millis()) { - String message = "[" + AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName() + "] must be less than [" + - ResumeFollowAction.MAX_RETRY_DELAY + - "] but was [" + maxRetryDelay.getStringRep() + "]"; - validationException = addValidationError(message, validationException); - } - } return validationException; } @@ -199,84 +147,12 @@ public void setFollowIndexNamePattern(String followIndexNamePattern) { this.followIndexNamePattern = followIndexNamePattern; } - public Integer getMaxReadRequestOperationCount() { - return maxReadRequestOperationCount; - } - - public void setMaxReadRequestOperationCount(Integer maxReadRequestOperationCount) { - this.maxReadRequestOperationCount = maxReadRequestOperationCount; - } - - public Integer getMaxConcurrentReadBatches() { - return maxConcurrentReadBatches; - } - - public void setMaxConcurrentReadBatches(Integer maxConcurrentReadBatches) { - this.maxConcurrentReadBatches = maxConcurrentReadBatches; - } - - public ByteSizeValue getMaxReadRequestSize() { - return maxReadRequestSize; - } - - public void setMaxReadRequestSize(ByteSizeValue maxReadRequestSize) { - this.maxReadRequestSize = maxReadRequestSize; - } - - public Integer getMaxWriteRequestOperationCount() { - return maxWriteRequestOperationCount; - } - - public void setMaxWriteRequestOperationCount(Integer maxWriteRequestOperationCount) { - this.maxWriteRequestOperationCount = maxWriteRequestOperationCount; - } - - public ByteSizeValue getMaxWriteRequestSize() { - return maxWriteRequestSize; - } - - public void setMaxWriteRequestSize(ByteSizeValue maxWriteRequestSize) { - this.maxWriteRequestSize = maxWriteRequestSize; - } - - public Integer getMaxConcurrentWriteBatches() { - return maxConcurrentWriteBatches; - } - - public void setMaxConcurrentWriteBatches(Integer maxConcurrentWriteBatches) { - this.maxConcurrentWriteBatches = maxConcurrentWriteBatches; - } - - public Integer getMaxWriteBufferCount() { - return maxWriteBufferCount; - } - - public void setMaxWriteBufferCount(Integer maxWriteBufferCount) { - this.maxWriteBufferCount = maxWriteBufferCount; - } - - public ByteSizeValue getMaxWriteBufferSize() { - return maxWriteBufferSize; - } - - public void setMaxWriteBufferSize(ByteSizeValue maxWriteBufferSize) { - this.maxWriteBufferSize = maxWriteBufferSize; - } - - public TimeValue getMaxRetryDelay() { - return maxRetryDelay; - } - - public void setMaxRetryDelay(TimeValue maxRetryDelay) { - this.maxRetryDelay = maxRetryDelay; - } - - public TimeValue getReadPollTimeout() { - return readPollTimeout; + public FollowParameters getParameters() { + return parameters; } - public void setReadPollTimeout(TimeValue readPollTimeout) { - this.readPollTimeout = readPollTimeout; + public void setParameters(FollowParameters parameters) { + this.parameters = parameters; } public Request(StreamInput in) throws IOException { @@ -285,16 +161,21 @@ public Request(StreamInput in) throws IOException { remoteCluster = in.readString(); leaderIndexPatterns = in.readStringList(); followIndexNamePattern = in.readOptionalString(); - maxReadRequestOperationCount = in.readOptionalVInt(); - maxReadRequestSize = in.readOptionalWriteable(ByteSizeValue::new); - maxConcurrentReadBatches = in.readOptionalVInt(); - maxWriteRequestOperationCount = in.readOptionalVInt(); - maxWriteRequestSize = in.readOptionalWriteable(ByteSizeValue::new); - maxConcurrentWriteBatches = in.readOptionalVInt(); - maxWriteBufferCount = in.readOptionalVInt(); - maxWriteBufferSize = in.readOptionalWriteable(ByteSizeValue::new); - maxRetryDelay = in.readOptionalTimeValue(); - readPollTimeout = in.readOptionalTimeValue(); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + parameters = new FollowParameters(in); + } else { + parameters = new FollowParameters(); + parameters.maxReadRequestOperationCount = in.readOptionalVInt(); + parameters.maxReadRequestSize = in.readOptionalWriteable(ByteSizeValue::new); + parameters.maxOutstandingReadRequests = in.readOptionalVInt(); + parameters.maxWriteRequestOperationCount = in.readOptionalVInt(); + parameters.maxWriteRequestSize = in.readOptionalWriteable(ByteSizeValue::new); + parameters.maxOutstandingWriteRequests = in.readOptionalVInt(); + parameters.maxWriteBufferCount = in.readOptionalVInt(); + parameters.maxWriteBufferSize = in.readOptionalWriteable(ByteSizeValue::new); + parameters.maxRetryDelay = in.readOptionalTimeValue(); + parameters.readPollTimeout = in.readOptionalTimeValue(); + } } @Override @@ -304,58 +185,32 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(remoteCluster); out.writeStringCollection(leaderIndexPatterns); out.writeOptionalString(followIndexNamePattern); - out.writeOptionalVInt(maxReadRequestOperationCount); - out.writeOptionalWriteable(maxReadRequestSize); - out.writeOptionalVInt(maxConcurrentReadBatches); - out.writeOptionalVInt(maxWriteRequestOperationCount); - out.writeOptionalWriteable(maxWriteRequestSize); - out.writeOptionalVInt(maxConcurrentWriteBatches); - out.writeOptionalVInt(maxWriteBufferCount); - out.writeOptionalWriteable(maxWriteBufferSize); - out.writeOptionalTimeValue(maxRetryDelay); - out.writeOptionalTimeValue(readPollTimeout); + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + parameters.writeTo(out); + } else { + out.writeOptionalVInt(parameters.maxReadRequestOperationCount); + out.writeOptionalWriteable(parameters.maxReadRequestSize); + out.writeOptionalVInt(parameters.maxOutstandingReadRequests); + out.writeOptionalVInt(parameters.maxWriteRequestOperationCount); + out.writeOptionalWriteable(parameters.maxWriteRequestSize); + out.writeOptionalVInt(parameters.maxOutstandingWriteRequests); + out.writeOptionalVInt(parameters.maxWriteBufferCount); + out.writeOptionalWriteable(parameters.maxWriteBufferSize); + out.writeOptionalTimeValue(parameters.maxRetryDelay); + out.writeOptionalTimeValue(parameters.readPollTimeout); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); { - builder.field(NAME_FIELD.getPreferredName(), name); builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(AutoFollowPattern.LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns); if (followIndexNamePattern != null) { builder.field(AutoFollowPattern.FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern); } - if (maxReadRequestOperationCount != null) { - builder.field(AutoFollowPattern.MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName(), maxReadRequestOperationCount); - } - if (maxReadRequestSize != null) { - builder.field(AutoFollowPattern.MAX_READ_REQUEST_SIZE.getPreferredName(), maxReadRequestSize.getStringRep()); - } - if (maxWriteRequestOperationCount != null) { - builder.field(AutoFollowPattern.MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName(), maxWriteRequestOperationCount); - } - if (maxWriteRequestSize != null) { - builder.field(AutoFollowPattern.MAX_WRITE_REQUEST_SIZE.getPreferredName(), maxWriteRequestSize.getStringRep()); - } - if (maxWriteBufferCount != null) { - builder.field(AutoFollowPattern.MAX_WRITE_BUFFER_COUNT.getPreferredName(), maxWriteBufferCount); - } - if (maxWriteBufferSize != null) { - builder.field(AutoFollowPattern.MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize.getStringRep()); - } - if (maxConcurrentReadBatches != null) { - builder.field(AutoFollowPattern.MAX_OUTSTANDING_READ_REQUESTS.getPreferredName(), maxConcurrentReadBatches); - } - if (maxConcurrentWriteBatches != null) { - builder.field(AutoFollowPattern.MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName(), maxConcurrentWriteBatches); - } - if (maxRetryDelay != null) { - builder.field(AutoFollowPattern.MAX_RETRY_DELAY.getPreferredName(), maxRetryDelay.getStringRep()); - } - if (readPollTimeout != null) { - builder.field(AutoFollowPattern.READ_POLL_TIMEOUT.getPreferredName(), readPollTimeout.getStringRep()); - } + parameters.toXContentFragment(builder); } builder.endObject(); return builder; @@ -367,39 +222,25 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; return Objects.equals(name, request.name) && - Objects.equals(remoteCluster, request.remoteCluster) && - Objects.equals(leaderIndexPatterns, request.leaderIndexPatterns) && - Objects.equals(followIndexNamePattern, request.followIndexNamePattern) && - Objects.equals(maxReadRequestOperationCount, request.maxReadRequestOperationCount) && - Objects.equals(maxReadRequestSize, request.maxReadRequestSize) && - Objects.equals(maxConcurrentReadBatches, request.maxConcurrentReadBatches) && - Objects.equals(maxWriteRequestOperationCount, request.maxWriteRequestOperationCount) && - Objects.equals(maxWriteRequestSize, request.maxWriteRequestSize) && - Objects.equals(maxConcurrentWriteBatches, request.maxConcurrentWriteBatches) && - Objects.equals(maxWriteBufferCount, request.maxWriteBufferCount) && - Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) && - Objects.equals(maxRetryDelay, request.maxRetryDelay) && - Objects.equals(readPollTimeout, request.readPollTimeout); + Objects.equals(remoteCluster, request.remoteCluster) && + Objects.equals(leaderIndexPatterns, request.leaderIndexPatterns) && + Objects.equals(followIndexNamePattern, request.followIndexNamePattern) && + Objects.equals(parameters, request.parameters); } @Override public int hashCode() { - return Objects.hash( - name, - remoteCluster, - leaderIndexPatterns, - followIndexNamePattern, - maxReadRequestOperationCount, - maxReadRequestSize, - maxConcurrentReadBatches, - maxWriteRequestOperationCount, - maxWriteRequestSize, - maxConcurrentWriteBatches, - maxWriteBufferCount, - maxWriteBufferSize, - maxRetryDelay, - readPollTimeout); + return Objects.hash(name, remoteCluster, leaderIndexPatterns, followIndexNamePattern, parameters); } + + // This class only exists for reuse of the FollowParameters class, see comment above the parser field. + private static class PutAutoFollowPatternParameters extends FollowParameters { + + private String remoteCluster; + private List leaderIndexPatterns; + private String followIndexNamePattern; + } + } public static class RequestBuilder extends ActionRequestBuilder { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java index b5c76debac148..3bd6526889d1c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java @@ -20,8 +20,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -31,17 +29,6 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.FOLLOWER_INDEX_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_READ_REQUESTS; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_OUTSTANDING_WRITE_REQUESTS; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_READ_REQUEST_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_RETRY_DELAY_FIELD; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_BUFFER_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_REQUEST_OPERATION_COUNT; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.MAX_WRITE_REQUEST_SIZE; -import static org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction.Request.READ_POLL_TIMEOUT; public final class PutFollowAction extends Action< PutFollowAction.Request, @@ -70,72 +57,47 @@ public static class Request extends AcknowledgedRequest implements Indi private static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster"); private static final ParseField LEADER_INDEX_FIELD = new ParseField("leader_index"); - private static final ObjectParser PARSER = new ObjectParser<>(NAME, () -> { - Request request = new Request(); - request.setFollowRequest(new ResumeFollowAction.Request()); - return request; - }); + // Note that Request should be the Value class here for this parser with a 'parameters' field that maps to + // PutFollowParameters class. But since two minor version are already released with duplicate follow parameters + // in several APIs, PutFollowParameters is now the Value class here. + private static final ObjectParser PARSER = new ObjectParser<>(NAME, PutFollowParameters::new); static { - PARSER.declareString(Request::setRemoteCluster, REMOTE_CLUSTER_FIELD); - PARSER.declareString(Request::setLeaderIndex, LEADER_INDEX_FIELD); - PARSER.declareString((req, val) -> req.followRequest.setFollowerIndex(val), FOLLOWER_INDEX_FIELD); - PARSER.declareInt((req, val) -> req.followRequest.setMaxReadRequestOperationCount(val), MAX_READ_REQUEST_OPERATION_COUNT); - PARSER.declareField( - (req, val) -> req.followRequest.setMaxReadRequestSize(val), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_READ_REQUEST_SIZE.getPreferredName()), - MAX_READ_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt((req, val) -> req.followRequest.setMaxOutstandingReadRequests(val), MAX_OUTSTANDING_READ_REQUESTS); - PARSER.declareInt((req, val) -> req.followRequest.setMaxWriteRequestOperationCount(val), MAX_WRITE_REQUEST_OPERATION_COUNT); - PARSER.declareField( - (req, val) -> req.followRequest.setMaxWriteRequestSize(val), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_REQUEST_SIZE.getPreferredName()), - MAX_WRITE_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt((req, val) -> req.followRequest.setMaxOutstandingWriteRequests(val), MAX_OUTSTANDING_WRITE_REQUESTS); - PARSER.declareInt((req, val) -> req.followRequest.setMaxWriteBufferCount(val), MAX_WRITE_BUFFER_COUNT); - PARSER.declareField( - (req, val) -> req.followRequest.setMaxWriteBufferSize(val), - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_BUFFER_SIZE.getPreferredName()), - MAX_WRITE_BUFFER_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareField( - (req, val) -> req.followRequest.setMaxRetryDelay(val), - (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY_FIELD.getPreferredName()), - MAX_RETRY_DELAY_FIELD, - ObjectParser.ValueType.STRING); - PARSER.declareField( - (req, val) -> req.followRequest.setReadPollTimeout(val), - (p, c) -> TimeValue.parseTimeValue(p.text(), READ_POLL_TIMEOUT.getPreferredName()), - READ_POLL_TIMEOUT, - ObjectParser.ValueType.STRING); + PARSER.declareString((putFollowParameters, value) -> putFollowParameters.remoteCluster = value, REMOTE_CLUSTER_FIELD); + PARSER.declareString((putFollowParameters, value) -> putFollowParameters.leaderIndex = value, LEADER_INDEX_FIELD); + FollowParameters.initParser(PARSER); } public static Request fromXContent(final XContentParser parser, final String followerIndex, ActiveShardCount waitForActiveShards) throws IOException { - Request request = PARSER.parse(parser, followerIndex); - if (followerIndex != null) { - if (request.getFollowRequest().getFollowerIndex() == null) { - request.getFollowRequest().setFollowerIndex(followerIndex); - } else { - if (request.getFollowRequest().getFollowerIndex().equals(followerIndex) == false) { - throw new IllegalArgumentException("provided follower_index is not equal"); - } - } - } + PutFollowParameters parameters = PARSER.parse(parser, null); + + Request request = new Request(); request.waitForActiveShards(waitForActiveShards); + request.setFollowerIndex(followerIndex); + request.setRemoteCluster(parameters.remoteCluster); + request.setLeaderIndex(parameters.leaderIndex); + request.setParameters(parameters); return request; } private String remoteCluster; private String leaderIndex; + private String followerIndex; + private FollowParameters parameters = new FollowParameters(); private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE; - private ResumeFollowAction.Request followRequest; public Request() { } + public String getFollowerIndex() { + return followerIndex; + } + + public void setFollowerIndex(String followerIndex) { + this.followerIndex = followerIndex; + } + public String getRemoteCluster() { return remoteCluster; } @@ -152,6 +114,14 @@ public void setLeaderIndex(String leaderIndex) { this.leaderIndex = leaderIndex; } + public FollowParameters getParameters() { + return parameters; + } + + public void setParameters(FollowParameters parameters) { + this.parameters = parameters; + } + public ActiveShardCount waitForActiveShards() { return waitForActiveShards; } @@ -173,29 +143,24 @@ public void waitForActiveShards(ActiveShardCount waitForActiveShards) { } } - public ResumeFollowAction.Request getFollowRequest() { - return followRequest; - } - - public void setFollowRequest(ResumeFollowAction.Request followRequest) { - this.followRequest = followRequest; - } - @Override public ActionRequestValidationException validate() { - ActionRequestValidationException e = followRequest.validate(); + ActionRequestValidationException e = parameters.validate(); if (remoteCluster == null) { e = addValidationError(REMOTE_CLUSTER_FIELD.getPreferredName() + " is missing", e); } if (leaderIndex == null) { e = addValidationError(LEADER_INDEX_FIELD.getPreferredName() + " is missing", e); } + if (followerIndex == null) { + e = addValidationError("follower_index is missing", e); + } return e; } @Override public String[] indices() { - return new String[]{followRequest.getFollowerIndex()}; + return new String[]{followerIndex}; } @Override @@ -205,12 +170,13 @@ public IndicesOptions indicesOptions() { public Request(StreamInput in) throws IOException { super(in); - remoteCluster = in.readString(); - leaderIndex = in.readString(); + this.remoteCluster = in.readString(); + this.leaderIndex = in.readString(); + this.followerIndex = in.readString(); + this.parameters = new FollowParameters(in); if (in.getVersion().onOrAfter(Version.V_6_7_0)) { waitForActiveShards(ActiveShardCount.readFrom(in)); } - followRequest = new ResumeFollowAction.Request(in); } @Override @@ -218,10 +184,11 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(remoteCluster); out.writeString(leaderIndex); + out.writeString(followerIndex); + parameters.writeTo(out); if (out.getVersion().onOrAfter(Version.V_6_7_0)) { waitForActiveShards.writeTo(out); } - followRequest.writeTo(out); } @Override @@ -230,7 +197,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex); - followRequest.toXContentFragment(builder, params); + parameters.toXContentFragment(builder); } builder.endObject(); return builder; @@ -243,24 +210,23 @@ public boolean equals(Object o) { Request request = (Request) o; return Objects.equals(remoteCluster, request.remoteCluster) && Objects.equals(leaderIndex, request.leaderIndex) && - Objects.equals(waitForActiveShards, request.waitForActiveShards) && - Objects.equals(followRequest, request.followRequest); + Objects.equals(followerIndex, request.followerIndex) && + Objects.equals(parameters, request.parameters) && + Objects.equals(waitForActiveShards, request.waitForActiveShards); } @Override public int hashCode() { - return Objects.hash(remoteCluster, leaderIndex, waitForActiveShards, followRequest); + return Objects.hash(remoteCluster, leaderIndex, followerIndex, parameters, waitForActiveShards); } - @Override - public String toString() { - return "PutFollowAction.Request{" + - "remoteCluster='" + remoteCluster + '\'' + - ", leaderIndex='" + leaderIndex + '\'' + - ", waitForActiveShards=" + waitForActiveShards + - ", followRequest=" + followRequest + - '}'; + // This class only exists for reuse of the FollowParameters class, see comment above the parser field. + private static class PutFollowParameters extends FollowParameters { + + private String remoteCluster; + private String leaderIndex; } + } public static class Response extends ActionResponse implements ToXContentObject { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java index 7439d521e21f8..f195379c96fc6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java @@ -12,11 +12,8 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -35,8 +32,6 @@ public final class ResumeFollowAction extends Action< public static final ResumeFollowAction INSTANCE = new ResumeFollowAction(); public static final String NAME = "cluster:admin/xpack/ccr/resume_follow"; - public static final TimeValue MAX_RETRY_DELAY = TimeValue.timeValueMinutes(5); - private ResumeFollowAction() { super(NAME); } @@ -48,65 +43,28 @@ public AcknowledgedResponse newResponse() { public static class Request extends MasterNodeRequest implements ToXContentObject { - static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index"); - static final ParseField MAX_READ_REQUEST_OPERATION_COUNT = new ParseField("max_read_request_operation_count"); - static final ParseField MAX_READ_REQUEST_SIZE = new ParseField("max_read_request_size"); - static final ParseField MAX_OUTSTANDING_READ_REQUESTS = new ParseField("max_outstanding_read_requests"); - static final ParseField MAX_WRITE_REQUEST_OPERATION_COUNT = new ParseField("max_write_request_operation_count"); - static final ParseField MAX_WRITE_REQUEST_SIZE = new ParseField("max_write_request_size"); - static final ParseField MAX_OUTSTANDING_WRITE_REQUESTS = new ParseField("max_outstanding_write_requests"); - static final ParseField MAX_WRITE_BUFFER_COUNT = new ParseField("max_write_buffer_count"); - static final ParseField MAX_WRITE_BUFFER_SIZE = new ParseField("max_write_buffer_size"); - static final ParseField MAX_RETRY_DELAY_FIELD = new ParseField("max_retry_delay"); - static final ParseField READ_POLL_TIMEOUT = new ParseField("read_poll_timeout"); - static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + // Note that Request should be the Value class here for this parser with a 'parameters' field that maps to FollowParameters class + // But since two minor version are already released with duplicate follow parameters in several APIs, FollowParameters + // is now the Value class here. + static final ObjectParser PARSER = new ObjectParser<>(NAME, FollowParameters::new); static { - PARSER.declareString(Request::setFollowerIndex, FOLLOWER_INDEX_FIELD); - PARSER.declareInt(Request::setMaxReadRequestOperationCount, MAX_READ_REQUEST_OPERATION_COUNT); - PARSER.declareField( - Request::setMaxReadRequestSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_READ_REQUEST_SIZE.getPreferredName()), MAX_READ_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt(Request::setMaxOutstandingReadRequests, MAX_OUTSTANDING_READ_REQUESTS); - PARSER.declareInt(Request::setMaxWriteRequestOperationCount, MAX_WRITE_REQUEST_OPERATION_COUNT); - PARSER.declareField(Request::setMaxWriteRequestSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_REQUEST_SIZE.getPreferredName()), MAX_WRITE_REQUEST_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareInt(Request::setMaxOutstandingWriteRequests, MAX_OUTSTANDING_WRITE_REQUESTS); - PARSER.declareInt(Request::setMaxWriteBufferCount, MAX_WRITE_BUFFER_COUNT); - PARSER.declareField( - Request::setMaxWriteBufferSize, - (p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_WRITE_BUFFER_SIZE.getPreferredName()), - MAX_WRITE_BUFFER_SIZE, - ObjectParser.ValueType.STRING); - PARSER.declareField( - Request::setMaxRetryDelay, - (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_RETRY_DELAY_FIELD.getPreferredName()), - MAX_RETRY_DELAY_FIELD, - ObjectParser.ValueType.STRING); - PARSER.declareField( - Request::setReadPollTimeout, - (p, c) -> TimeValue.parseTimeValue(p.text(), READ_POLL_TIMEOUT.getPreferredName()), - READ_POLL_TIMEOUT, - ObjectParser.ValueType.STRING); + FollowParameters.initParser(PARSER); } public static Request fromXContent(final XContentParser parser, final String followerIndex) throws IOException { - Request request = PARSER.parse(parser, followerIndex); - if (followerIndex != null) { - if (request.followerIndex == null) { - request.followerIndex = followerIndex; - } else { - if (request.followerIndex.equals(followerIndex) == false) { - throw new IllegalArgumentException("provided follower_index is not equal"); - } - } - } + FollowParameters parameters = PARSER.parse(parser, null); + Request request = new Request(); + request.setFollowerIndex(followerIndex); + request.setParameters(parameters); return request; } private String followerIndex; + private FollowParameters parameters = new FollowParameters(); + + public Request() { + } public String getFollowerIndex() { return followerIndex; @@ -116,261 +74,58 @@ public void setFollowerIndex(String followerIndex) { this.followerIndex = followerIndex; } - private Integer maxReadRequestOperationCount; - - public Integer getMaxReadRequestOperationCount() { - return maxReadRequestOperationCount; - } - - public void setMaxReadRequestOperationCount(Integer maxReadRequestOperationCount) { - this.maxReadRequestOperationCount = maxReadRequestOperationCount; - } - - private Integer maxOutstandingReadRequests; - - public Integer getMaxOutstandingReadRequests() { - return maxOutstandingReadRequests; - } - - public void setMaxOutstandingReadRequests(Integer maxOutstandingReadRequests) { - this.maxOutstandingReadRequests = maxOutstandingReadRequests; - } - - private ByteSizeValue maxReadRequestSize; - - public ByteSizeValue getMaxReadRequestSize() { - return maxReadRequestSize; - } - - public void setMaxReadRequestSize(ByteSizeValue maxReadRequestSize) { - this.maxReadRequestSize = maxReadRequestSize; - } - - private Integer maxWriteRequestOperationCount; - - public Integer getMaxWriteRequestOperationCount() { - return maxWriteRequestOperationCount; - } - - public void setMaxWriteRequestOperationCount(Integer maxWriteRequestOperationCount) { - this.maxWriteRequestOperationCount = maxWriteRequestOperationCount; - } - - private ByteSizeValue maxWriteRequestSize; - - public ByteSizeValue getMaxWriteRequestSize() { - return maxWriteRequestSize; - } - - public void setMaxWriteRequestSize(ByteSizeValue maxWriteRequestSize) { - this.maxWriteRequestSize = maxWriteRequestSize; - } - - private Integer maxOutstandingWriteRequests; - - public Integer getMaxOutstandingWriteRequests() { - return maxOutstandingWriteRequests; - } - - public void setMaxOutstandingWriteRequests(Integer maxOutstandingWriteRequests) { - this.maxOutstandingWriteRequests = maxOutstandingWriteRequests; - } - - private Integer maxWriteBufferCount; - - public Integer getMaxWriteBufferCount() { - return maxWriteBufferCount; - } - - public void setMaxWriteBufferCount(Integer maxWriteBufferCount) { - this.maxWriteBufferCount = maxWriteBufferCount; - } - - private ByteSizeValue maxWriteBufferSize; - - public ByteSizeValue getMaxWriteBufferSize() { - return maxWriteBufferSize; - } - - public void setMaxWriteBufferSize(ByteSizeValue maxWriteBufferSize) { - this.maxWriteBufferSize = maxWriteBufferSize; - } - - private TimeValue maxRetryDelay; - - public void setMaxRetryDelay(TimeValue maxRetryDelay) { - this.maxRetryDelay = maxRetryDelay; - } - - public TimeValue getMaxRetryDelay() { - return maxRetryDelay; - } - - private TimeValue readPollTimeout; - - public TimeValue getReadPollTimeout() { - return readPollTimeout; + public FollowParameters getParameters() { + return parameters; } - public void setReadPollTimeout(TimeValue readPollTimeout) { - this.readPollTimeout = readPollTimeout; - } - - public Request() { + public void setParameters(FollowParameters parameters) { + this.parameters = parameters; } @Override public ActionRequestValidationException validate() { - ActionRequestValidationException e = null; - + ActionRequestValidationException e = parameters.validate(); if (followerIndex == null) { - e = addValidationError(FOLLOWER_INDEX_FIELD.getPreferredName() + " is missing", e); - } - if (maxReadRequestOperationCount != null && maxReadRequestOperationCount < 1) { - e = addValidationError(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e); - } - if (maxReadRequestSize != null && maxReadRequestSize.compareTo(ByteSizeValue.ZERO) <= 0) { - e = addValidationError(MAX_READ_REQUEST_SIZE.getPreferredName() + " must be larger than 0", e); - } - if (maxOutstandingReadRequests != null && maxOutstandingReadRequests < 1) { - e = addValidationError(MAX_OUTSTANDING_READ_REQUESTS.getPreferredName() + " must be larger than 0", e); - } - if (maxWriteRequestOperationCount != null && maxWriteRequestOperationCount < 1) { - e = addValidationError(MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName() + " must be larger than 0", e); + e = addValidationError("follower_index is missing", e); } - if (maxWriteRequestSize != null && maxWriteRequestSize.compareTo(ByteSizeValue.ZERO) <= 0) { - e = addValidationError(MAX_WRITE_REQUEST_SIZE.getPreferredName() + " must be larger than 0", e); - } - if (maxOutstandingWriteRequests != null && maxOutstandingWriteRequests < 1) { - e = addValidationError(MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName() + " must be larger than 0", e); - } - if (maxWriteBufferCount != null && maxWriteBufferCount < 1) { - e = addValidationError(MAX_WRITE_BUFFER_COUNT.getPreferredName() + " must be larger than 0", e); - } - if (maxWriteBufferSize != null && maxWriteBufferSize.compareTo(ByteSizeValue.ZERO) <= 0) { - e = addValidationError(MAX_WRITE_BUFFER_SIZE.getPreferredName() + " must be larger than 0", e); - } - if (maxRetryDelay != null && maxRetryDelay.millis() <= 0) { - String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be positive but was [" + - maxRetryDelay.getStringRep() + "]"; - e = addValidationError(message, e); - } - if (maxRetryDelay != null && maxRetryDelay.millis() > ResumeFollowAction.MAX_RETRY_DELAY.millis()) { - String message = "[" + MAX_RETRY_DELAY_FIELD.getPreferredName() + "] must be less than [" + MAX_RETRY_DELAY + - "] but was [" + maxRetryDelay.getStringRep() + "]"; - e = addValidationError(message, e); - } - return e; } public Request(StreamInput in) throws IOException { super(in); followerIndex = in.readString(); - maxReadRequestOperationCount = in.readOptionalVInt(); - maxOutstandingReadRequests = in.readOptionalVInt(); - maxReadRequestSize = in.readOptionalWriteable(ByteSizeValue::new); - maxWriteRequestOperationCount = in.readOptionalVInt(); - maxWriteRequestSize = in.readOptionalWriteable(ByteSizeValue::new); - maxOutstandingWriteRequests = in.readOptionalVInt(); - maxWriteBufferCount = in.readOptionalVInt(); - maxWriteBufferSize = in.readOptionalWriteable(ByteSizeValue::new); - maxRetryDelay = in.readOptionalTimeValue(); - readPollTimeout = in.readOptionalTimeValue(); + parameters = new FollowParameters(in); } @Override public void writeTo(final StreamOutput out) throws IOException { super.writeTo(out); out.writeString(followerIndex); - out.writeOptionalVInt(maxReadRequestOperationCount); - out.writeOptionalVInt(maxOutstandingReadRequests); - out.writeOptionalWriteable(maxReadRequestSize); - out.writeOptionalVInt(maxWriteRequestOperationCount); - out.writeOptionalWriteable(maxWriteRequestSize); - out.writeOptionalVInt(maxOutstandingWriteRequests); - out.writeOptionalVInt(maxWriteBufferCount); - out.writeOptionalWriteable(maxWriteBufferSize); - out.writeOptionalTimeValue(maxRetryDelay); - out.writeOptionalTimeValue(readPollTimeout); + parameters.writeTo(out); } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); { - toXContentFragment(builder, params); + parameters.toXContentFragment(builder); } builder.endObject(); return builder; } - void toXContentFragment(final XContentBuilder builder, final Params params) throws IOException { - builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex); - if (maxReadRequestOperationCount != null) { - builder.field(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName(), maxReadRequestOperationCount); - } - if (maxReadRequestSize != null) { - builder.field(MAX_READ_REQUEST_SIZE.getPreferredName(), maxReadRequestSize.getStringRep()); - } - if (maxWriteRequestOperationCount != null) { - builder.field(MAX_WRITE_REQUEST_OPERATION_COUNT.getPreferredName(), maxWriteRequestOperationCount); - } - if (maxWriteRequestSize != null) { - builder.field(MAX_WRITE_REQUEST_SIZE.getPreferredName(), maxWriteRequestSize.getStringRep()); - } - if (maxWriteBufferCount != null) { - builder.field(MAX_WRITE_BUFFER_COUNT.getPreferredName(), maxWriteBufferCount); - } - if (maxWriteBufferSize != null) { - builder.field(MAX_WRITE_BUFFER_SIZE.getPreferredName(), maxWriteBufferSize.getStringRep()); - } - if (maxOutstandingReadRequests != null) { - builder.field(MAX_OUTSTANDING_READ_REQUESTS.getPreferredName(), maxOutstandingReadRequests); - } - if (maxOutstandingWriteRequests != null) { - builder.field(MAX_OUTSTANDING_WRITE_REQUESTS.getPreferredName(), maxOutstandingWriteRequests); - } - if (maxRetryDelay != null) { - builder.field(MAX_RETRY_DELAY_FIELD.getPreferredName(), maxRetryDelay.getStringRep()); - } - if (readPollTimeout != null) { - builder.field(READ_POLL_TIMEOUT.getPreferredName(), readPollTimeout.getStringRep()); - } - } - @Override - public boolean equals(final Object o) { + public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(maxReadRequestOperationCount, request.maxReadRequestOperationCount) && - Objects.equals(maxReadRequestSize, request.maxReadRequestSize) && - Objects.equals(maxOutstandingReadRequests, request.maxOutstandingReadRequests) && - Objects.equals(maxWriteRequestOperationCount, request.maxWriteRequestOperationCount) && - Objects.equals(maxWriteRequestSize, request.maxWriteRequestSize) && - Objects.equals(maxOutstandingWriteRequests, request.maxOutstandingWriteRequests) && - Objects.equals(maxWriteBufferCount, request.maxWriteBufferCount) && - Objects.equals(maxWriteBufferSize, request.maxWriteBufferSize) && - Objects.equals(maxRetryDelay, request.maxRetryDelay) && - Objects.equals(readPollTimeout, request.readPollTimeout) && - Objects.equals(followerIndex, request.followerIndex); + return Objects.equals(followerIndex, request.followerIndex) && + Objects.equals(parameters, request.parameters); } @Override public int hashCode() { - return Objects.hash( - followerIndex, - maxReadRequestOperationCount, - maxReadRequestSize, - maxOutstandingReadRequests, - maxWriteRequestOperationCount, - maxWriteRequestSize, - maxOutstandingWriteRequests, - maxWriteBufferCount, - maxWriteBufferSize, - maxRetryDelay, - readPollTimeout); + return Objects.hash(followerIndex, parameters); } } From 46cca7fe558d929fbf917b0f85b121f0aef2e462 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Tue, 5 Feb 2019 23:12:52 +0200 Subject: [PATCH 17/25] SQL: Fix esType for DATETIME/DATE and INTERVALS (#38179) Since introduction of data types that don't have a corresponding type in ES the `esType` is error-prone when used for `unmappedType()` calls. Moreover since the renaming of `DATE` to `DATETIME` and the introduction of an actual date-only `DATE` the `esType` would return `datetime` which is not a valid type for ES mapping. Fixes: #38051 --- .../xpack/sql/analysis/analyzer/Analyzer.java | 4 +- .../xpack/sql/analysis/analyzer/Verifier.java | 4 +- .../sql/execution/search/SourceGenerator.java | 2 +- .../search/extractor/FieldHitExtractor.java | 2 +- .../xpack/sql/expression/Expressions.java | 2 +- .../function/aggregate/TopHits.java | 4 +- .../predicate/operator/arithmetic/Sub.java | 4 +- .../sql/plan/logical/command/ShowColumns.java | 2 +- .../plan/logical/command/sys/SysColumns.java | 5 +- .../plan/logical/command/sys/SysTypes.java | 7 +-- .../sql/plugin/TransportSqlQueryAction.java | 4 +- .../sql/querydsl/container/TopHitsAggRef.java | 2 +- .../xpack/sql/type/DataType.java | 59 +++++++++++-------- .../elasticsearch/xpack/sql/type/Schema.java | 4 +- .../analysis/index/IndexResolverTests.java | 4 +- .../xpack/sql/expression/ParameterTests.java | 16 ++--- .../sql/parser/EscapedFunctionsTests.java | 6 +- .../sql/parser/LikeEscapingParsingTests.java | 5 +- .../logical/command/sys/SysTablesTests.java | 2 +- .../sql/planner/QueryTranslatorTests.java | 20 +++---- .../sql/type/DataTypeConversionTests.java | 2 +- .../xpack/sql/type/TypesTests.java | 6 +- 22 files changed, 86 insertions(+), 80 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index ccc8a1e798a67..0208082c56889 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -219,7 +219,7 @@ else if (DataTypes.isUnsupported(fa.dataType())) { // compound fields else if (allowCompound == false && fa.dataType().isPrimitive() == false) { named = u.withUnresolvedMessage( - "Cannot use field [" + fa.name() + "] type [" + fa.dataType().esType + "] only its subfields"); + "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName + "] only its subfields"); } } return named; @@ -1228,4 +1228,4 @@ protected boolean skipResolved() { return true; } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index ed9bd1f106830..ac59b08dbb726 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -682,7 +682,7 @@ private static void validateInExpression(LogicalPlan p, Set localFailur for (Expression value : in.list()) { if (areTypesCompatible(dt, value.dataType()) == false) { localFailures.add(fail(value, "expected data type [{}], value provided is of type [{}]", - dt.esType, value.dataType().esType)); + dt.typeName, value.dataType().typeName)); return; } } @@ -703,7 +703,7 @@ private static void validateConditional(LogicalPlan p, Set localFailure } else { if (areTypesCompatible(dt, child.dataType()) == false) { localFailures.add(fail(child, "expected data type [{}], value provided is of type [{}]", - dt.esType, child.dataType().esType)); + dt.typeName, child.dataType().typeName)); return; } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 08bb737e65cc4..c22b1213d09dc 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -175,4 +175,4 @@ private static void disableSource(SearchSourceBuilder builder) { builder.storedFields(NO_STORED_FIELD); } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index 503da62dc30ee..589481247ac39 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -87,7 +87,7 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(fieldName); - out.writeOptionalString(dataType == null ? null : dataType.esType); + out.writeOptionalString(dataType == null ? null : dataType.typeName); out.writeBoolean(useDocValue); out.writeOptionalString(hitName); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index 4959e73c15ae5..04d660642c8b2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -191,7 +191,7 @@ public static TypeResolution typeMustBe(Expression e, paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT), acceptedTypesForErrorMsg(acceptedTypes), Expressions.name(e), - e.dataType().esType)); + e.dataType().typeName)); } private static String acceptedTypesForErrorMsg(String... acceptedTypes) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/TopHits.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/TopHits.java index 2e1901ccdb43a..5ec96ea41c87b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/TopHits.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/TopHits.java @@ -46,7 +46,7 @@ protected TypeResolution resolveType() { ((FieldAttribute) field()).exactAttribute(); } catch (MappingException ex) { return new TypeResolution(format(null, "[{}] cannot operate on first argument field of data type [{}]", - functionName(), field().dataType().esType)); + functionName(), field().dataType().typeName)); } if (orderField() != null) { @@ -59,7 +59,7 @@ protected TypeResolution resolveType() { ((FieldAttribute) orderField()).exactAttribute(); } catch (MappingException ex) { return new TypeResolution(format(null, "[{}] cannot operate on second argument field of data type [{}]", - functionName(), orderField().dataType().esType)); + functionName(), orderField().dataType().typeName)); } } return TypeResolution.TYPE_RESOLVED; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java index e2454ffd26742..cad2d7ffa625a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java @@ -7,8 +7,8 @@ import org.elasticsearch.xpack.sql.expression.Expression; import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; -import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataTypes; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; @@ -36,7 +36,7 @@ protected Sub replaceChildren(Expression newLeft, Expression newRight) { protected TypeResolution resolveWithIntervals() { if (right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) { return new TypeResolution(format(null, "Cannot subtract a {}[{}] from an interval[{}]; do you mean the reverse?", - right().dataType().esType, right().source().text(), left().source().text())); + right().dataType().typeName, right().source().text(), left().source().text())); } return TypeResolution.TYPE_RESOLVED; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index f68bbeeffb63c..dfcf75565ef11 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -81,7 +81,7 @@ private void fillInRows(Map mapping, String prefix, List liste List> rows = values // sort by SQL int type (that's what the JDBC/ODBC specs want) followed by name .sorted(Comparator.comparing((DataType t) -> t.sqlType.getVendorTypeNumber()).thenComparing(DataType::sqlName)) - .map(t -> asList(t.esType.toUpperCase(Locale.ROOT), + .map(t -> asList(t.toString(), t.sqlType.getVendorTypeNumber(), //https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size?view=sql-server-2017 t.defaultPrecision, @@ -132,4 +131,4 @@ public boolean equals(Object obj) { return type.equals(((SysTypes) obj).type); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index 1bef191c737b9..862498d67ed2b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -88,9 +88,9 @@ static SqlQueryResponse createResponse(SqlQueryRequest request, SchemaRowSet row List columns = new ArrayList<>(rowSet.columnCount()); for (Schema.Entry entry : rowSet.schema()) { if (Mode.isDriver(request.mode())) { - columns.add(new ColumnInfo("", entry.name(), entry.type().esType, entry.type().displaySize)); + columns.add(new ColumnInfo("", entry.name(), entry.type().typeName, entry.type().displaySize)); } else { - columns.add(new ColumnInfo("", entry.name(), entry.type().esType)); + columns.add(new ColumnInfo("", entry.name(), entry.type().typeName)); } } columns = unmodifiableList(columns); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TopHitsAggRef.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TopHitsAggRef.java index 8e5eabec9b861..740c655c5069b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TopHitsAggRef.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/TopHitsAggRef.java @@ -33,6 +33,6 @@ public DataType fieldDataType() { @Override public String toString() { - return ">" + name + "[" + fieldDataType.esType + "]"; + return ">" + name + "[" + fieldDataType.typeName + "]"; } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index d2699692d746e..14062b4caaf01 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -23,41 +23,41 @@ public enum DataType { // @formatter:off - // jdbc type, size, defPrecision,dispSize, int, rat, docvals - NULL( JDBCType.NULL, 0, 0, 0, false, false, false), - UNSUPPORTED( JDBCType.OTHER, 0, 0, 0, false, false, false), - BOOLEAN( JDBCType.BOOLEAN, 1, 1, 1, false, false, false), - BYTE( JDBCType.TINYINT, Byte.BYTES, 3, 5, true, false, true), - SHORT( JDBCType.SMALLINT, Short.BYTES, 5, 6, true, false, true), - INTEGER( JDBCType.INTEGER, Integer.BYTES, 10, 11, true, false, true), - LONG( JDBCType.BIGINT, Long.BYTES, 19, 20, true, false, true), + // esType jdbc type, size, defPrecision,dispSize, int, rat, docvals + NULL( "null", JDBCType.NULL, 0, 0, 0, false, false, false), + UNSUPPORTED( JDBCType.OTHER, 0, 0, 0, false, false, false), + BOOLEAN( "boolean", JDBCType.BOOLEAN, 1, 1, 1, false, false, false), + BYTE( "byte", JDBCType.TINYINT, Byte.BYTES, 3, 5, true, false, true), + SHORT( "short", JDBCType.SMALLINT, Short.BYTES, 5, 6, true, false, true), + INTEGER( "integer", JDBCType.INTEGER, Integer.BYTES, 10, 11, true, false, true), + LONG( "long", JDBCType.BIGINT, Long.BYTES, 19, 20, true, false, true), // 53 bits defaultPrecision ~ 15(15.95) decimal digits (53log10(2)), - DOUBLE( JDBCType.DOUBLE, Double.BYTES, 15, 25, false, true, true), + DOUBLE( "double", JDBCType.DOUBLE, Double.BYTES, 15, 25, false, true, true), // 24 bits defaultPrecision - 24*log10(2) =~ 7 (7.22) - FLOAT( JDBCType.REAL, Float.BYTES, 7, 15, false, true, true), - HALF_FLOAT( JDBCType.FLOAT, Double.BYTES, 16, 25, false, true, true), + FLOAT( "float", JDBCType.REAL, Float.BYTES, 7, 15, false, true, true), + HALF_FLOAT( "half_float", JDBCType.FLOAT, Double.BYTES, 16, 25, false, true, true), // precision is based on long - SCALED_FLOAT( JDBCType.FLOAT, Double.BYTES, 19, 25, false, true, true), - KEYWORD( JDBCType.VARCHAR, Integer.MAX_VALUE, 256, 0, false, false, true), - TEXT( JDBCType.VARCHAR, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false), - OBJECT( JDBCType.STRUCT, -1, 0, 0, false, false, false), - NESTED( JDBCType.STRUCT, -1, 0, 0, false, false, false), - BINARY( JDBCType.VARBINARY, -1, Integer.MAX_VALUE, 0, false, false, false), - DATE( JDBCType.DATE, Long.BYTES, 10, 10, false, false, true), + SCALED_FLOAT( "scaled_float", JDBCType.FLOAT, Double.BYTES, 19, 25, false, true, true), + KEYWORD( "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE, 256, 0, false, false, true), + TEXT( "text", JDBCType.VARCHAR, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false), + OBJECT( "object", JDBCType.STRUCT, -1, 0, 0, false, false, false), + NESTED( "nested", JDBCType.STRUCT, -1, 0, 0, false, false, false), + BINARY( "binary", JDBCType.VARBINARY, -1, Integer.MAX_VALUE, 0, false, false, false), + DATE( JDBCType.DATE, Long.BYTES, 10, 10, false, false, true), // since ODBC and JDBC interpret precision for Date as display size // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone) // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288 - DATETIME( JDBCType.TIMESTAMP, Long.BYTES, 24, 24, false, false, true), + DATETIME( "date", JDBCType.TIMESTAMP, Long.BYTES, 24, 24, false, false, true), // // specialized types // // IP can be v4 or v6. The latter has 2^128 addresses or 340,282,366,920,938,463,463,374,607,431,768,211,456 // aka 39 chars - IP( JDBCType.VARCHAR, 39, 39, 0, false, false, true), + IP( "ip", JDBCType.VARCHAR, 39, 39, 0, false, false, true), // // INTERVALS // the list is long as there are a lot of variations and that's what clients (ODBC) expect - // jdbc type, size, prec,disp, int, rat, docvals + // esType:null jdbc type, size, prec,disp, int, rat, docvals INTERVAL_YEAR( ExtTypes.INTERVAL_YEAR, Integer.BYTES, 7, 7, false, false, false), INTERVAL_MONTH( ExtTypes.INTERVAL_MONTH, Integer.BYTES, 7, 7, false, false, false), INTERVAL_DAY( ExtTypes.INTERVAL_DAY, Long.BYTES, 23, 23, false, false, false), @@ -126,7 +126,12 @@ public enum DataType { /** - * Elasticsearch type name + * Type's name used for error messages and column info for the clients + */ + public final String typeName; + + /** + * Elasticsearch data type that it maps to */ public final String esType; @@ -176,7 +181,13 @@ public enum DataType { DataType(SQLType sqlType, int size, int defaultPrecision, int displaySize, boolean isInteger, boolean isRational, boolean defaultDocValues) { - this.esType = name().toLowerCase(Locale.ROOT); + this(null, sqlType, size, defaultPrecision, displaySize, isInteger, isRational, defaultDocValues); + } + + DataType(String esType, SQLType sqlType, int size, int defaultPrecision, int displaySize, boolean isInteger, + boolean isRational, boolean defaultDocValues) { + this.typeName = name().toLowerCase(Locale.ROOT); + this.esType = esType; this.sqlType = sqlType; this.size = size; this.defaultPrecision = defaultPrecision; @@ -228,8 +239,6 @@ public static DataType fromOdbcType(String odbcType) { /** * Creates returns DataType enum corresponding to the specified es type - *

- * For any dataType DataType.fromTypeName(dataType.esType) == dataType */ public static DataType fromTypeName(String esType) { String uppercase = esType.toUpperCase(Locale.ROOT); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Schema.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Schema.java index 62a7881b6adc1..6faf2e5b224f4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Schema.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/Schema.java @@ -120,9 +120,9 @@ public String toString() { } sb.append(names.get(i)); sb.append(":"); - sb.append(types.get(i).esType); + sb.append(types.get(i).typeName); } sb.append("]"); return sb.toString(); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index bb328b2d8ffdc..6123bdf5d8fbb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -154,7 +154,7 @@ public static Map> fromMappings(EsIndex.. if (entry.getValue().size() > 1) { for (EsIndex index : indices) { EsField field = index.mapping().get(fieldName); - UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esType); + UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().typeName); fieldCaps.indices.add(index.name()); } //TODO: what about nonAgg/SearchIndices? @@ -171,7 +171,7 @@ private static void addFieldCaps(String parent, EsField field, String indexName, map = new HashMap<>(); merged.put(fieldName, map); } - FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esType, + FieldCapabilities caps = map.computeIfAbsent(field.getDataType().typeName, esType -> new UpdateableFieldCapabilities(fieldName, esType, isSearchable(field.getDataType()), isAggregatable(field.getDataType()))); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ParameterTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ParameterTests.java index 52f14550b1449..97b7415786341 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ParameterTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/ParameterTests.java @@ -28,7 +28,7 @@ public class ParameterTests extends ESTestCase { public void testSingleParameter() { Expression expression = new SqlParser().createExpression("a = \n?", Collections.singletonList( - new SqlTypedParamValue(DataType.KEYWORD.esType, "foo") + new SqlTypedParamValue(DataType.KEYWORD.typeName, "foo") )); logger.info(expression); assertThat(expression, instanceOf(Equals.class)); @@ -42,10 +42,10 @@ public void testSingleParameter() { public void testMultipleParameters() { Expression expression = new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList( - new SqlTypedParamValue(DataType.LONG.esType, 1L), - new SqlTypedParamValue(DataType.LONG.esType, 2L), - new SqlTypedParamValue(DataType.LONG.esType, 3L), - new SqlTypedParamValue(DataType.LONG.esType, 4L) + new SqlTypedParamValue(DataType.LONG.typeName, 1L), + new SqlTypedParamValue(DataType.LONG.typeName, 2L), + new SqlTypedParamValue(DataType.LONG.typeName, 3L), + new SqlTypedParamValue(DataType.LONG.typeName, 4L) )); assertThat(expression, instanceOf(Sub.class)); Sub sub = (Sub) expression; @@ -62,9 +62,9 @@ public void testMultipleParameters() { public void testNotEnoughParameters() { ParsingException ex = expectThrows(ParsingException.class, () -> new SqlParser().createExpression("(? + ? * ?) - ?", Arrays.asList( - new SqlTypedParamValue(DataType.LONG.esType, 1L), - new SqlTypedParamValue(DataType.LONG.esType, 2L), - new SqlTypedParamValue(DataType.LONG.esType, 3L) + new SqlTypedParamValue(DataType.LONG.typeName, 1L), + new SqlTypedParamValue(DataType.LONG.typeName, 2L), + new SqlTypedParamValue(DataType.LONG.typeName, 3L) ))); assertThat(ex.getMessage(), containsString("Not enough actual parameters")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java index 01b1d0d077930..8cbb0b528e9a6 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -144,9 +144,9 @@ public void testFunctionWithFunctionWithArg() { public void testFunctionWithFunctionWithArgAndParams() { String e = "POWER(?, {fn POWER({fn ABS(?)}, {fN ABS(?)})})"; Function f = (Function) parser.createExpression(e, - asList(new SqlTypedParamValue(DataType.LONG.esType, 1), - new SqlTypedParamValue(DataType.LONG.esType, 1), - new SqlTypedParamValue(DataType.LONG.esType, 1))); + asList(new SqlTypedParamValue(DataType.LONG.typeName, 1), + new SqlTypedParamValue(DataType.LONG.typeName, 1), + new SqlTypedParamValue(DataType.LONG.typeName, 1))); assertEquals(e, f.sourceText()); assertEquals(2, f.arguments().size()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/LikeEscapingParsingTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/LikeEscapingParsingTests.java index 5221f9695699d..a9b9723c8cc1f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/LikeEscapingParsingTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/LikeEscapingParsingTests.java @@ -13,11 +13,10 @@ import org.elasticsearch.xpack.sql.type.DataType; import static java.util.Collections.singletonList; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; - public class LikeEscapingParsingTests extends ESTestCase { private final SqlParser parser = new SqlParser(); @@ -33,7 +32,7 @@ private LikePattern like(String pattern) { Expression exp = null; boolean parameterized = randomBoolean(); if (parameterized) { - exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.esType, pattern))); + exp = parser.createExpression("exp LIKE ?", singletonList(new SqlTypedParamValue(DataType.KEYWORD.typeName, pattern))); } else { exp = parser.createExpression(format(null, "exp LIKE '{}'", pattern)); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index 9487986a711e7..d319fdb2a8feb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -231,7 +231,7 @@ public void testSysTablesTypesEnumerationWoString() throws Exception { } private SqlTypedParamValue param(Object value) { - return new SqlTypedParamValue(DataTypes.fromJava(value).esType, value); + return new SqlTypedParamValue(DataTypes.fromJava(value).typeName, value); } private Tuple sql(String sql, List params) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 99105af2b4aa6..314c53213dd92 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -632,16 +632,16 @@ public void testTopHitsAggregationWithOneArg() { "\"sort\":[{\"keyword\":{\"order\":\"asc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}")); } { - PhysicalPlan p = optimizeAndPlan("SELECT LAST(keyword) FROM test"); + PhysicalPlan p = optimizeAndPlan("SELECT LAST(date) FROM test"); assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec eqe = (EsQueryExec) p; assertEquals(1, eqe.output().size()); - assertEquals("LAST(keyword)", eqe.output().get(0).qualifiedName()); - assertTrue(eqe.output().get(0).dataType() == DataType.KEYWORD); + assertEquals("LAST(date)", eqe.output().get(0).qualifiedName()); + assertTrue(eqe.output().get(0).dataType() == DataType.DATETIME); assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""), endsWith("\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false," + - "\"explain\":false,\"docvalue_fields\":[{\"field\":\"keyword\"}]," + - "\"sort\":[{\"keyword\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}")); + "\"explain\":false,\"docvalue_fields\":[{\"field\":\"date\",\"format\":\"epoch_millis\"}]," + + "\"sort\":[{\"date\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"date\"}}]}}}}}")); } } @@ -661,17 +661,17 @@ public void testTopHitsAggregationWithTwoArgs() { } { - PhysicalPlan p = optimizeAndPlan("SELECT LAST(keyword, int) FROM test"); + PhysicalPlan p = optimizeAndPlan("SELECT LAST(date, int) FROM test"); assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec eqe = (EsQueryExec) p; assertEquals(1, eqe.output().size()); - assertEquals("LAST(keyword, int)", eqe.output().get(0).qualifiedName()); - assertTrue(eqe.output().get(0).dataType() == DataType.KEYWORD); + assertEquals("LAST(date, int)", eqe.output().get(0).qualifiedName()); + assertTrue(eqe.output().get(0).dataType() == DataType.DATETIME); assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""), endsWith("\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false," + - "\"explain\":false,\"docvalue_fields\":[{\"field\":\"keyword\"}]," + + "\"explain\":false,\"docvalue_fields\":[{\"field\":\"date\",\"format\":\"epoch_millis\"}]," + "\"sort\":[{\"int\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"integer\"}}," + - "{\"keyword\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"keyword\"}}]}}}}}")); + "{\"date\":{\"order\":\"desc\",\"missing\":\"_last\",\"unmapped_type\":\"date\"}}]}}}}}")); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java index 546d276e4ceb5..73b4ea8fa8daa 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java @@ -463,7 +463,7 @@ public void testCommonType() { public void testEsDataTypes() { for (DataType type : values()) { if (type != DATE) { // Doesn't have a corresponding type in ES - assertEquals(type, fromTypeName(type.esType)); + assertEquals(type, fromTypeName(type.typeName)); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java index a09a28ced7d5a..2a2488dda722f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java @@ -171,20 +171,20 @@ public void testNestedDoc() { public void testGeoField() { Map mapping = loadMapping("mapping-geo.json"); EsField dt = mapping.get("location"); - assertThat(dt.getDataType().esType, is("unsupported")); + assertThat(dt.getDataType().typeName, is("unsupported")); } public void testIpField() { Map mapping = loadMapping("mapping-ip.json"); assertThat(mapping.size(), is(1)); EsField dt = mapping.get("ip_addr"); - assertThat(dt.getDataType().esType, is("ip")); + assertThat(dt.getDataType().typeName, is("ip")); } public void testUnsupportedTypes() { Map mapping = loadMapping("mapping-unsupported.json"); EsField dt = mapping.get("range"); - assertThat(dt.getDataType().esType, is("unsupported")); + assertThat(dt.getDataType().typeName, is("unsupported")); } public static Map loadMapping(String name) { From df3fe00eb121fa3457ab0611c226d8c1f794abf7 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 5 Feb 2019 14:42:34 -0700 Subject: [PATCH 18/25] Update ilm-api.asciidoc, point to REMOVE policy (#38235) (#38464) --- docs/reference/ilm/apis/ilm-api.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/ilm/apis/ilm-api.asciidoc b/docs/reference/ilm/apis/ilm-api.asciidoc index edfc96d113fc7..aba8301a9a677 100644 --- a/docs/reference/ilm/apis/ilm-api.asciidoc +++ b/docs/reference/ilm/apis/ilm-api.asciidoc @@ -19,6 +19,7 @@ about Index Lifecycle Management. * <> * <> +* <> [float] [[ilm-api-management-endpoint]] From 6e0624a88bf55bf543fb8584136205fcc5c4f9d7 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 23:02:04 +0100 Subject: [PATCH 19/25] await fix CurtIT#testIndex until https://github.com/elastic/elasticsearch/pull/38451 is merged (#38466) --- .../src/test/java/org/elasticsearch/client/CrudIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 27e6221f0ec84..1fcbc73dc50b3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -427,6 +427,7 @@ public void testMultiGet() throws IOException { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/38451 contains a fix. sliencing for now") public void testIndex() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); { From 39c7acbfef490c6289c8a30098af03dadcc1c7bc Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 5 Feb 2019 22:17:26 +0000 Subject: [PATCH 20/25] [ML] Report index unavailable instead of waiting for lazy node (#38444) If a job cannot be assigned to a node because an index it requires is unavailable and there are lazy ML nodes then index unavailable should be reported as the assignment explanation rather than waiting for a lazy ML node. --- .../ml/action/TransportOpenJobAction.java | 50 ++++++----- .../action/TransportOpenJobActionTests.java | 83 ++++++++++++++----- 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index cbbae041a131c..34b4cf164984a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -152,32 +152,15 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j int fallbackMaxNumberOfOpenJobs, int maxMachineMemoryPercent, MlMemoryTracker memoryTracker, + boolean isMemoryTrackerRecentlyRefreshed, Logger logger) { - String resultsWriteAlias = AnomalyDetectorsIndex.resultsWriteAlias(jobId); - List unavailableIndices = verifyIndicesPrimaryShardsAreActive(resultsWriteAlias, clusterState); - if (unavailableIndices.size() != 0) { - String reason = "Not opening job [" + jobId + "], because not all primary shards are active for the following indices [" + - String.join(",", unavailableIndices) + "]"; - logger.debug(reason); - return new PersistentTasksCustomMetaData.Assignment(null, reason); - } // Try to allocate jobs according to memory usage, but if that's not possible (maybe due to a mixed version cluster or maybe // because of some weird OS problem) then fall back to the old mechanism of only considering numbers of assigned jobs - boolean allocateByMemory = true; - - if (memoryTracker.isRecentlyRefreshed() == false) { - - boolean scheduledRefresh = memoryTracker.asyncRefresh(); - if (scheduledRefresh) { - String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; - logger.debug(reason); - return new PersistentTasksCustomMetaData.Assignment(null, reason); - } else { - allocateByMemory = false; - logger.warn("Falling back to allocating job [{}] by job counts because a memory requirement refresh could not be scheduled", - jobId); - } + boolean allocateByMemory = isMemoryTrackerRecentlyRefreshed; + if (isMemoryTrackerRecentlyRefreshed == false) { + logger.warn("Falling back to allocating job [{}] by job counts because a memory requirement refresh could not be scheduled", + jobId); } List reasons = new LinkedList<>(); @@ -722,13 +705,34 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobP return AWAITING_UPGRADE; } - PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(params.getJobId(), + String jobId = params.getJobId(); + String resultsWriteAlias = AnomalyDetectorsIndex.resultsWriteAlias(jobId); + List unavailableIndices = verifyIndicesPrimaryShardsAreActive(resultsWriteAlias, clusterState); + if (unavailableIndices.size() != 0) { + String reason = "Not opening job [" + jobId + "], because not all primary shards are active for the following indices [" + + String.join(",", unavailableIndices) + "]"; + logger.debug(reason); + return new PersistentTasksCustomMetaData.Assignment(null, reason); + } + + boolean isMemoryTrackerRecentlyRefreshed = memoryTracker.isRecentlyRefreshed(); + if (isMemoryTrackerRecentlyRefreshed == false) { + boolean scheduledRefresh = memoryTracker.asyncRefresh(); + if (scheduledRefresh) { + String reason = "Not opening job [" + jobId + "] because job memory requirements are stale - refresh requested"; + logger.debug(reason); + return new PersistentTasksCustomMetaData.Assignment(null, reason); + } + } + + PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(jobId, params.getJob(), clusterState, maxConcurrentJobAllocations, fallbackMaxNumberOfOpenJobs, maxMachineMemoryPercent, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); if (assignment.getExecutorNode() == null) { int numMlNodes = 0; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 01183274852ce..9c8e8ffc4aa8e 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.persistent.PersistentTasksCustomMetaData; @@ -60,11 +61,9 @@ import java.net.InetAddress; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -80,11 +79,13 @@ public class TransportOpenJobActionTests extends ESTestCase { private MlMemoryTracker memoryTracker; + private boolean isMemoryTrackerRecentlyRefreshed; @Before public void setup() { memoryTracker = mock(MlMemoryTracker.class); - when(memoryTracker.isRecentlyRefreshed()).thenReturn(true); + isMemoryTrackerRecentlyRefreshed = true; + when(memoryTracker.isRecentlyRefreshed()).thenReturn(isMemoryTrackerRecentlyRefreshed); } public void testValidate_jobMissing() { @@ -141,7 +142,7 @@ public void testSelectLeastLoadedMlNode_byCount() { jobBuilder.setJobVersion(Version.CURRENT); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id4", jobBuilder.build(), - cs.build(), 2, 10, 30, memoryTracker, logger); + cs.build(), 2, 10, 30, memoryTracker, isMemoryTrackerRecentlyRefreshed, logger); assertEquals("", result.getExplanation()); assertEquals("_node_id3", result.getExecutorNode()); } @@ -177,7 +178,7 @@ public void testSelectLeastLoadedMlNode_maxCapacity() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id0", new ByteSizeValue(150, ByteSizeUnit.MB)).build(new Date()); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id0", job, cs.build(), 2, - maxRunningJobsPerNode, 30, memoryTracker, logger); + maxRunningJobsPerNode, 30, memoryTracker, isMemoryTrackerRecentlyRefreshed, logger); assertNull(result.getExecutorNode()); assertTrue(result.getExplanation().contains("because this node is full. Number of opened jobs [" + maxRunningJobsPerNode + "], xpack.ml.max_open_jobs [" + maxRunningJobsPerNode + "]")); @@ -203,7 +204,8 @@ public void testSelectLeastLoadedMlNode_noMlNodes() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id2", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 2, 10, 30, memoryTracker, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id2", job, cs.build(), 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertTrue(result.getExplanation().contains("because this node isn't a ml node")); assertNull(result.getExecutorNode()); } @@ -237,7 +239,8 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id6", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); ClusterState cs = csBuilder.build(); - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 2, 10, 30, memoryTracker, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id6", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertEquals("_node_id3", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -247,7 +250,8 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -258,7 +262,8 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertNull("no node selected, because stale task", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); @@ -269,7 +274,8 @@ public void testSelectLeastLoadedMlNode_maxConcurrentOpeningJobs() { csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertNull("no node selected, because null state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -307,7 +313,8 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); // Allocation won't be possible if the stale failed job is treated as opening - Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id7", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertEquals("_node_id1", result.getExecutorNode()); tasksBuilder = PersistentTasksCustomMetaData.builder(tasks); @@ -317,7 +324,8 @@ public void testSelectLeastLoadedMlNode_concurrentOpeningJobsAndStaleFailedJob() csBuilder = ClusterState.builder(cs); csBuilder.metaData(MetaData.builder(cs.metaData()).putCustom(PersistentTasksCustomMetaData.TYPE, tasks)); cs = csBuilder.build(); - result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 2, 10, 30, memoryTracker, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("job_id8", job, cs, 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertNull("no node selected, because OPENING state", result.getExecutorNode()); assertTrue(result.getExplanation().contains("because node exceeds [2] the maximum number of jobs [2] in opening state")); } @@ -349,7 +357,7 @@ public void testSelectLeastLoadedMlNode_noCompatibleJobTypeNodes() { metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); cs.metaData(metaData); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, - memoryTracker, logger); + memoryTracker, isMemoryTrackerRecentlyRefreshed, logger); assertThat(result.getExplanation(), containsString("because this node does not support jobs of type [incompatible_type]")); assertNull(result.getExecutorNode()); } @@ -377,7 +385,7 @@ public void testSelectLeastLoadedMlNode_noNodesPriorTo_V_5_5() { Job job = BaseMlIntegTestCase.createFareQuoteJob("job_id7", new ByteSizeValue(2, ByteSizeUnit.MB)).build(new Date()); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("incompatible_type_job", job, cs.build(), 2, 10, 30, - memoryTracker, logger); + memoryTracker, isMemoryTrackerRecentlyRefreshed, logger); assertThat(result.getExplanation(), containsString("because this node does not support machine learning jobs")); assertNull(result.getExecutorNode()); } @@ -404,7 +412,7 @@ public void testSelectLeastLoadedMlNode_jobWithRulesButNoNodeMeetsRequiredVersio Job job = jobWithRules("job_with_rules"); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, memoryTracker, - logger); + isMemoryTrackerRecentlyRefreshed, logger); assertThat(result.getExplanation(), containsString( "because jobs using custom_rules require a node of version [6.4.0] or higher")); assertNull(result.getExecutorNode()); @@ -432,7 +440,7 @@ public void testSelectLeastLoadedMlNode_jobWithRulesAndNodeMeetsRequiredVersion( Job job = jobWithRules("job_with_rules"); Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", job, cs.build(), 2, 10, 30, memoryTracker, - logger); + isMemoryTrackerRecentlyRefreshed, logger); assertNotNull(result.getExecutorNode()); } @@ -448,8 +456,8 @@ public void testSelectLeastLoadedMlNode_indexJobsCannotBeAssignedToPre660Node() cs.nodes(nodes); Job job = jobWithRules("post-v650-job"); - Assignment result = - TransportOpenJobAction.selectLeastLoadedMlNode("post-v650-job", job, cs.build(), 2, 10, 30, memoryTracker, logger); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("post-v650-job", job, cs.build(), 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), containsString("Not opening job [post-v650-job] on node [_node_name1] version [6.5.0], " + "because this node does not support jobs of version ")); @@ -460,7 +468,8 @@ public void testSelectLeastLoadedMlNode_indexJobsCannotBeAssignedToPre660Node() .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), nodeAttr, Collections.emptySet(), Version.V_6_6_0)); cs.nodes(nodes); - result = TransportOpenJobAction.selectLeastLoadedMlNode("post-v650-job", job, cs.build(), 2, 10, 30, memoryTracker, logger); + result = TransportOpenJobAction.selectLeastLoadedMlNode("post-v650-job", job, cs.build(), 2, 10, 30, memoryTracker, + isMemoryTrackerRecentlyRefreshed, logger); assertThat(result.getExplanation(), isEmptyOrNullString()); assertEquals("_node_id2", result.getExecutorNode()); } @@ -550,10 +559,10 @@ public void testJobTaskMatcherMatch() { public void testGetAssignment_GivenJobThatRequiresMigration() { ClusterService clusterService = mock(ClusterService.class); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>( - Arrays.asList(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, - MachineLearning.MAX_LAZY_ML_NODES) - )); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, + Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_LAZY_ML_NODES) + ); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); TransportOpenJobAction.OpenJobPersistentTasksExecutor executor = new TransportOpenJobAction.OpenJobPersistentTasksExecutor( @@ -563,6 +572,34 @@ public void testGetAssignment_GivenJobThatRequiresMigration() { assertEquals(TransportOpenJobAction.AWAITING_MIGRATION, executor.getAssignment(params, mock(ClusterState.class))); } + // An index being unavailable should take precedence over waiting for a lazy node + public void testGetAssignment_GivenUnavailableIndicesWithLazyNode() { + Settings settings = Settings.builder().put(MachineLearning.MAX_LAZY_ML_NODES.getKey(), 1).build(); + ClusterService clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings(settings, + Sets.newHashSet(MachineLearning.CONCURRENT_JOB_ALLOCATIONS, MachineLearning.MAX_MACHINE_MEMORY_PERCENT, + MachineLearning.MAX_LAZY_ML_NODES) + ); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + + ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name")); + MetaData.Builder metaData = MetaData.builder(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + addIndices(metaData, routingTable); + routingTable.remove(".ml-state"); + csBuilder.metaData(metaData); + csBuilder.routingTable(routingTable.build()); + + TransportOpenJobAction.OpenJobPersistentTasksExecutor executor = new TransportOpenJobAction.OpenJobPersistentTasksExecutor( + settings, clusterService, mock(AutodetectProcessManager.class), mock(MlMemoryTracker.class), mock(Client.class)); + + OpenJobAction.JobParams params = new OpenJobAction.JobParams("unavailable_index_with_lazy_node"); + params.setJob(mock(Job.class)); + assertEquals("Not opening job [unavailable_index_with_lazy_node], " + + "because not all primary shards are active for the following indices [.ml-state]", + executor.getAssignment(params, csBuilder.build()).getExplanation()); + } + public static void addJobTask(String jobId, String nodeId, JobState jobState, PersistentTasksCustomMetaData.Builder builder) { addJobTask(jobId, nodeId, jobState, builder, false); } From 1f2cd9fae85ab47854ebd77244e65f15e09fa3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 5 Feb 2019 23:25:52 +0100 Subject: [PATCH 21/25] Add typless client side GetIndexRequest calls and response class (#38422) The HLRC client currently uses `org.elasticsearch.action.admin.indices.get.GetIndexRequest` and `org.elasticsearch.action.admin.indices.get.GetIndexResponse` in its get index calls. Both request and response are designed for the typed APIs, including some return types e.g. for `getMappings()` which in the maps it returns still use a level including the type name. In order to change this without breaking existing users of the HLRC API, this PR introduces two new request and response objects in the `org.elasticsearch.client.indices` client package. These are used by the IndicesClient#get and IndicesClient#exists calls now by default and support the type-less API. The old request and response objects are still kept for use in similarly named, but deprecated methods. The newly introduced client side classes are simplified versions of the server side request/response classes since they don't need to support wire serialization, and only the response needs fromXContent parsing (but no xContent-serialization, since this is the responsibility of the server-side class). Also changing the return type of `GetIndexResponse#getMapping` to `Map getMappings()`, while it previously was returning another map keyed by the type-name. Similar getters return simple Maps instead of the ImmutableOpenMaps that the server side response objects return. Backport for #37778 Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 99 +++++++- .../client/IndicesRequestConverters.java | 55 ++++- .../client/indices/GetIndexRequest.java | 132 +++++++++++ .../client/indices/GetIndexResponse.java | 222 ++++++++++++++++++ .../client/ClusterRequestConvertersTests.java | 2 +- .../java/org/elasticsearch/client/CrudIT.java | 4 +- .../elasticsearch/client/IndicesClientIT.java | 93 +++++--- .../client/IndicesRequestConvertersTests.java | 92 ++++++-- .../client/RequestConvertersTests.java | 14 +- .../SnapshotRequestConvertersTests.java | 2 +- .../IndicesClientDocumentationIT.java | 24 +- .../client/indices/GetIndexRequestTests.java | 68 ++++++ .../client/indices/GetIndexResponseTests.java | 195 +++++++++++++++ .../admin/indices/get/GetIndexResponse.java | 2 +- .../admin/indices/RestGetIndicesAction.java | 5 +- .../indices/get/GetIndexResponseTests.java | 11 + 16 files changed, 936 insertions(+), 84 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index f484d7375ba6f..18ecac6a164ac 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -34,10 +34,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -59,6 +55,10 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; @@ -901,6 +901,41 @@ public void getAsync(GetIndexRequest getIndexRequest, RequestOptions options, GetIndexResponse::fromXContent, listener, emptySet()); } + /** + * Retrieve information about one or more indexes + * See + * Indices Get Index API on elastic.co + * @param getIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #get(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public org.elasticsearch.action.admin.indices.get.GetIndexResponse get( + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options, + org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, emptySet()); + } + + /** + * Retrieve information about one or more indexes + * See + * Indices Get Index API on elastic.co + * @param getIndexRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #getAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object. + */ + @Deprecated + public void getAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getIndexRequest, IndicesRequestConverters::getIndex, options, + org.elasticsearch.action.admin.indices.get.GetIndexResponse::fromXContent, listener, emptySet()); + } + /** * Force merge one or more indices using the Force Merge API. * See @@ -1058,15 +1093,38 @@ public boolean exists(GetIndexRequest request, RequestOptions options) throws IO ); } + /** + * Checks if the index (indices) exists or not. + * See + * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #exists(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public boolean exists(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequest( + request, + IndicesRequestConverters::indicesExist, + options, + RestHighLevelClient::convertExistsResponse, + Collections.emptySet() + ); + } + /** * Checks if the index (indices) exists or not. *

* See * Indices Exists API on elastic.co - * @deprecated Prefer {@link #exists(GetIndexRequest, RequestOptions)} + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #exists(GetIndexRequest, RequestOptions)} should be used instead, which accepts a new request object. */ @Deprecated - public boolean exists(GetIndexRequest request, Header... headers) throws IOException { + public boolean exists(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, Header... headers) throws IOException { return restHighLevelClient.performRequest( request, IndicesRequestConverters::indicesExist, @@ -1095,15 +1153,40 @@ public void existsAsync(GetIndexRequest request, RequestOptions options, ActionL ); } + /** + * Asynchronously checks if the index (indices) exists or not. + * See + * Indices Exists API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object. + */ + @Deprecated + public void existsAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsync( + request, + IndicesRequestConverters::indicesExist, + options, + RestHighLevelClient::convertExistsResponse, + listener, + Collections.emptySet() + ); + } + /** * Asynchronously checks if the index (indices) exists or not. *

* See * Indices Exists API on elastic.co - * @deprecated Prefer {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} + * @deprecated This method uses an old request object which still refers to types, a deprecated feature. The method + * {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} should be used instead, which accepts a new request object. */ @Deprecated - public void existsAsync(GetIndexRequest request, ActionListener listener, Header... headers) { + public void existsAsync(org.elasticsearch.action.admin.indices.get.GetIndexRequest request, ActionListener listener, + Header... headers) { restHighLevelClient.performRequestAsync( request, IndicesRequestConverters::indicesExist, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index d26f9babed889..13fc6a8e08bd3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -33,8 +33,6 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; @@ -48,6 +46,8 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; @@ -151,6 +151,10 @@ static Request putMapping(PutMappingRequest putMappingRequest) throws IOExceptio return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest} that still supports + * types + */ @Deprecated static Request putMapping(org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest putMappingRequest) throws IOException { // The concreteIndex is an internal concept, not applicable to requests made over the REST API. @@ -375,6 +379,28 @@ static Request getSettings(GetSettingsRequest getSettingsRequest) { return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.get.GetIndexRequest} that + * still supports types + */ + @Deprecated + static Request getIndex(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest) { + String[] indices = getIndexRequest.indices() == null ? Strings.EMPTY_ARRAY : getIndexRequest.indices(); + + String endpoint = RequestConverters.endpoint(indices); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withIndicesOptions(getIndexRequest.indicesOptions()); + params.withLocal(getIndexRequest.local()); + params.withIncludeDefaults(getIndexRequest.includeDefaults()); + params.withHuman(getIndexRequest.humanReadable()); + params.withMasterTimeout(getIndexRequest.masterNodeTimeout()); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); + + return request; + } + static Request getIndex(GetIndexRequest getIndexRequest) { String[] indices = getIndexRequest.indices() == null ? Strings.EMPTY_ARRAY : getIndexRequest.indices(); @@ -388,11 +414,33 @@ static Request getIndex(GetIndexRequest getIndexRequest) { params.withHuman(getIndexRequest.humanReadable()); params.withMasterTimeout(getIndexRequest.masterNodeTimeout()); // Force "include_type_name" parameter since responses need to be compatible when coming from 7.0 nodes - params.withIncludeTypeName(true); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, Boolean.FALSE.toString()); return request; } + /** + * converter for the legacy server-side {@link org.elasticsearch.action.admin.indices.get.GetIndexRequest} that + * still supports types + */ + @Deprecated + static Request indicesExist(org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest) { + // this can be called with no indices as argument by transport client, not via REST though + if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) { + throw new IllegalArgumentException("indices are mandatory"); + } + String endpoint = RequestConverters.endpoint(getIndexRequest.indices(), ""); + Request request = new Request(HttpHead.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withLocal(getIndexRequest.local()); + params.withHuman(getIndexRequest.humanReadable()); + params.withIndicesOptions(getIndexRequest.indicesOptions()); + params.withIncludeDefaults(getIndexRequest.includeDefaults()); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); + return request; + } + static Request indicesExist(GetIndexRequest getIndexRequest) { // this can be called with no indices as argument by transport client, not via REST though if (getIndexRequest.indices() == null || getIndexRequest.indices().length == 0) { @@ -406,6 +454,7 @@ static Request indicesExist(GetIndexRequest getIndexRequest) { params.withHuman(getIndexRequest.humanReadable()); params.withIndicesOptions(getIndexRequest.indicesOptions()); params.withIncludeDefaults(getIndexRequest.includeDefaults()); + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, Boolean.FALSE.toString()); return request; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java new file mode 100644 index 0000000000000..227b1b4d36abc --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexRequest.java @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.TimedRequest; +import org.elasticsearch.common.util.ArrayUtils; + +/** + * A request to retrieve information about an index. + */ +public class GetIndexRequest extends TimedRequest { + + public enum Feature { + ALIASES, + MAPPINGS, + SETTINGS; + } + + static final Feature[] DEFAULT_FEATURES = new Feature[] { Feature.ALIASES, Feature.MAPPINGS, Feature.SETTINGS }; + private Feature[] features = DEFAULT_FEATURES; + private boolean humanReadable = false; + private transient boolean includeDefaults = false; + + private final String[] indices; + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true); + private boolean local = false; + + public GetIndexRequest(String... indices) { + this.indices = indices; + } + + /** + * The indices into which the mappings will be put. + */ + public String[] indices() { + return indices; + } + + public IndicesOptions indicesOptions() { + return indicesOptions; + } + + public GetIndexRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + + public final GetIndexRequest local(boolean local) { + this.local = local; + return this; + } + + /** + * Return local information, do not retrieve the state from master node (default: false). + * @return true if local information is to be returned; + * false if information is to be retrieved from master node (default). + */ + public final boolean local() { + return local; + } + + public GetIndexRequest features(Feature... features) { + if (features == null) { + throw new IllegalArgumentException("features cannot be null"); + } else { + this.features = features; + } + return this; + } + + public GetIndexRequest addFeatures(Feature... features) { + if (this.features == DEFAULT_FEATURES) { + return features(features); + } else { + return features(ArrayUtils.concat(features(), features, Feature.class)); + } + } + + public Feature[] features() { + return features; + } + + public GetIndexRequest humanReadable(boolean humanReadable) { + this.humanReadable = humanReadable; + return this; + } + + public boolean humanReadable() { + return humanReadable; + } + + /** + * Sets the value of "include_defaults". + * + * @param includeDefaults value of "include_defaults" to be set. + * @return this request + */ + public GetIndexRequest includeDefaults(boolean includeDefaults) { + this.includeDefaults = includeDefaults; + return this; + } + + /** + * Whether to return all default settings for each of the indices. + * + * @return true if defaults settings for each of the indices need to returned; + * false otherwise. + */ + public boolean includeDefaults() { + return includeDefaults; + } + + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java new file mode 100644 index 0000000000000..3d98f93df47d9 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/GetIndexResponse.java @@ -0,0 +1,222 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.index.mapper.MapperService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * A client side response for a get index action. + */ +public class GetIndexResponse { + + private Map mappings; + private Map> aliases; + private Map settings; + private Map defaultSettings; + private String[] indices; + + GetIndexResponse(String[] indices, + Map mappings, + Map> aliases, + Map settings, + Map defaultSettings) { + this.indices = indices; + // to have deterministic order + Arrays.sort(indices); + if (mappings != null) { + this.mappings = mappings; + } + if (aliases != null) { + this.aliases = aliases; + } + if (settings != null) { + this.settings = settings; + } + if (defaultSettings != null) { + this.defaultSettings = defaultSettings; + } + } + + public String[] getIndices() { + return indices; + } + + public Map getMappings() { + return mappings; + } + + public Map> getAliases() { + return aliases; + } + + /** + * If the originating {@link GetIndexRequest} object was configured to include + * defaults, this will contain a mapping of index name to {@link Settings} objects. + * The returned {@link Settings} objects will contain only those settings taking + * effect as defaults. Any settings explicitly set on the index will be available + * via {@link #getSettings()}. + * See also {@link GetIndexRequest#includeDefaults(boolean)} + */ + public Map getDefaultSettings() { + return defaultSettings; + } + + public Map getSettings() { + return settings; + } + + /** + * Returns the string value for the specified index and setting. If the includeDefaults flag was not set or set to + * false on the {@link GetIndexRequest}, this method will only return a value where the setting was explicitly set + * on the index. If the includeDefaults flag was set to true on the {@link GetIndexRequest}, this method will fall + * back to return the default value if the setting was not explicitly set. + */ + public String getSetting(String index, String setting) { + Settings indexSettings = settings.get(index); + if (setting != null) { + if (indexSettings != null && indexSettings.hasValue(setting)) { + return indexSettings.get(setting); + } else { + Settings defaultIndexSettings = defaultSettings.get(index); + if (defaultIndexSettings != null) { + return defaultIndexSettings.get(setting); + } else { + return null; + } + } + } else { + return null; + } + } + + private static List parseAliases(XContentParser parser) throws IOException { + List indexAliases = new ArrayList<>(); + // We start at START_OBJECT since parseIndexEntry ensures that + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + indexAliases.add(AliasMetaData.Builder.fromXContent(parser)); + } + return indexAliases; + } + + private static MappingMetaData parseMappings(XContentParser parser) throws IOException { + return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, parser.map()); + } + + private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException { + List indexAliases = null; + MappingMetaData indexMappings = null; + Settings indexSettings = null; + Settings indexDefaultSettings = null; + // We start at START_OBJECT since fromXContent ensures that + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == Token.START_OBJECT) { + switch (parser.currentName()) { + case "aliases": + indexAliases = parseAliases(parser); + break; + case "mappings": + indexMappings = parseMappings(parser); + break; + case "settings": + indexSettings = Settings.fromXContent(parser); + break; + case "defaults": + indexDefaultSettings = Settings.fromXContent(parser); + break; + default: + parser.skipChildren(); + } + } else if (parser.currentToken() == Token.START_ARRAY) { + parser.skipChildren(); + } + } + return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings); + } + + // This is just an internal container to make stuff easier for returning + private static class IndexEntry { + List indexAliases = new ArrayList<>(); + MappingMetaData indexMappings; + Settings indexSettings = Settings.EMPTY; + Settings indexDefaultSettings = Settings.EMPTY; + IndexEntry(List indexAliases, MappingMetaData indexMappings, Settings indexSettings, Settings indexDefaultSettings) { + if (indexAliases != null) this.indexAliases = indexAliases; + if (indexMappings != null) this.indexMappings = indexMappings; + if (indexSettings != null) this.indexSettings = indexSettings; + if (indexDefaultSettings != null) this.indexDefaultSettings = indexDefaultSettings; + } + } + + public static GetIndexResponse fromXContent(XContentParser parser) throws IOException { + Map> aliases = new HashMap<>(); + Map mappings = new HashMap<>(); + Map settings = new HashMap<>(); + Map defaultSettings = new HashMap<>(); + List indices = new ArrayList<>(); + + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + + while (!parser.isClosed()) { + if (parser.currentToken() == Token.START_OBJECT) { + // we assume this is an index entry + String indexName = parser.currentName(); + indices.add(indexName); + IndexEntry indexEntry = parseIndexEntry(parser); + // make the order deterministic + CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias)); + aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases)); + mappings.put(indexName, indexEntry.indexMappings); + settings.put(indexName, indexEntry.indexSettings); + if (indexEntry.indexDefaultSettings.isEmpty() == false) { + defaultSettings.put(indexName, indexEntry.indexDefaultSettings); + } + } else if (parser.currentToken() == Token.START_ARRAY) { + parser.skipChildren(); + } else { + parser.nextToken(); + } + } + return new GetIndexResponse(indices.toArray(new String[0]), mappings, aliases, settings, defaultSettings); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java index c78935464d9fc..963b4b783bdae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java @@ -75,7 +75,7 @@ public void testClusterGetSettings() throws IOException { public void testClusterHealth() { ClusterHealthRequest healthRequest = new ClusterHealthRequest(); Map expectedParams = new HashMap<>(); - setRandomLocal(healthRequest, expectedParams); + setRandomLocal(healthRequest::local, expectedParams); String timeoutType = randomFrom("timeout", "masterTimeout", "both", "none"); String timeout = randomTimeValue(); String masterTimeout = randomTimeValue(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 1fcbc73dc50b3..7baa4d0fbfc55 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; import org.elasticsearch.action.bulk.BulkRequest; @@ -49,6 +48,7 @@ import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -1210,7 +1210,7 @@ public void testUrlEncode() throws IOException { assertEquals(docId, getResponse.getId()); } - assertTrue(highLevelClient().indices().exists(new GetIndexRequest().indices(indexPattern, "index"), RequestOptions.DEFAULT)); + assertTrue(highLevelClient().indices().exists(new GetIndexRequest(indexPattern, "index"), RequestOptions.DEFAULT)); } public void testParamsEncode() throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index c549399d193a4..0aed9f3baaba0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -39,10 +39,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -70,6 +66,10 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; @@ -79,6 +79,7 @@ import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Setting; @@ -96,6 +97,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import org.elasticsearch.rest.action.admin.indices.RestGetFieldMappingAction; +import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; @@ -134,14 +136,11 @@ public void testIndicesExists() throws IOException { String indexName = "test_index_exists_index_present"; createIndex(indexName, Settings.EMPTY); - GetIndexRequest request = new GetIndexRequest(); - request.indices(indexName); + GetIndexRequest request = new GetIndexRequest(indexName); boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertTrue(response); @@ -151,14 +150,11 @@ public void testIndicesExists() throws IOException { { String indexName = "non_existent_index"; - GetIndexRequest request = new GetIndexRequest(); - request.indices(indexName); + GetIndexRequest request = new GetIndexRequest(indexName); boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); @@ -171,19 +167,34 @@ public void testIndicesExists() throws IOException { String nonExistentIndex = "oranges"; - GetIndexRequest request = new GetIndexRequest(); - request.indices(existingIndex, nonExistentIndex); + GetIndexRequest request = new GetIndexRequest(existingIndex, nonExistentIndex); boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); } + } + public void testIndicesExistsWithTypes() throws IOException { + // Index present + String indexName = "test_index_exists_index_present"; + createIndex(indexName, Settings.EMPTY); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest request + = new org.elasticsearch.action.admin.indices.get.GetIndexRequest(); + request.indices(indexName); + + boolean response = execute( + request, + highLevelClient().indices()::exists, + highLevelClient().indices()::existsAsync, + highLevelClient().indices()::exists, + highLevelClient().indices()::existsAsync + ); + assertTrue(response); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -423,8 +434,7 @@ public void testGetIndex() throws IOException { String mappings = "\"_doc\":{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}"; createIndex(indexName, basicSettings, mappings); - GetIndexRequest getIndexRequest = new GetIndexRequest() - .indices(indexName).includeDefaults(false); + GetIndexRequest getIndexRequest = new GetIndexRequest(indexName).includeDefaults(false); GetIndexResponse getIndexResponse = execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync); @@ -433,8 +443,12 @@ public void testGetIndex() throws IOException { assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); assertNotNull(getIndexResponse.getMappings().get(indexName)); - assertNotNull(getIndexResponse.getMappings().get(indexName).get("_doc")); - Object o = getIndexResponse.getMappings().get(indexName).get("_doc").getSourceAsMap().get("properties"); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + MappingMetaData mappingMetaData = getIndexResponse.getMappings().get(indexName); + assertNotNull(mappingMetaData); + assertEquals("_doc", mappingMetaData.type()); + assertEquals("{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}", mappingMetaData.source().string()); + Object o = mappingMetaData.getSourceAsMap().get("properties"); assertThat(o, instanceOf(Map.class)); //noinspection unchecked assertThat(((Map) o).get("field-1"), instanceOf(Map.class)); @@ -443,6 +457,33 @@ public void testGetIndex() throws IOException { assertEquals("integer", fieldMapping.get("type")); } + @SuppressWarnings("unchecked") + public void testGetIndexWithTypes() throws IOException { + String indexName = "get_index_test"; + Settings basicSettings = Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + String mappings = "\"_doc\":{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}"; + createIndex(indexName, basicSettings, mappings); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indexName).includeDefaults(false); + org.elasticsearch.action.admin.indices.get.GetIndexResponse getIndexResponse = execute(getIndexRequest, + highLevelClient().indices()::get, highLevelClient().indices()::getAsync, + expectWarnings(RestGetIndicesAction.TYPES_DEPRECATION_MESSAGE)); + + // default settings should be null + assertNull(getIndexResponse.getSetting(indexName, "index.refresh_interval")); + assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); + assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + MappingMetaData mappingMetaData = getIndexResponse.getMappings().get(indexName).get("_doc"); + assertNotNull(mappingMetaData); + assertEquals("_doc", mappingMetaData.type()); + assertEquals("{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}", mappingMetaData.source().string()); + } + @SuppressWarnings("unchecked") public void testGetIndexWithDefaults() throws IOException { String indexName = "get_index_test"; @@ -453,19 +494,18 @@ public void testGetIndexWithDefaults() throws IOException { String mappings = "\"_doc\":{\"properties\":{\"field-1\":{\"type\":\"integer\"}}}"; createIndex(indexName, basicSettings, mappings); - GetIndexRequest getIndexRequest = new GetIndexRequest() - .indices(indexName).includeDefaults(true); + GetIndexRequest getIndexRequest = new GetIndexRequest(indexName).includeDefaults(true); GetIndexResponse getIndexResponse = execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync); assertNotNull(getIndexResponse.getSetting(indexName, "index.refresh_interval")); assertEquals(IndexSettings.DEFAULT_REFRESH_INTERVAL, - getIndexResponse.defaultSettings().get(indexName).getAsTime("index.refresh_interval", null)); + getIndexResponse.getDefaultSettings().get(indexName).getAsTime("index.refresh_interval", null)); assertEquals("1", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_SHARDS)); assertEquals("0", getIndexResponse.getSetting(indexName, SETTING_NUMBER_OF_REPLICAS)); assertNotNull(getIndexResponse.getMappings().get(indexName)); - assertNotNull(getIndexResponse.getMappings().get(indexName).get("_doc")); - Object o = getIndexResponse.getMappings().get(indexName).get("_doc").getSourceAsMap().get("properties"); + assertNotNull(getIndexResponse.getMappings().get(indexName)); + Object o = getIndexResponse.getMappings().get(indexName).getSourceAsMap().get("properties"); assertThat(o, instanceOf(Map.class)); assertThat(((Map) o).get("field-1"), instanceOf(Map.class)); Map fieldMapping = (Map) ((Map) o).get("field-1"); @@ -476,7 +516,7 @@ public void testGetIndexNonExistentIndex() throws IOException { String nonExistentIndex = "index_that_doesnt_exist"; assertFalse(indexExists(nonExistentIndex)); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(nonExistentIndex); + GetIndexRequest getIndexRequest = new GetIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(getIndexRequest, highLevelClient().indices()::get, highLevelClient().indices()::getAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); @@ -1564,7 +1604,6 @@ public void testCRUDIndexTemplate() throws Exception { assertTrue(template2.aliases().isEmpty()); assertThat(template2.settings().get("index.number_of_shards"), equalTo("2")); assertThat(template2.settings().get("index.number_of_replicas"), equalTo("0")); - List names = randomBoolean() ? Arrays.asList("*-1", "template-2") : Arrays.asList("template-*"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index 5ddaf873b9d65..e4909787146c4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; @@ -51,6 +50,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.IndexTemplatesExistRequest; @@ -103,13 +103,14 @@ public void testAnalyzeRequest() throws Exception { public void testIndicesExist() { String[] indices = RequestConvertersTests.randomIndicesNames(1, 10); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(indices); + GetIndexRequest getIndexRequest = new GetIndexRequest(indices); Map expectedParams = new HashMap<>(); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, Boolean.FALSE.toString()); RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomHumanReadable(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest::includeDefaults, expectedParams); final Request request = IndicesRequestConverters.indicesExist(getIndexRequest); @@ -123,7 +124,35 @@ public void testIndicesExistEmptyIndices() { LuceneTestCase.expectThrows(IllegalArgumentException.class, () -> IndicesRequestConverters.indicesExist(new GetIndexRequest())); LuceneTestCase.expectThrows(IllegalArgumentException.class, () - -> IndicesRequestConverters.indicesExist(new GetIndexRequest().indices((String[]) null))); + -> IndicesRequestConverters.indicesExist(new GetIndexRequest((String[]) null))); + } + + public void testIndicesExistEmptyIndicesWithTypes() { + LuceneTestCase.expectThrows(IllegalArgumentException.class, + () -> IndicesRequestConverters.indicesExist(new org.elasticsearch.action.admin.indices.get.GetIndexRequest())); + LuceneTestCase.expectThrows(IllegalArgumentException.class, () -> IndicesRequestConverters + .indicesExist(new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices((String[]) null))); + } + + public void testIndicesExistWithTypes() { + String[] indices = RequestConvertersTests.randomIndicesNames(1, 10); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indices); + + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + RequestConvertersTests.setRandomIncludeDefaults(getIndexRequest::includeDefaults, expectedParams); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); + + final Request request = IndicesRequestConverters.indicesExist(getIndexRequest); + + Assert.assertEquals(HttpHead.METHOD_NAME, request.getMethod()); + Assert.assertEquals("/" + String.join(",", indices), request.getEndpoint()); + Assert.assertThat(expectedParams, equalTo(request.getParameters())); + Assert.assertNull(request.getEntity()); } public void testCreateIndex() throws IOException { @@ -289,7 +318,7 @@ public void testGetMappingWithTypes() { RequestConvertersTests.setRandomIndicesOptions(getMappingRequest::indicesOptions, getMappingRequest::indicesOptions, expectedParams); RequestConvertersTests.setRandomMasterTimeout(getMappingRequest, expectedParams); - RequestConvertersTests.setRandomLocal(getMappingRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getMappingRequest::local, expectedParams); expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); Request request = IndicesRequestConverters.getMappings(getMappingRequest); @@ -436,7 +465,7 @@ public void testGetSettings() throws IOException { RequestConvertersTests.setRandomIndicesOptions(getSettingsRequest::indicesOptions, getSettingsRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getSettingsRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getSettingsRequest::local, expectedParams); if (ESTestCase.randomBoolean()) { // the request object will not have include_defaults present unless it is set to @@ -477,15 +506,50 @@ public void testGetSettings() throws IOException { public void testGetIndex() throws IOException { String[] indicesUnderTest = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); - GetIndexRequest getIndexRequest = new GetIndexRequest().indices(indicesUnderTest); + GetIndexRequest getIndexRequest = new GetIndexRequest(indicesUnderTest); Map expectedParams = new HashMap<>(); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, Boolean.FALSE.toString()); RequestConvertersTests.setRandomMasterTimeout(getIndexRequest, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); - RequestConvertersTests.setRandomLocal(getIndexRequest, expectedParams); - RequestConvertersTests.setRandomHumanReadable(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); + + if (ESTestCase.randomBoolean()) { + // the request object will not have include_defaults present unless it is set to + // true + getIndexRequest.includeDefaults(ESTestCase.randomBoolean()); + if (getIndexRequest.includeDefaults()) { + expectedParams.put("include_defaults", Boolean.toString(true)); + } + } + + StringJoiner endpoint = new StringJoiner("/", "/", ""); + if (indicesUnderTest != null && indicesUnderTest.length > 0) { + endpoint.add(String.join(",", indicesUnderTest)); + } + + Request request = IndicesRequestConverters.getIndex(getIndexRequest); + + Assert.assertThat(endpoint.toString(), equalTo(request.getEndpoint())); + Assert.assertThat(request.getParameters(), equalTo(expectedParams)); + Assert.assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); + Assert.assertThat(request.getEntity(), nullValue()); + } + + public void testGetIndexWithTypes() throws IOException { + String[] indicesUnderTest = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); + + org.elasticsearch.action.admin.indices.get.GetIndexRequest getIndexRequest = + new org.elasticsearch.action.admin.indices.get.GetIndexRequest().indices(indicesUnderTest); + + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomMasterTimeout(getIndexRequest, expectedParams); + RequestConvertersTests.setRandomIndicesOptions(getIndexRequest::indicesOptions, getIndexRequest::indicesOptions, expectedParams); + RequestConvertersTests.setRandomLocal(getIndexRequest::local, expectedParams); + RequestConvertersTests.setRandomHumanReadable(getIndexRequest::humanReadable, expectedParams); // Force "include_type_name" parameter since responses need to be compatible when coming from 7.0 nodes - expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, Boolean.TRUE.toString()); if (ESTestCase.randomBoolean()) { // the request object will not have include_defaults present unless it is set to @@ -737,7 +801,7 @@ public void testExistsAlias() { } getAliasesRequest.aliases(aliases); Map expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomLocal(getAliasesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getAliasesRequest::local, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getAliasesRequest::indicesOptions, getAliasesRequest::indicesOptions, expectedParams); @@ -872,7 +936,7 @@ public void testGetAlias() { GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); Map expectedParams = new HashMap<>(); - RequestConvertersTests.setRandomLocal(getAliasesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getAliasesRequest::local, expectedParams); RequestConvertersTests.setRandomIndicesOptions(getAliasesRequest::indicesOptions, getAliasesRequest::indicesOptions, expectedParams); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 4b15774568a6a..780b58750c89c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkShardRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -50,7 +49,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -1763,20 +1761,20 @@ static IndicesOptions setRandomIndicesOptions(IndicesOptions indicesOptions, Map return indicesOptions; } - static void setRandomIncludeDefaults(GetIndexRequest request, Map expectedParams) { + static void setRandomIncludeDefaults(Consumer setter, Map expectedParams) { if (randomBoolean()) { boolean includeDefaults = randomBoolean(); - request.includeDefaults(includeDefaults); + setter.accept(includeDefaults); if (includeDefaults) { expectedParams.put("include_defaults", String.valueOf(includeDefaults)); } } } - static void setRandomHumanReadable(GetIndexRequest request, Map expectedParams) { + static void setRandomHumanReadable(Consumer setter, Map expectedParams) { if (randomBoolean()) { boolean humanReadable = randomBoolean(); - request.humanReadable(humanReadable); + setter.accept(humanReadable); if (humanReadable) { expectedParams.put("human", String.valueOf(humanReadable)); } @@ -1793,10 +1791,6 @@ static void setRandomLocal(Consumer setter, Map expecte } } - static void setRandomLocal(MasterNodeReadRequest request, Map expectedParams) { - setRandomLocal(request::local, expectedParams); - } - static void setRandomTimeout(TimedRequest request, TimeValue defaultTimeout, Map expectedParams) { setRandomTimeout(s -> request.setTimeout(TimeValue.parseTimeValue(s, request.getClass().getName() + ".timeout")), diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java index ca86a9120422b..66720b70ee3a6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotRequestConvertersTests.java @@ -58,7 +58,7 @@ public void testGetRepositories() { GetRepositoriesRequest getRepositoriesRequest = new GetRepositoriesRequest(); RequestConvertersTests.setRandomMasterTimeout(getRepositoriesRequest, expectedParams); - RequestConvertersTests.setRandomLocal(getRepositoriesRequest, expectedParams); + RequestConvertersTests.setRandomLocal(getRepositoriesRequest::local, expectedParams); if (randomBoolean()) { String[] entries = new String[]{"a", "b", "c"}; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index c20389c24b10a..9c6d0d4effe08 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -38,10 +38,6 @@ import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexResponse; -import org.elasticsearch.client.indices.GetFieldMappingsRequest; -import org.elasticsearch.client.indices.GetFieldMappingsResponse; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -73,6 +69,10 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; import org.elasticsearch.client.indices.FreezeIndexRequest; +import org.elasticsearch.client.indices.GetFieldMappingsRequest; +import org.elasticsearch.client.indices.GetFieldMappingsResponse; +import org.elasticsearch.client.indices.GetIndexRequest; +import org.elasticsearch.client.indices.GetIndexResponse; import org.elasticsearch.client.indices.GetIndexTemplatesRequest; import org.elasticsearch.client.indices.GetMappingsRequest; import org.elasticsearch.client.indices.GetMappingsResponse; @@ -81,9 +81,7 @@ import org.elasticsearch.client.indices.UnfreezeIndexRequest; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -140,8 +138,7 @@ public void testIndicesExist() throws IOException { { // tag::indices-exists-request - GetIndexRequest request = new GetIndexRequest(); - request.indices("twitter"); // <1> + GetIndexRequest request = new GetIndexRequest("twitter"); // <1> // end::indices-exists-request IndicesOptions indicesOptions = IndicesOptions.strictExpand(); @@ -168,8 +165,7 @@ public void testIndicesExistAsync() throws Exception { } { - GetIndexRequest request = new GetIndexRequest(); - request.indices("twitter"); + GetIndexRequest request = new GetIndexRequest("twitter"); // tag::indices-exists-execute-listener ActionListener listener = new ActionListener() { @@ -1227,7 +1223,7 @@ public void testGetIndex() throws Exception { } // tag::get-index-request - GetIndexRequest request = new GetIndexRequest().indices("index"); // <1> + GetIndexRequest request = new GetIndexRequest("index"); // <1> // end::get-index-request // tag::get-index-request-indicesOptions @@ -1243,13 +1239,13 @@ public void testGetIndex() throws Exception { // end::get-index-execute // tag::get-index-response - ImmutableOpenMap indexMappings = getIndexResponse.getMappings().get("index"); // <1> - Map indexTypeMappings = indexMappings.get("_doc").getSourceAsMap(); // <2> + MappingMetaData indexMappings = getIndexResponse.getMappings().get("index"); // <1> + Map indexTypeMappings = indexMappings.getSourceAsMap(); // <2> List indexAliases = getIndexResponse.getAliases().get("index"); // <3> String numberOfShardsString = getIndexResponse.getSetting("index", "index.number_of_shards"); // <4> Settings indexSettings = getIndexResponse.getSettings().get("index"); // <5> Integer numberOfShards = indexSettings.getAsInt("index.number_of_shards", null); // <6> - TimeValue time = getIndexResponse.defaultSettings().get("index") + TimeValue time = getIndexResponse.getDefaultSettings().get("index") .getAsTime("index.refresh_interval", null); // <7> // end::get-index-response diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java new file mode 100644 index 0000000000000..46b64aab6d406 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexRequestTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.indices.GetIndexRequest.Feature; +import org.elasticsearch.test.ESTestCase; + +public class GetIndexRequestTests extends ESTestCase { + + public void testIndices() { + String[] indices = generateRandomStringArray(5, 5, false, true); + GetIndexRequest request = new GetIndexRequest(indices); + assertArrayEquals(indices, request.indices()); + } + + public void testFeatures() { + int numFeature = randomIntBetween(0, 3); + Feature[] features = new Feature[numFeature]; + for (int i = 0; i < numFeature; i++) { + features[i] = randomFrom(GetIndexRequest.DEFAULT_FEATURES); + } + GetIndexRequest request = new GetIndexRequest().addFeatures(features); + assertArrayEquals(features, request.features()); + } + + public void testLocal() { + boolean local = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().local(local); + assertEquals(local, request.local()); + } + + public void testHumanReadable() { + boolean humanReadable = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().humanReadable(humanReadable); + assertEquals(humanReadable, request.humanReadable()); + } + + public void testIncludeDefaults() { + boolean includeDefaults = randomBoolean(); + GetIndexRequest request = new GetIndexRequest().includeDefaults(includeDefaults); + assertEquals(includeDefaults, request.includeDefaults()); + } + + public void testIndicesOptions() { + IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); + GetIndexRequest request = new GetIndexRequest().indicesOptions(indicesOptions); + assertEquals(indicesOptions, request.indicesOptions()); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java new file mode 100644 index 0000000000000..19c25fd11f6ed --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexResponseTests.java @@ -0,0 +1,195 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices; + +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.client.GetAliasesResponseTests; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.RandomCreateIndexGenerator; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class GetIndexResponseTests extends ESTestCase { + + // Because the client-side class does not have a toXContent method, we test xContent serialization by creating + // a random client object, converting it to a server object then serializing it to xContent, and finally + // parsing it back as a client object. We check equality between the original client object, and the parsed one. + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + GetIndexResponseTests::createTestInstance, + GetIndexResponseTests::toXContent, + GetIndexResponse::fromXContent) + .supportsUnknownFields(false) + .assertToXContentEquivalence(false) + .assertEqualsConsumer(GetIndexResponseTests::assertEqualInstances) + .test(); + } + + private static void assertEqualInstances(GetIndexResponse expected, GetIndexResponse actual) { + assertArrayEquals(expected.getIndices(), actual.getIndices()); + assertEquals(expected.getMappings(), actual.getMappings()); + assertEquals(expected.getSettings(), actual.getSettings()); + assertEquals(expected.getDefaultSettings(), actual.getDefaultSettings()); + assertEquals(expected.getAliases(), actual.getAliases()); + } + + private static GetIndexResponse createTestInstance() { + String[] indices = generateRandomStringArray(5, 5, false, false); + Map mappings = new HashMap<>(); + Map> aliases = new HashMap<>(); + Map settings = new HashMap<>(); + Map defaultSettings = new HashMap<>(); + IndexScopedSettings indexScopedSettings = IndexScopedSettings.DEFAULT_SCOPED_SETTINGS; + boolean includeDefaults = randomBoolean(); + for (String index: indices) { + mappings.put(index, createMappingsForIndex()); + + List aliasMetaDataList = new ArrayList<>(); + int aliasesNum = randomIntBetween(0, 3); + for (int i=0; i mappings = new HashMap<>(); + mappings.put("field-" + i, randomFieldMapping()); + if (randomBoolean()) { + mappings.put("field2-" + i, randomFieldMapping()); + } + + try { + String typeName = MapperService.SINGLE_MAPPING_NAME; + mmd = new MappingMetaData(typeName, mappings); + } catch (IOException e) { + fail("shouldn't have failed " + e); + } + } + } + return mmd; + } + + // Not meant to be exhaustive + private static Map randomFieldMapping() { + Map mappings = new HashMap<>(); + if (randomBoolean()) { + mappings.put("type", randomBoolean() ? "text" : "keyword"); + mappings.put("index", "analyzed"); + mappings.put("analyzer", "english"); + } else if (randomBoolean()) { + mappings.put("type", randomFrom("integer", "float", "long", "double")); + mappings.put("index", Objects.toString(randomBoolean())); + } else if (randomBoolean()) { + mappings.put("type", "object"); + mappings.put("dynamic", "strict"); + Map properties = new HashMap<>(); + Map props1 = new HashMap<>(); + props1.put("type", randomFrom("text", "keyword")); + props1.put("analyzer", "keyword"); + properties.put("subtext", props1); + Map props2 = new HashMap<>(); + props2.put("type", "object"); + Map prop2properties = new HashMap<>(); + Map props3 = new HashMap<>(); + props3.put("type", "integer"); + props3.put("index", "false"); + prop2properties.put("subsubfield", props3); + props2.put("properties", prop2properties); + mappings.put("properties", properties); + } else { + mappings.put("type", "keyword"); + } + return mappings; + } + + private static void toXContent(GetIndexResponse response, XContentBuilder builder) throws IOException { + // first we need to repackage from GetIndexResponse to org.elasticsearch.action.admin.indices.get.GetIndexResponse + ImmutableOpenMap.Builder> allMappings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> aliases = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder settings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder defaultSettings = ImmutableOpenMap.builder(); + + Map indexMappings = response.getMappings(); + for (String index : response.getIndices()) { + MappingMetaData mmd = indexMappings.get(index); + ImmutableOpenMap.Builder typedMappings = ImmutableOpenMap.builder(); + if (mmd != null) { + typedMappings.put(MapperService.SINGLE_MAPPING_NAME, mmd); + } + allMappings.put(index, typedMappings.build()); + aliases.put(index, response.getAliases().get(index)); + settings.put(index, response.getSettings().get(index)); + defaultSettings.put(index, response.getDefaultSettings().get(index)); + } + + org.elasticsearch.action.admin.indices.get.GetIndexResponse serverResponse + = new org.elasticsearch.action.admin.indices.get.GetIndexResponse( + response.getIndices(), + allMappings.build(), + aliases.build(), + settings.build(), + defaultSettings.build()); + + // then we can call its toXContent method, forcing no output of types + Params params = new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); + serverResponse.toXContent(builder, params); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java index 43e02f69bfff0..ded9f81dd7c85 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java @@ -60,7 +60,7 @@ public class GetIndexResponse extends ActionResponse implements ToXContentObject private ImmutableOpenMap defaultSettings = ImmutableOpenMap.of(); private String[] indices; - GetIndexResponse(String[] indices, + public GetIndexResponse(String[] indices, ImmutableOpenMap> mappings, ImmutableOpenMap> aliases, ImmutableOpenMap settings, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java index a2a539e8a8d6c..73435dec46c32 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java @@ -46,9 +46,8 @@ */ public class RestGetIndicesAction extends BaseRestHandler { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger( - LogManager.getLogger(RestGetIndicesAction.class)); - static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of get indices requests " + + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestGetIndicesAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of get indices requests " + "will change in 7.0. Please start using the include_type_name parameter set to false to move to the new, " + "typeless response format that will become the default."; diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java index a4360fe941bad..7f61d2961949a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/get/GetIndexResponseTests.java @@ -31,8 +31,10 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.RandomCreateIndexGenerator; +import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import org.junit.Assert; @@ -192,4 +194,13 @@ public void testCanOutput622Response() throws IOException { Assert.assertEquals(TEST_6_3_0_RESPONSE_BYTES, base64OfResponse); } + + /** + * For xContent roundtrip testing we force the xContent output to still contain types because the parser still expects them. + * The new typeless parsing is implemented in the client side GetIndexResponse. + */ + @Override + protected ToXContent.Params getToXContentParams() { + return new ToXContent.MapParams(Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "true")); + } } From aa9816c70befa2997367e42d3a9512f142105f9c Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 5 Feb 2019 17:31:08 -0500 Subject: [PATCH 22/25] Deprecate types in rollover index API (#38389) (#38458) Backport for #38039 Relates to #35190 --- .../elasticsearch/client/IndicesClient.java | 67 ++++++-- .../client/IndicesRequestConverters.java | 24 ++- .../elasticsearch/client/TimedRequest.java | 8 + .../client/indices/CreateIndexRequest.java | 8 +- .../indices/rollover/RolloverRequest.java | 148 ++++++++++++++++++ .../indices/rollover/RolloverResponse.java | 129 +++++++++++++++ .../elasticsearch/client/IndicesClientIT.java | 39 ++++- .../client/IndicesRequestConvertersTests.java | 51 +++++- .../IndicesClientDocumentationIT.java | 15 +- .../indices/RandomCreateIndexGenerator.java | 13 ++ .../rollover/RolloverRequestTests.java | 65 ++++++++ .../rollover/RolloverResponseTests.java | 100 ++++++++++++ .../high-level/indices/rollover.asciidoc | 15 +- .../rest-api-spec/api/indices.rollover.json | 4 + .../test/indices.rollover/40_mapping.yml | 47 ++++++ .../41_mapping_with_types.yml | 47 ++++++ .../admin/indices/rollover/Condition.java | 4 + .../indices/rollover/RolloverRequest.java | 22 ++- .../indices/rollover/RolloverResponse.java | 7 + .../indices/RestRolloverIndexAction.java | 13 +- .../rollover/RolloverRequestTests.java | 9 +- .../rollover/RolloverResponseTests.java | 3 +- .../index/RandomCreateIndexGenerator.java | 2 +- 23 files changed, 784 insertions(+), 56 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 18ecac6a164ac..060953036b8d7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -38,8 +38,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -65,6 +63,8 @@ import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.rest.RestStatus; import java.io.IOException; @@ -1317,17 +1317,54 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, RequestOptions RolloverResponse::fromXContent, emptySet()); } + /** + * Asynchronously rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + RolloverResponse::fromXContent, listener, emptySet()); + } + + /** + * Rolls over an index using the Rollover Index API. + * See + * Rollover Index API on elastic.co + * @param rolloverRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. + */ + @Deprecated + public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet()); + } + /** * Rolls over an index using the Rollover Index API. *

* See * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rollover(RolloverRequest, RequestOptions)} + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rollover(RolloverRequest, RequestOptions)} should be used instead, which accepts a new request object. */ @Deprecated - public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... headers) throws IOException { + public org.elasticsearch.action.admin.indices.rollover.RolloverResponse rollover( + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + Header... headers) throws IOException { return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - RolloverResponse::fromXContent, emptySet(), headers); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, emptySet(), headers); } /** @@ -1337,10 +1374,16 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... head * @param rolloverRequest the request * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized * @param listener the listener to be notified upon request completion + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which + * accepts a new request object. */ - public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions options, ActionListener listener) { + @Deprecated + public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + RequestOptions options, ActionListener listener) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, options, - RolloverResponse::fromXContent, listener, emptySet()); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet()); } /** @@ -1348,12 +1391,16 @@ public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions option *

* See * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} + * + * @deprecated This method uses deprecated request and response objects. + * The method {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} should be used instead, which + * accepts a new request object. */ @Deprecated - public void rolloverAsync(RolloverRequest rolloverRequest, ActionListener listener, Header... headers) { + public void rolloverAsync(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest, + ActionListener listener, Header... headers) { restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, IndicesRequestConverters::rollover, - RolloverResponse::fromXContent, listener, emptySet(), headers); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse::fromXContent, listener, emptySet(), headers); } /** diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index 13fc6a8e08bd3..1ee6bc8d94186 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -35,7 +35,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -53,6 +52,7 @@ import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.Strings; import java.io.IOException; @@ -348,7 +348,7 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { static Request rollover(RolloverRequest rolloverRequest) throws IOException { String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") - .addPathPart(rolloverRequest.getNewIndexName()).build(); + .addPathPart(rolloverRequest.getNewIndexName()).build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); RequestConverters.Params params = new RequestConverters.Params(request); @@ -358,11 +358,31 @@ static Request rollover(RolloverRequest rolloverRequest) throws IOException { if (rolloverRequest.isDryRun()) { params.putParam("dry_run", Boolean.TRUE.toString()); } + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "false"); request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); return request; } + @Deprecated + static Request rollover(org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest) throws IOException { + String endpoint = new RequestConverters.EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover") + .addPathPart(rolloverRequest.getNewIndexName()).build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + + RequestConverters.Params params = new RequestConverters.Params(request); + params.withTimeout(rolloverRequest.timeout()); + params.withMasterTimeout(rolloverRequest.masterNodeTimeout()); + params.withWaitForActiveShards(rolloverRequest.getCreateIndexRequest().waitForActiveShards()); + if (rolloverRequest.isDryRun()) { + params.putParam("dry_run", Boolean.TRUE.toString()); + } + params.putParam(INCLUDE_TYPE_NAME_PARAMETER, "true"); + request.setEntity(RequestConverters.createEntity(rolloverRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + + return request; + } + static Request getSettings(GetSettingsRequest getSettingsRequest) { String[] indices = getSettingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.indices(); String[] names = getSettingsRequest.names() == null ? Strings.EMPTY_ARRAY : getSettingsRequest.names(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java index 60ecea39ae093..d03e183dc1029 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TimedRequest.java @@ -36,11 +36,19 @@ public abstract class TimedRequest implements Validatable { private TimeValue timeout = DEFAULT_ACK_TIMEOUT; private TimeValue masterTimeout = DEFAULT_MASTER_NODE_TIMEOUT; + /** + * Sets the timeout to wait for the all the nodes to acknowledge + * @param timeout timeout as a {@link TimeValue} + */ public void setTimeout(TimeValue timeout) { this.timeout = timeout; } + /** + * Sets the timeout to connect to the master node + * @param masterTimeout timeout as a {@link TimeValue} + */ public void setMasterTimeout(TimeValue masterTimeout) { this.masterTimeout = masterTimeout; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java index f0bff6e6f4307..1a018591dc770 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java @@ -338,10 +338,14 @@ public CreateIndexRequest waitForActiveShards(ActiveShardCount waitForActiveShar return this; } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + innerToXContent(builder, params); + builder.endObject(); + return builder; + } + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(SETTINGS.getPreferredName()); settings.toXContent(builder, params); builder.endObject(); @@ -356,8 +360,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (Alias alias : aliases) { alias.toXContent(builder, params); } - builder.endObject(); - builder.endObject(); return builder; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java new file mode 100644 index 0000000000000..ef78fb7353067 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverRequest.java @@ -0,0 +1,148 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.client.TimedRequest; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Request class to swap index under an alias upon satisfying conditions + */ +public class RolloverRequest extends TimedRequest implements ToXContentObject { + + private final String alias; + private final String newIndexName; + private boolean dryRun; + private final Map> conditions = new HashMap<>(2); + //the index name "_na_" is never read back, what matters are settings, mappings and aliases + private final CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_"); + + public RolloverRequest(String alias, String newIndexName) { + if (alias == null) { + throw new IllegalArgumentException("The index alias cannot be null!"); + } + this.alias = alias; + this.newIndexName = newIndexName; + } + + /** + * Returns the alias of the rollover operation + */ + public String getAlias() { + return alias; + } + + /** + * Returns the new index name for the rollover + */ + public String getNewIndexName() { + return newIndexName; + } + + + /** + * Sets if the rollover should not be executed when conditions are met + */ + public RolloverRequest dryRun(boolean dryRun) { + this.dryRun = dryRun; + return this; + } + /** + * Returns if the rollover should not be executed when conditions are met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Adds condition to check if the index is at least age old + */ + public RolloverRequest addMaxIndexAgeCondition(TimeValue age) { + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(age); + if (this.conditions.containsKey(maxAgeCondition.name())) { + throw new IllegalArgumentException(maxAgeCondition.name() + " condition is already set"); + } + this.conditions.put(maxAgeCondition.name(), maxAgeCondition); + return this; + } + + /** + * Adds condition to check if the index has at least numDocs + */ + public RolloverRequest addMaxIndexDocsCondition(long numDocs) { + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(numDocs); + if (this.conditions.containsKey(maxDocsCondition.name())) { + throw new IllegalArgumentException(maxDocsCondition.name() + " condition is already set"); + } + this.conditions.put(maxDocsCondition.name(), maxDocsCondition); + return this; + } + /** + * Adds a size-based condition to check if the index size is at least size. + */ + public RolloverRequest addMaxIndexSizeCondition(ByteSizeValue size) { + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(size); + if (this.conditions.containsKey(maxSizeCondition.name())) { + throw new IllegalArgumentException(maxSizeCondition + " condition is already set"); + } + this.conditions.put(maxSizeCondition.name(), maxSizeCondition); + return this; + } + /** + * Returns all set conditions + */ + public Map> getConditions() { + return conditions; + } + + /** + * Returns the inner {@link CreateIndexRequest}. Allows to configure mappings, settings and aliases for the new index. + */ + public CreateIndexRequest getCreateIndexRequest() { + return createIndexRequest; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + createIndexRequest.innerToXContent(builder, params); + + builder.startObject("conditions"); + for (Condition condition : conditions.values()) { + condition.toXContent(builder, params); + } + builder.endObject(); + + builder.endObject(); + return builder; + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java new file mode 100644 index 0000000000000..2bcd683d7b1f6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/rollover/RolloverResponse.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + */ +public final class RolloverResponse extends ShardsAcknowledgedResponse { + + private static final ParseField NEW_INDEX = new ParseField("new_index"); + private static final ParseField OLD_INDEX = new ParseField("old_index"); + private static final ParseField DRY_RUN = new ParseField("dry_run"); + private static final ParseField ROLLED_OVER = new ParseField("rolled_over"); + private static final ParseField CONDITIONS = new ParseField("conditions"); + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rollover", + true, args -> new RolloverResponse((String) args[0], (String) args[1], (Map) args[2], + (Boolean)args[3], (Boolean)args[4], (Boolean) args[5], (Boolean) args[6])); + + static { + PARSER.declareString(constructorArg(), OLD_INDEX); + PARSER.declareString(constructorArg(), NEW_INDEX); + PARSER.declareObject(constructorArg(), (parser, context) -> parser.map(), CONDITIONS); + PARSER.declareBoolean(constructorArg(), DRY_RUN); + PARSER.declareBoolean(constructorArg(), ROLLED_OVER); + declareAcknowledgedAndShardsAcknowledgedFields(PARSER); + } + + private final String oldIndex; + private final String newIndex; + private final Map conditionStatus; + private final boolean dryRun; + private final boolean rolledOver; + + public RolloverResponse(String oldIndex, String newIndex, Map conditionResults, + boolean dryRun, boolean rolledOver, boolean acknowledged, boolean shardsAcknowledged) { + super(acknowledged, shardsAcknowledged); + this.oldIndex = oldIndex; + this.newIndex = newIndex; + this.dryRun = dryRun; + this.rolledOver = rolledOver; + this.conditionStatus = conditionResults; + } + + /** + * Returns the name of the index that the request alias was pointing to + */ + public String getOldIndex() { + return oldIndex; + } + + /** + * Returns the name of the index that the request alias currently points to + */ + public String getNewIndex() { + return newIndex; + } + + /** + * Returns the statuses of all the request conditions + */ + public Map getConditionStatus() { + return conditionStatus; + } + + /** + * Returns if the rollover execution was skipped even when conditions were met + */ + public boolean isDryRun() { + return dryRun; + } + + /** + * Returns true if the rollover was not simulated and the conditions were met + */ + public boolean isRolledOver() { + return rolledOver; + } + + public static RolloverResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (super.equals(o)) { + RolloverResponse that = (RolloverResponse) o; + return dryRun == that.dryRun && + rolledOver == that.rolledOver && + Objects.equals(oldIndex, that.oldIndex) && + Objects.equals(newIndex, that.newIndex) && + Objects.equals(conditionStatus, that.conditionStatus); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), oldIndex, newIndex, conditionStatus, dryRun, rolledOver); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 0aed9f3baaba0..81cc12765a7d2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -43,8 +43,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -76,6 +74,8 @@ import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -90,6 +90,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.QueryBuilder; @@ -100,6 +101,7 @@ import org.elasticsearch.rest.action.admin.indices.RestGetIndicesAction; import org.elasticsearch.rest.action.admin.indices.RestGetMappingAction; import org.elasticsearch.rest.action.admin.indices.RestPutMappingAction; +import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import java.io.IOException; import java.util.Arrays; @@ -1146,7 +1148,6 @@ public void testRollover() throws IOException { { RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1166,7 +1167,6 @@ public void testRollover() throws IOException { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1)); rolloverRequest.dryRun(true); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertTrue(rolloverResponse.isDryRun()); @@ -1178,10 +1178,11 @@ public void testRollover() throws IOException { assertEquals("test_new", rolloverResponse.getNewIndex()); } { + String mappings = "{\"properties\":{\"field2\":{\"type\":\"keyword\"}}}"; + rolloverRequest.getCreateIndexRequest().mapping(mappings, XContentType.JSON); rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1195,6 +1196,34 @@ public void testRollover() throws IOException { } } + public void testRolloverWithTypes() throws IOException { + highLevelClient().indices().create(new CreateIndexRequest("test").alias(new Alias("alias")), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test", "_doc", "1").source("field", "value"), RequestOptions.DEFAULT); + highLevelClient().index(new IndexRequest("test", "_doc", "2").source("field", "value") + .setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL), RequestOptions.DEFAULT); + + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest("alias", "test_new"); + rolloverRequest.addMaxIndexDocsCondition(1); + + Settings.Builder settings = Settings.builder().put("number_of_shards", 1); + rolloverRequest.getCreateIndexRequest().mapping("_doc", "field2", "type=keyword") + .settings(settings); + + org.elasticsearch.action.admin.indices.rollover.RolloverResponse rolloverResponse = execute( + rolloverRequest, + highLevelClient().indices()::rollover, + highLevelClient().indices()::rolloverAsync, + expectWarnings(RestRolloverIndexAction.TYPES_DEPRECATION_MESSAGE) + ); + assertTrue(rolloverResponse.isRolledOver()); + assertFalse(rolloverResponse.isDryRun()); + Map conditionStatus = rolloverResponse.getConditionStatus(); + assertTrue(conditionStatus.get("[max_docs: 1]")); + assertEquals("test", rolloverResponse.getOldIndex()); + assertEquals("test_new", rolloverResponse.getNewIndex()); + } + public void testGetAlias() throws IOException { { createIndex("index1", Settings.EMPTY); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java index e4909787146c4..5193a0c9c0e25 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesRequestConvertersTests.java @@ -38,7 +38,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.open.OpenIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; @@ -56,6 +55,7 @@ import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.RandomCreateIndexGenerator; +import org.elasticsearch.client.indices.rollover.RolloverRequest; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -75,7 +75,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomAliases; +import static org.elasticsearch.client.indices.RandomCreateIndexGenerator.randomMapping; import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings; import static org.elasticsearch.index.alias.RandomAliasActionsGenerator.randomAliasAction; import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER; @@ -876,7 +877,7 @@ private void resizeTest(ResizeType resizeType, CheckedFunction expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomTimeout(rolloverRequest, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); + if (ESTestCase.randomBoolean()) { + rolloverRequest.dryRun(ESTestCase.randomBoolean()); + if (rolloverRequest.isDryRun()) { + expectedParams.put("dry_run", "true"); + } + } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "false"); + if (ESTestCase.randomBoolean()) { + rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().mapping(randomMapping()); + } + if (ESTestCase.randomBoolean()) { + randomAliases(rolloverRequest.getCreateIndexRequest()); + } + if (ESTestCase.randomBoolean()) { + rolloverRequest.getCreateIndexRequest().settings( + org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings()); + } + RequestConvertersTests.setRandomWaitForActiveShards(rolloverRequest.getCreateIndexRequest()::waitForActiveShards, expectedParams); + + Request request = IndicesRequestConverters.rollover(rolloverRequest); + if (rolloverRequest.getNewIndexName() == null) { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover", request.getEndpoint()); + } else { + Assert.assertEquals("/" + rolloverRequest.getAlias() + "/_rollover/" + rolloverRequest.getNewIndexName(), + request.getEndpoint()); + } + Assert.assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + RequestConvertersTests.assertToXContentBody(rolloverRequest, request.getEntity()); + Assert.assertEquals(expectedParams, request.getParameters()); + } + + public void testRolloverWithTypes() throws IOException { + org.elasticsearch.action.admin.indices.rollover.RolloverRequest rolloverRequest = + new org.elasticsearch.action.admin.indices.rollover.RolloverRequest(ESTestCase.randomAlphaOfLengthBetween(3, 10), + ESTestCase.randomBoolean() ? null : ESTestCase.randomAlphaOfLengthBetween(3, 10)); + Map expectedParams = new HashMap<>(); RequestConvertersTests.setRandomTimeout(rolloverRequest::timeout, rolloverRequest.timeout(), expectedParams); RequestConvertersTests.setRandomMasterTimeout(rolloverRequest, expectedParams); if (ESTestCase.randomBoolean()) { @@ -903,6 +945,7 @@ public void testRollover() throws IOException { expectedParams.put("dry_run", "true"); } } + expectedParams.put(INCLUDE_TYPE_NAME_PARAMETER, "true"); if (ESTestCase.randomBoolean()) { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(ESTestCase.randomNonNegativeLong())); } @@ -912,7 +955,7 @@ public void testRollover() throws IOException { org.elasticsearch.index.RandomCreateIndexGenerator.randomMapping(type)); } if (ESTestCase.randomBoolean()) { - randomAliases(rolloverRequest.getCreateIndexRequest()); + org.elasticsearch.index.RandomCreateIndexGenerator.randomAliases(rolloverRequest.getCreateIndexRequest()); } if (ESTestCase.randomBoolean()) { rolloverRequest.getCreateIndexRequest().settings( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index 9c6d0d4effe08..4f492129bd00e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -42,8 +42,6 @@ import org.elasticsearch.action.admin.indices.open.OpenIndexResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; @@ -79,6 +77,8 @@ import org.elasticsearch.client.indices.IndexTemplatesExistRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.client.indices.UnfreezeIndexRequest; +import org.elasticsearch.client.indices.rollover.RolloverRequest; +import org.elasticsearch.client.indices.rollover.RolloverResponse; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -1820,18 +1820,16 @@ public void testRolloverIndex() throws Exception { // end::rollover-index-request // tag::rollover-index-request-timeout - request.timeout(TimeValue.timeValueMinutes(2)); // <1> - request.timeout("2m"); // <2> + request.setTimeout(TimeValue.timeValueMinutes(2)); // <1> // end::rollover-index-request-timeout // tag::rollover-index-request-masterTimeout - request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> - request.masterNodeTimeout("1m"); // <2> + request.setMasterTimeout(TimeValue.timeValueMinutes(1)); // <1> // end::rollover-index-request-masterTimeout // tag::rollover-index-request-dryRun request.dryRun(true); // <1> // end::rollover-index-request-dryRun // tag::rollover-index-request-waitForActiveShards - request.getCreateIndexRequest().waitForActiveShards(2); // <1> + request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.from(2)); // <1> request.getCreateIndexRequest().waitForActiveShards(ActiveShardCount.DEFAULT); // <2> // end::rollover-index-request-waitForActiveShards // tag::rollover-index-request-settings @@ -1839,7 +1837,8 @@ public void testRolloverIndex() throws Exception { .put("index.number_of_shards", 4)); // <1> // end::rollover-index-request-settings // tag::rollover-index-request-mapping - request.getCreateIndexRequest().mapping("type", "field", "type=keyword"); // <1> + String mappings = "{\"properties\":{\"field-1\":{\"type\":\"keyword\"}}}"; + request.getCreateIndexRequest().mapping(mappings, XContentType.JSON); // <1> // end::rollover-index-request-mapping // tag::rollover-index-request-alias request.getCreateIndexRequest().alias(new Alias("another_alias")); // <1> diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java index 179b7e728b620..610cc54678ae0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/RandomCreateIndexGenerator.java @@ -24,6 +24,9 @@ import java.io.IOException; +import static org.elasticsearch.index.RandomCreateIndexGenerator.randomAlias; +import static org.elasticsearch.test.ESTestCase.randomIntBetween; + public class RandomCreateIndexGenerator { /** @@ -58,4 +61,14 @@ public static XContentBuilder randomMapping() throws IOException { builder.endObject(); return builder; } + + /** + * Sets random aliases to the provided {@link CreateIndexRequest} + */ + public static void randomAliases(CreateIndexRequest request) { + int aliasesNo = randomIntBetween(0, 2); + for (int i = 0; i < aliasesNo; i++) { + request.alias(randomAlias()); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java new file mode 100644 index 0000000000000..57798c393db8f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverRequestTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; + + +public class RolloverRequestTests extends ESTestCase { + public void testConstructorAndFieldAssignments() { + // test constructor + String alias = randomAlphaOfLength(5); + String newIndexName = null; + if (randomBoolean()) { + newIndexName = randomAlphaOfLength(8); + } + RolloverRequest rolloverRequest = new RolloverRequest(alias, newIndexName); + assertEquals(alias, rolloverRequest.getAlias()); + assertEquals(newIndexName, rolloverRequest.getNewIndexName()); + + // test assignment of conditions + MaxAgeCondition maxAgeCondition = new MaxAgeCondition(new TimeValue(10)); + MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(2000)); + MaxDocsCondition maxDocsCondition = new MaxDocsCondition(10000L); + Condition[] expectedConditions = new Condition[] {maxAgeCondition, maxSizeCondition, maxDocsCondition}; + rolloverRequest.addMaxIndexAgeCondition(maxAgeCondition.value()); + rolloverRequest.addMaxIndexSizeCondition(maxSizeCondition.value()); + rolloverRequest.addMaxIndexDocsCondition(maxDocsCondition.value()); + List> requestConditions = new ArrayList<>(rolloverRequest.getConditions().values()); + assertThat(requestConditions, containsInAnyOrder(expectedConditions)); + } + + public void testValidation() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> + new RolloverRequest(null, null)); + assertEquals("The index alias cannot be null!", exception.getMessage()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java new file mode 100644 index 0000000000000..53fe3bb279e3f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/rollover/RolloverResponseTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.indices.rollover; + +import org.elasticsearch.action.admin.indices.rollover.Condition; +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; +import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.common.xcontent.ToXContent.Params; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.Collections; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class RolloverResponseTests extends ESTestCase { + + private static final List>> conditionSuppliers = new ArrayList<>(); + static { + conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); + conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + } + + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + RolloverResponseTests::createTestInstance, + RolloverResponseTests::toXContent, + RolloverResponse::fromXContent) + .supportsUnknownFields(true) + .randomFieldsExcludeFilter(getRandomFieldsExcludeFilter()) + .test(); + } + + private static RolloverResponse createTestInstance() { + final String oldIndex = randomAlphaOfLength(8); + final String newIndex = randomAlphaOfLength(8); + final boolean dryRun = randomBoolean(); + final boolean rolledOver = randomBoolean(); + final boolean acknowledged = randomBoolean(); + final boolean shardsAcknowledged = acknowledged && randomBoolean(); + + Map results = new HashMap<>(); + int numResults = randomIntBetween(0, 3); + List>> conditions = randomSubsetOf(numResults, conditionSuppliers); + conditions.forEach(condition -> results.put(condition.get().name(), randomBoolean())); + + return new RolloverResponse(oldIndex, newIndex, results, dryRun, rolledOver, acknowledged, shardsAcknowledged); + } + + private Predicate getRandomFieldsExcludeFilter() { + return field -> field.startsWith("conditions"); + } + + private static void toXContent(RolloverResponse response, XContentBuilder builder) throws IOException { + Params params = new ToXContent.MapParams( + Collections.singletonMap(BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER, "false")); + org.elasticsearch.action.admin.indices.rollover.RolloverResponse serverResponse = + new org.elasticsearch.action.admin.indices.rollover.RolloverResponse( + response.getOldIndex(), + response.getNewIndex(), + response.getConditionStatus(), + response.isDryRun(), + response.isRolledOver(), + response.isAcknowledged(), + response.isShardsAcknowledged() + ); + serverResponse.toXContent(builder, params); + } +} diff --git a/docs/java-rest/high-level/indices/rollover.asciidoc b/docs/java-rest/high-level/indices/rollover.asciidoc index c6134cd5579df..6b7a82a11ae2b 100644 --- a/docs/java-rest/high-level/indices/rollover.asciidoc +++ b/docs/java-rest/high-level/indices/rollover.asciidoc @@ -19,7 +19,8 @@ one or more conditions that determine when the index has to be rolled over: include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> The alias (first argument) that points to the index to rollover, and -optionally the name of the new index in case the rollover operation is performed +the name of the new index in case the rollover operation is performed. +The new index argument is optional, and can be set to null <2> Condition on the age of the index <3> Condition on the number of documents in the index <4> Condition on the size of the index @@ -39,24 +40,20 @@ include-tagged::{doc-tests-file}[{api}-request-timeout] -------------------------------------------------- <1> Timeout to wait for the all the nodes to acknowledge the index is opened as a `TimeValue` -<2> Timeout to wait for the all the nodes to acknowledge the index is opened -as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-masterTimeout] -------------------------------------------------- <1> Timeout to connect to the master node as a `TimeValue` -<2> Timeout to connect to the master node as a `String` ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-waitForActiveShards] -------------------------------------------------- -<1> The number of active shard copies to wait for before the rollover index API -returns a response, as an `int` -<2> The number of active shard copies to wait for before the rollover index API -returns a response, as an `ActiveShardCount` +<1> Sets the number of active shard copies to wait for before the rollover +index API returns a response +<2> Resets the number of active shard copies to wait for to the default value ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -98,5 +95,3 @@ each shard in the index before timing out <5> Whether the index has been rolled over <6> Whether the operation was performed or it was a dry run <7> The different conditions and whether they were matched or not - - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 5e5ba1367ad3e..7bf1513969fb3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -18,6 +18,10 @@ } }, "params": { + "include_type_name": { + "type" : "boolean", + "description" : "Whether a type should be included in the body of the mappings." + }, "timeout": { "type" : "time", "description" : "Explicit operation timeout" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml new file mode 100644 index 0000000000000..7ed78c6e3159a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml @@ -0,0 +1,47 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.6.99" + reason: include_type_name was introduced in 6.7.0 + + - do: + indices.create: + include_type_name: false + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + include_type_name: false + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml new file mode 100644 index 0000000000000..c5d0bd518937e --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/41_mapping_with_types.yml @@ -0,0 +1,47 @@ +--- +"Typeless mapping": + - skip: + version: " - 6.6.99" + reason: include_type_name was introduced in 6.7.0 + + - do: + indices.create: + index: logs-1 + body: + aliases: + logs_search: {} + + # index first document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "1" + body: { "foo": "hello world" } + refresh: true + + # index second document and wait for refresh + - do: + index: + index: logs-1 + type: _doc + id: "2" + body: { "foo": "hello world" } + refresh: true + + # perform alias rollover with new typeless mapping + - do: + indices.rollover: + include_type_name: true + alias: "logs_search" + body: + conditions: + max_docs: 2 + mappings: + _doc: + properties: + foo2: + type: keyword + + - match: { conditions: { "[max_docs: 2]": true } } + - match: { rolled_over: true } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java index 1b385ed9d0dbc..a2cad7a35a398 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java @@ -75,6 +75,10 @@ public T value() { return value; } + public String name() { + return name; + } + /** * Holder for index stats used to evaluate conditions */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java index f36636594a4d6..49db34aa5386e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.util.HashMap; @@ -41,10 +42,13 @@ /** * Request class to swap index under an alias upon satisfying conditions + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. */ public class RolloverRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>("rollover"); + private static final ObjectParser PARSER = new ObjectParser<>("rollover"); private static final ObjectParser, Void> CONDITION_PARSER = new ObjectParser<>("conditions"); private static final ParseField CONDITIONS = new ParseField("conditions"); @@ -66,9 +70,14 @@ public class RolloverRequest extends AcknowledgedRequest implem CONDITIONS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.settings(parser.map()), CreateIndexRequest.SETTINGS, ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, context) -> { - for (Map.Entry mappingsEntry : parser.map().entrySet()) { - request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + PARSER.declareField((parser, request, isTypeIncluded) -> { + if (isTypeIncluded) { + for (Map.Entry mappingsEntry : parser.map().entrySet()) { + request.createIndexRequest.mapping(mappingsEntry.getKey(), (Map) mappingsEntry.getValue()); + } + } else { + // a type is not included, add a dummy _doc type + request.createIndexRequest.mapping(MapperService.SINGLE_MAPPING_NAME, parser.map()); } }, CreateIndexRequest.MAPPINGS, ObjectParser.ValueType.OBJECT); PARSER.declareField((parser, request, context) -> request.createIndexRequest.aliases(parser.map()), @@ -230,7 +239,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public void fromXContent(XContentParser parser) throws IOException { - PARSER.parse(parser, this, null); + // param isTypeIncluded decides how mappings should be parsed from XContent + public void fromXContent(boolean isTypeIncluded, XContentParser parser) throws IOException { + PARSER.parse(parser, this, isTypeIncluded); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java index 356f805c24bd4..78bdde7b7bd18 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java @@ -37,6 +37,13 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * Response object for {@link RolloverRequest} API + * + * Note: there is a new class with the same name for the Java HLRC that uses a typeless format. + * Any changes done to this class should also go to that client class. + */ public final class RolloverResponse extends ShardsAcknowledgedResponse implements ToXContentObject { private static final ParseField NEW_INDEX = new ParseField("new_index"); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index 489001bf2a14f..a7e0b3b22d6f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -19,9 +19,11 @@ package org.elasticsearch.rest.action.admin.indices; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestController; @@ -31,6 +33,11 @@ import java.io.IOException; public class RestRolloverIndexAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger( + LogManager.getLogger(RestRolloverIndexAction.class)); + public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] The response format of rollover " + + "index requests will change in 7.0. Please start using the include_type_name parameter set to false " + + "to move to the new, typeless response format that will become the default."; public RestRolloverIndexAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this); @@ -44,8 +51,12 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, DEFAULT_INCLUDE_TYPE_NAME_POLICY); + if (includeTypeName) { + deprecationLogger.deprecatedAndMaybeLog("index_rollover_with_types", TYPES_DEPRECATION_MESSAGE); + } RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index")); - request.applyContentParser(rolloverIndexRequest::fromXContent); + request.applyContentParser(parser -> rolloverIndexRequest.fromXContent(includeTypeName, parser)); rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false)); rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout())); rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java index 1e8d8e2a2932c..e4ccb70f4862a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java @@ -52,7 +52,6 @@ import static org.hamcrest.Matchers.equalTo; public class RolloverRequestTests extends ESTestCase { - private NamedWriteableRegistry writeableRegistry; @Override @@ -72,7 +71,7 @@ public void testConditionsParsing() throws Exception { .field("max_size", "45gb") .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(false, createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(3)); MaxAgeCondition maxAgeCondition = (MaxAgeCondition)conditions.get(MaxAgeCondition.NAME); @@ -108,7 +107,7 @@ public void testParsingWithIndexSettings() throws Exception { .startObject("alias1").endObject() .endObject() .endObject(); - request.fromXContent(createParser(builder)); + request.fromXContent(true, createParser(builder)); Map conditions = request.getConditions(); assertThat(conditions.size(), equalTo(2)); assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1)); @@ -147,7 +146,7 @@ public void testToAndFromXContent() throws IOException { BytesReference originalBytes = toShuffledXContent(rolloverRequest, xContentType, EMPTY_PARAMS, humanReadable); RolloverRequest parsedRolloverRequest = new RolloverRequest(); - parsedRolloverRequest.fromXContent(createParser(xContentType.xContent(), originalBytes)); + parsedRolloverRequest.fromXContent(true, createParser(xContentType.xContent(), originalBytes)); CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest(); CreateIndexRequest parsedCreateIndexRequest = parsedRolloverRequest.getCreateIndexRequest(); @@ -172,7 +171,7 @@ public void testUnknownFields() throws IOException { } builder.endObject(); BytesReference mutated = XContentTestUtils.insertRandomFields(xContentType, BytesReference.bytes(builder), null, random()); - expectThrows(XContentParseException.class, () -> request.fromXContent(createParser(xContentType.xContent(), mutated))); + expectThrows(XContentParseException.class, () -> request.fromXContent(false, createParser(xContentType.xContent(), mutated))); } public void testSameConditionCanOnlyBeAddedOnce() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java index e0f58ecd92daf..05eaefffaebf0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.rollover; import org.elasticsearch.Version; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractStreamableXContentTestCase; @@ -57,7 +58,7 @@ private static Map randomResults(boolean allowNoItems) { private static final List>> conditionSuppliers = new ArrayList<>(); static { conditionSuppliers.add(() -> new MaxAgeCondition(new TimeValue(randomNonNegativeLong()))); - conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); + conditionSuppliers.add(() -> new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()))); conditionSuppliers.add(() -> new MaxDocsCondition(randomNonNegativeLong())); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java index 345ef1f58bcac..9732504cac6d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/index/RandomCreateIndexGenerator.java @@ -126,7 +126,7 @@ public static void randomAliases(CreateIndexRequest request) { } } - private static Alias randomAlias() { + public static Alias randomAlias() { Alias alias = new Alias(randomAlphaOfLength(5)); if (randomBoolean()) { From 9507d94da5d7892da41bc5bae655317c444e6f57 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Wed, 6 Feb 2019 00:35:06 +0100 Subject: [PATCH 23/25] Deprecate support for internal versioning for concurrency control (#38451) Relates #38254 Relates #10708 --- .../java/org/elasticsearch/client/CrudIT.java | 21 ++++++++++-- .../documentation/CRUDDocumentationIT.java | 11 ++++--- docs/reference/docs/index_.asciidoc | 2 ++ docs/reference/docs/update.asciidoc | 2 ++ docs/reference/migration/migrate_6_7.asciidoc | 14 ++++++++ .../rest-api-spec/test/bulk/60_deprecated.yml | 6 ++-- .../test/delete/20_internal_version.yml | 8 +++++ .../rest-api-spec/test/delete/21_cas.yml | 33 +++++++++++++++++++ .../test/index/30_internal_version.yml | 10 ++++++ .../test/update/30_internal_version.yml | 8 +++++ .../elasticsearch/action/DocWriteRequest.java | 12 +++++++ .../action/delete/DeleteRequest.java | 5 +++ .../action/index/IndexRequest.java | 5 +++ .../action/update/UpdateRequest.java | 3 ++ .../action/bulk/BulkRequestTests.java | 2 ++ .../action/update/UpdateRequestTests.java | 2 ++ .../xpack/watcher/PutWatchRequest.java | 10 ++++++ .../80_put_get_watch_with_passwords.yml | 5 ++- 18 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 7baa4d0fbfc55..770c8831eec30 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -105,7 +105,8 @@ public void testDelete() throws IOException { IndexResponse indexResponse = highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); - if (randomBoolean()) { + final boolean useSeqNo = randomBoolean(); + if (useSeqNo) { deleteRequest.setIfSeqNo(indexResponse.getSeqNo()); deleteRequest.setIfPrimaryTerm(indexResponse.getPrimaryTerm()); } else { @@ -117,6 +118,11 @@ public void testDelete() throws IOException { assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); + if (useSeqNo == false) { + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed." + + " Please use the `if_seq_no` and `if_primary_term` parameters instead." + + " (request for index [index], type [type], id [id])"); + } } { // Testing non existing document @@ -153,6 +159,9 @@ public void testDelete() throws IOException { } else { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed." + + " Please use the `if_seq_no` and `if_primary_term` parameters instead." + + " (request for index [index], type [type], id [version_conflict])"); } assertEquals("index", exception.getMetadata("es.index").get(0)); } @@ -430,6 +439,9 @@ public void testMultiGet() throws IOException { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/38451 contains a fix. sliencing for now") public void testIndex() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); + highLevelClient().indices().create( + new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), + RequestOptions.DEFAULT); { IndexRequest indexRequest = new IndexRequest("index", "type"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("test", "test").endObject()); @@ -480,7 +492,7 @@ public void testIndex() throws IOException { IndexRequest wrongRequest = new IndexRequest("index", "type", "id"); wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); if (seqNosForConflict) { - wrongRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + wrongRequest.setIfSeqNo(5).setIfPrimaryTerm(2); } else { wrongRequest.version(5); } @@ -491,11 +503,14 @@ public void testIndex() throws IOException { assertEquals(RestStatus.CONFLICT, exception.status()); if (seqNosForConflict) { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + - "version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]", + "version conflict, required seqNo [5], primary term [2]. current document has seqNo [2] and primary term [1]]", exception.getMessage()); } else { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. " + + "(request for index [index], type [type], id [id])"); } assertEquals("index", exception.getMetadata("es.index").get(0)); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 426e9bd397cc4..8b119915ae165 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -224,7 +224,8 @@ public void testIndex() throws Exception { // tag::index-conflict IndexRequest request = new IndexRequest("posts", "doc", "1") .source("field", "value") - .version(1); + .setIfSeqNo(100L) + .setIfPrimaryTerm(1L); try { IndexResponse response = client.index(request, RequestOptions.DEFAULT); } catch(ElasticsearchException e) { @@ -434,7 +435,8 @@ public void testUpdate() throws Exception { // tag::update-conflict UpdateRequest request = new UpdateRequest("posts", "doc", "1") .doc("field", "value") - .version(1); + .setIfSeqNo(100L) + .setIfPrimaryTerm(1L); try { UpdateResponse updateResponse = client.update( request, RequestOptions.DEFAULT); @@ -639,8 +641,9 @@ public void testDelete() throws Exception { // tag::delete-conflict try { DeleteResponse deleteResponse = client.delete( - new DeleteRequest("posts", "doc", "1").version(2), - RequestOptions.DEFAULT); + new DeleteRequest("posts", "doc", "1") + .setIfSeqNo(100L).setIfPrimaryTerm(2L), + RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.CONFLICT) { // <1> diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 95a6538c7b104..87d1ce62c3bdb 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -389,6 +389,8 @@ the different version types and their semantics. `internal`:: only index the document if the given version is identical to the version of the stored document. +deprecated[6.7.0, Please use `if_seq_no` & `if_primary_term` instead. See <> for more details.] + `external` or `external_gt`:: only index the document if the given version is strictly higher than the version of the stored document *or* if there is no existing document. The given diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 42840b1b0a5ec..f73d3ad71e7a1 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -338,6 +338,8 @@ The update API uses the Elasticsearch's versioning support internally to make sure the document doesn't change during the update. You can use the `version` parameter to specify that the document should only be updated if its version matches the one specified. +deprecated[6.7.0, Please use `if_seq_no` & `if_primary_term` instead. See <> for more details.] + [NOTE] .The update API does not support versioning other than internal diff --git a/docs/reference/migration/migrate_6_7.asciidoc b/docs/reference/migration/migrate_6_7.asciidoc index 194c1d6f20970..13b7d4b2cbc91 100644 --- a/docs/reference/migration/migrate_6_7.asciidoc +++ b/docs/reference/migration/migrate_6_7.asciidoc @@ -7,11 +7,25 @@ This section discusses the changes that you need to be aware of when migrating your application to Elasticsearch 6.7. +* <> * <> * <> See also <> and <>. +[float] +[[breaking_67_indexing_changes]] +=== Indexing changes + +[float] +==== Deprecated usage of `internal` versioning for optimistic concurrency control + +`internal` version may not uniquely identify a document's version if an indexed document +wasn't fully replicated when a primary fails. As such it is unsafe to use for +optimistic concurrency control, is deprecated and the option will no longer be available +in Elasticsearch 7.0.0. Please use the `if_seq_no` and `if_primary_term` parameters instead. +See <> for more details. + [float] [[breaking_67_plugin_changes]] === Plugin changes diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml index 17418893d6892..06d1dd1ceb92b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml @@ -3,8 +3,8 @@ "Deprecated parameters should produce warning in Bulk query": - skip: - version: " - 6.0.99" - reason: some parameters are deprecated starting from 6.1, their equivalents without underscore are used instead + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 features: "warnings" - do: @@ -16,6 +16,8 @@ { "doc": { "f1": "v2" } } warnings: - "Deprecated field [_version] used, expected [version] instead" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_1])" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_2])" - do: bulk: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml index 3d9ddb79366f7..a36dc52880af1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml @@ -1,5 +1,9 @@ --- "Internal version": + - skip: + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 + features: warnings - do: index: @@ -17,6 +21,8 @@ type: test id: 1 version: 2 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - do: delete: @@ -24,5 +30,7 @@ type: test id: 1 version: 1 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - match: { _version: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml new file mode 100644 index 0000000000000..3baa81090b0a0 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml @@ -0,0 +1,33 @@ +--- +"Compare and set with sequence numbers": + - skip: + version: " - 6.5.99" + reason: sequence numbers can be used for CAS as of 6.6.0 + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - match: { _seq_no: 0 } + + - do: + catch: conflict + delete: + index: test_1 + type: test + id: 1 + if_seq_no: 2 + if_primary_term: 1 + + - do: + delete: + index: test_1 + type: test + id: 1 + if_seq_no: 0 + if_primary_term: 1 + + - match: { _seq_no: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml index 1767fbebbf966..650f0bd6fa573 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml @@ -1,5 +1,9 @@ --- "Internal version": + - skip: + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 + features: warnings - do: index: @@ -15,6 +19,7 @@ type: test id: 1 body: { foo: bar } + - match: { _version: 2} - do: @@ -25,6 +30,9 @@ id: 1 body: { foo: bar } version: 1 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" + - do: index: index: test_1 @@ -32,5 +40,7 @@ id: 1 body: { foo: bar } version: 2 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - match: { _version: 3 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml index 17c4806c693ac..4ea5bb0e9cd90 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml @@ -1,5 +1,9 @@ --- "Internal version": + - skip: + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 + features: warnings - do: catch: missing @@ -10,6 +14,8 @@ version: 1 body: doc: { foo: baz } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - do: index: @@ -28,3 +34,5 @@ version: 2 body: doc: { foo: baz } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index fc5abf41bdd0d..807ed1a387009 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; @@ -252,6 +253,17 @@ static void writeDocumentRequest(StreamOutput out, DocWriteRequest request) thr } } + static void logDeprecationWarnings(DocWriteRequest request, DeprecationLogger logger) { + if (request.versionType() == VersionType.INTERNAL && + request.version() != Versions.MATCH_ANY && + request.version() != Versions.MATCH_DELETED) { + logger.deprecatedAndMaybeLog("occ_internal_version", + "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use" + + " the `if_seq_no` and `if_primary_term` parameters instead. (request for index [{}], type [{}], id [{}])", + request.index(), request.type(), request.id()); + } + } + static ActionRequestValidationException validateSeqNoBasedCASParams( DocWriteRequest request, ActionRequestValidationException validationException) { if (request.versionType().validateVersionForWrites(request.version()) == false) { diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 7a03dc6a35d3d..9d31e1fc831a4 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.delete; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; @@ -28,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.shard.ShardId; @@ -51,6 +53,7 @@ */ public class DeleteRequest extends ReplicatedWriteRequest implements DocWriteRequest, CompositeIndicesRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(DeleteRequest.class)); private String type; private String id; @@ -99,6 +102,8 @@ public ActionRequestValidationException validate() { validationException = DocWriteRequest.validateSeqNoBasedCASParams(this, validationException); + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index fa765b2d49b56..ba715102cf001 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.index; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -36,6 +37,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -73,6 +75,7 @@ * @see org.elasticsearch.client.Client#index(IndexRequest) */ public class IndexRequest extends ReplicatedWriteRequest implements DocWriteRequest, CompositeIndicesRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(IndexRequest.class)); /** * Max length of the source document to include into string() @@ -198,6 +201,8 @@ public ActionRequestValidationException validate() { } + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 325f52ab51e8f..bd47b89b52f12 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -160,6 +160,9 @@ public ActionRequestValidationException validate() { if (doc == null && docAsUpsert) { validationException = addValidationError("doc must be specified if doc_as_upsert is enabled", validationException); } + + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + return validationException; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index dc1532b8301eb..2bb64d62e739c 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -316,6 +316,8 @@ public void testToValidateUpsertRequestAndVersionInBulkRequest() throws IOExcept bulkRequest.add(data, null, null, xContentType); assertThat(bulkRequest.validate().validationErrors(), contains("can't provide both upsert request and a version", "can't provide version in upsert request")); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [index], type [type], id [id])"); } public void testBulkTerminatedByNewline() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index b7a1084683da6..133e20ae824c6 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -557,6 +557,8 @@ public void testToValidateUpsertRequestAndVersion() { updateRequest.doc("{}", XContentType.JSON); updateRequest.upsert(new IndexRequest("index","type", "id")); assertThat(updateRequest.validate().validationErrors(), contains("can't provide both upsert request and a version")); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [index], type [type], id [id])"); } public void testToValidateUpsertRequestWithVersion() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index c64629dd35af2..6509bfb03cd27 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -12,6 +13,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentHelper; @@ -31,6 +33,8 @@ */ public class PutWatchRequest extends MasterNodeRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(PutWatchRequest.class)); + private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(10); private static final Pattern NO_WS_PATTERN = Pattern.compile("\\S+"); @@ -180,6 +184,12 @@ public ActionRequestValidationException validate() { if (ifSeqNo != UNASSIGNED_SEQ_NO && version != Versions.MATCH_ANY) { validationException = addValidationError("compare and write operations can not use versioning", validationException); } + + if (version != Versions.MATCH_ANY) { + DEPRECATION_LOGGER.deprecated( + "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use" + + " the `if_seq_no` and `if_primary_term` parameters instead."); + } if (ifPrimaryTerm == UNASSIGNED_PRIMARY_TERM && ifSeqNo != UNASSIGNED_SEQ_NO) { validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml index 357425f547480..a4b1a934d510a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml @@ -392,7 +392,8 @@ setup: --- "Test putting a watch with a redacted password with current version works": - + - skip: + features: "warnings" - do: xpack.watcher.put_watch: id: "my_watch_with_version" @@ -462,6 +463,8 @@ setup: } } } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead." - match: { _id: "my_watch_with_version" } - match: { _version: 2 } From 7b743a0f4ebf6fd2747e1e347c69fabfd9e7febc Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Wed, 6 Feb 2019 11:48:36 +1100 Subject: [PATCH 24/25] Add support for API keys to access Elasticsearch (#38291) (#38399) X-Pack security supports built-in authentication service `token-service` that allows access tokens to be used to access Elasticsearch without using Basic authentication. The tokens are generated by `token-service` based on OAuth2 spec. The access token is a short-lived token (defaults to 20m) and refresh token with a lifetime of 24 hours, making them unsuitable for long-lived or recurring tasks where the system might go offline thereby failing refresh of tokens. This commit introduces a built-in authentication service `api-key-service` that adds support for long-lived tokens aka API keys to access Elasticsearch. The `api-key-service` is consulted after `token-service` in the authentication chain. By default, if TLS is enabled then `api-key-service` is also enabled. The service can be disabled using the configuration setting. The API keys:- - by default do not have an expiration but expiration can be configured where the API keys need to be expired after a certain amount of time. - when generated will keep authentication information of the user that generated them. - can be defined with a role describing the privileges for accessing Elasticsearch and will be limited by the role of the user that generated them - can be invalidated via invalidation API - information can be retrieved via a get API - that have been expired or invalidated will be retained for 1 week before being deleted. The expired API keys remover task handles this. Following are the API key management APIs:- 1. Create API Key - `PUT/POST /_security/api_key` 2. Get API key(s) - `GET /_security/api_key` 3. Invalidate API Key(s) `DELETE /_security/api_key` The API keys can be used to access Elasticsearch using `Authorization` header, where the auth scheme is `ApiKey` and the credentials, is the base64 encoding of API key Id and API key separated by a colon. Example:- ``` curl -H "Authorization: ApiKey YXBpLWtleS1pZDphcGkta2V5" http://localhost:9200/_cluster/health ``` Closes #34383 --- client/rest-high-level/build.gradle | 1 + .../elasticsearch/client/SecurityClient.java | 97 ++ .../client/SecurityRequestConverters.java | 35 + .../client/security/CreateApiKeyRequest.java | 128 +++ .../client/security/CreateApiKeyResponse.java | 105 +++ .../client/security/GetApiKeyRequest.java | 133 +++ .../client/security/GetApiKeyResponse.java | 91 ++ .../security/InvalidateApiKeyRequest.java | 145 +++ .../security/InvalidateApiKeyResponse.java | 121 +++ .../client/security/support/ApiKey.java | 152 ++++ .../SecurityRequestConvertersTests.java | 54 +- .../SecurityDocumentationIT.java | 401 ++++++++- .../security/CreateApiKeyRequestTests.java | 105 +++ .../security/CreateApiKeyResponseTests.java | 101 +++ .../security/GetApiKeyRequestTests.java | 72 ++ .../security/GetApiKeyResponseTests.java | 100 ++ .../InvalidateApiKeyRequestTests.java | 73 ++ .../InvalidateApiKeyResponseTests.java | 111 +++ .../security/create-api-key.asciidoc | 40 + .../high-level/security/get-api-key.asciidoc | 67 ++ .../security/invalidate-api-key.asciidoc | 75 ++ .../high-level/supported-apis.asciidoc | 6 + .../rest-api-spec/test/README.asciidoc | 25 + .../common/RandomBasedUUIDGenerator.java | 34 +- .../java/org/elasticsearch/common/UUIDs.java | 7 + .../common/io/stream/StreamInput.java | 17 + .../common/io/stream/StreamOutput.java | 21 + .../elasticsearch/common/util/set/Sets.java | 15 + .../common/io/stream/StreamTests.java | 32 + .../common/util/set/SetsTests.java | 12 + .../test/rest/yaml/Features.java | 4 +- .../rest/yaml/section/ExecutableSection.java | 1 + .../yaml/section/TransformAndSetSection.java | 106 +++ .../section/TransformAndSetSectionTests.java | 96 ++ x-pack/docs/build.gradle | 1 + x-pack/docs/en/rest-api/security.asciidoc | 14 + .../security/create-api-keys.asciidoc | 99 ++ .../rest-api/security/get-api-keys.asciidoc | 118 +++ .../security/invalidate-api-keys.asciidoc | 140 +++ x-pack/plugin/build.gradle | 1 + .../xpack/ccr/CcrLicenseChecker.java | 7 +- .../xpack/core/XPackClientPlugin.java | 6 + .../xpack/core/XPackSettings.java | 7 +- .../xpack/core/security/SecurityContext.java | 11 +- .../xpack/core/security/action/ApiKey.java | 165 ++++ .../security/action/CreateApiKeyAction.java | 39 + .../security/action/CreateApiKeyRequest.java | 136 +++ .../action/CreateApiKeyRequestBuilder.java | 85 ++ .../security/action/CreateApiKeyResponse.java | 168 ++++ .../core/security/action/GetApiKeyAction.java | 39 + .../security/action/GetApiKeyRequest.java | 150 +++ .../action/GetApiKeyRequestBuilder.java | 21 + .../security/action/GetApiKeyResponse.java | 88 ++ .../action/InvalidateApiKeyAction.java | 40 + .../action/InvalidateApiKeyRequest.java | 150 +++ .../InvalidateApiKeyRequestBuilder.java | 22 + .../action/InvalidateApiKeyResponse.java | 141 +++ .../action/role/GetRolesResponse.java | 4 +- .../action/user/HasPrivilegesResponse.java | 61 +- .../core/security/authc/Authentication.java | 62 +- .../DefaultAuthenticationFailureHandler.java | 6 +- .../core/security/authz/RoleDescriptor.java | 46 +- .../accesscontrol/IndicesAccessControl.java | 72 +- .../SecurityIndexSearcherWrapper.java | 179 +--- .../permission/ApplicationPermission.java | 36 + .../authz/permission/ClusterPermission.java | 10 + .../authz/permission/DocumentPermissions.java | 262 ++++++ .../authz/permission/FieldPermissions.java | 39 +- .../authz/permission/IndicesPermission.java | 48 +- .../authz/permission/LimitedRole.java | 152 ++++ .../authz/permission/ResourcePrivileges.java | 93 ++ .../permission/ResourcePrivilegesMap.java | 121 +++ .../core/security/authz/permission/Role.java | 81 +- .../SecurityQueryTemplateEvaluator.java | 92 ++ .../core/security/client/SecurityClient.java | 31 + .../core/security/support/Automatons.java | 6 + .../resources/security-index-template.json | 34 + .../CreateApiKeyRequestBuilderTests.java | 62 ++ .../action/CreateApiKeyRequestTests.java | 113 +++ .../action/CreateApiKeyResponseTests.java | 81 ++ .../action/GetApiKeyRequestTests.java | 103 +++ .../action/GetApiKeyResponseTests.java | 64 ++ .../action/InvalidateApiKeyRequestTests.java | 104 +++ .../action/InvalidateApiKeyResponseTests.java | 88 ++ .../user/HasPrivilegesResponseTests.java | 36 +- ...aultAuthenticationFailureHandlerTests.java | 5 +- ...yIndexSearcherWrapperIntegrationTests.java | 131 ++- ...SecurityIndexSearcherWrapperUnitTests.java | 152 +--- .../permission/DocumentPermissionsTests.java | 123 +++ .../permission/FieldPermissionsTests.java | 81 ++ .../authz/permission/LimitedRoleTests.java | 403 +++++++++ .../ResourcePrivilegesMapTests.java | 91 ++ .../permission/ResourcePrivilegesTests.java | 70 ++ .../SecurityQueryTemplateEvaluatorTests.java | 94 ++ .../ml/action/TransportPutDatafeedAction.java | 3 +- .../security/ApiKeySSLBootstrapCheck.java | 37 + .../xpack/security/Security.java | 57 +- .../action/TransportCreateApiKeyAction.java | 51 ++ .../action/TransportGetApiKeyAction.java | 48 + .../TransportInvalidateApiKeyAction.java | 46 + .../BulkShardRequestInterceptor.java | 2 +- ...cumentLevelSecurityRequestInterceptor.java | 2 +- .../IndicesAliasesRequestInterceptor.java | 2 +- .../interceptor/ResizeRequestInterceptor.java | 2 +- .../TransportGetUserPrivilegesAction.java | 5 +- .../user/TransportHasPrivilegesAction.java | 124 +-- .../xpack/security/authc/ApiKeyService.java | 851 ++++++++++++++++++ .../security/authc/AuthenticationService.java | 40 +- .../security/authc/ExpiredApiKeysRemover.java | 116 +++ .../xpack/security/authc/TokenService.java | 9 +- .../security/authz/AuthorizationService.java | 59 +- .../security/authz/AuthorizationUtils.java | 5 +- .../security/authz/AuthorizedIndices.java | 2 +- .../authz/store/CompositeRolesStore.java | 144 ++- .../rest/action/RestCreateApiKeyAction.java | 56 ++ .../rest/action/RestGetApiKeyAction.java | 63 ++ .../action/RestInvalidateApiKeyAction.java | 70 ++ .../ApiKeySSLBootstrapCheckTests.java | 32 + .../xpack/security/SecurityContextTests.java | 7 + .../xpack/security/SecurityTests.java | 9 +- .../security/TokenSSLBootsrapCheckTests.java | 9 +- .../filter/SecurityActionFilterTests.java | 16 +- ...IndicesAliasesRequestInterceptorTests.java | 4 +- .../ResizeRequestInterceptorTests.java | 4 +- .../TransportHasPrivilegesActionTests.java | 142 ++- .../security/authc/ApiKeyIntegTests.java | 494 ++++++++++ .../security/authc/ApiKeyServiceTests.java | 295 ++++++ .../authc/AuthenticationServiceTests.java | 166 +++- .../security/authc/TokenServiceTests.java | 20 +- .../authz/AuthorizationServiceTests.java | 213 +++-- .../authz/IndicesAndAliasesResolverTests.java | 13 +- .../security/authz/RoleDescriptorTests.java | 8 +- .../IndicesAccessControlTests.java | 101 +++ .../accesscontrol/IndicesPermissionTests.java | 53 +- .../accesscontrol/OptOutQueryCacheTests.java | 28 +- .../authz/store/CompositeRolesStoreTests.java | 74 +- .../action/RestCreateApiKeyActionTests.java | 116 +++ .../rest/action/RestGetApiKeyActionTests.java | 142 +++ .../RestInvalidateApiKeyActionTests.java | 126 +++ .../transport/ServerTransportFilterTests.java | 20 +- .../api/security.create_api_key.json | 22 + .../api/security.get_api_key.json | 30 + .../api/security.invalidate_api_key.json | 15 + .../rest-api-spec/test/api_key/10_basic.yml | 290 ++++++ 144 files changed, 10792 insertions(+), 918 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java create mode 100644 docs/java-rest/high-level/security/create-api-key.asciidoc create mode 100644 docs/java-rest/high-level/security/get-api-key.asciidoc create mode 100644 docs/java-rest/high-level/security/invalidate-api-key.asciidoc create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java create mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java create mode 100644 x-pack/docs/en/rest-api/security/create-api-keys.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/get-api-keys.asciidoc create mode 100644 x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestBuilder.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestBuilder.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissionsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissionsTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMapTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ApiKeySSLBootstrapCheck.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportCreateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportGetApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/TransportInvalidateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredApiKeysRemover.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestCreateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestGetApiKeyAction.java create mode 100644 x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/RestInvalidateApiKeyAction.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ApiKeySSLBootstrapCheckTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestCreateApiKeyActionTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestGetApiKeyActionTests.java create mode 100644 x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/RestInvalidateApiKeyActionTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.create_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.get_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/security.invalidate_api_key.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/api_key/10_basic.yml diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 07156c3ef3e09..57af034610e13 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -104,6 +104,7 @@ integTestCluster { setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt' setting 'xpack.security.http.ssl.supported_protocols', 'TLSv1.2,TLSv1.1,TLSv1' diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java index 4d8d1d5db43aa..f66826b3c5b3b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java @@ -27,6 +27,8 @@ import org.elasticsearch.client.security.ClearRealmCacheResponse; import org.elasticsearch.client.security.ClearRolesCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheResponse; +import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyResponse; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.CreateTokenResponse; import org.elasticsearch.client.security.DeletePrivilegesRequest; @@ -40,6 +42,8 @@ import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EmptyResponse; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; +import org.elasticsearch.client.security.GetApiKeyResponse; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetPrivilegesResponse; import org.elasticsearch.client.security.GetRoleMappingsRequest; @@ -54,6 +58,8 @@ import org.elasticsearch.client.security.GetUsersResponse; import org.elasticsearch.client.security.HasPrivilegesRequest; import org.elasticsearch.client.security.HasPrivilegesResponse; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; +import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.InvalidateTokenResponse; import org.elasticsearch.client.security.PutPrivilegesRequest; @@ -850,4 +856,95 @@ public void deletePrivilegesAsync(DeletePrivilegesRequest request, RequestOption restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::deletePrivileges, options, DeletePrivilegesResponse::fromXContent, listener, singleton(404)); } + + /** + * Create an API Key.
+ * See + * the docs for more. + * + * @param request the request to create a API key + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the create API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public CreateApiKeyResponse createApiKey(final CreateApiKeyRequest request, final RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::createApiKey, options, + CreateApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously creates an API key.
+ * See + * the docs for more. + * + * @param request the request to create a API key + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void createApiKeyAsync(final CreateApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::createApiKey, options, + CreateApiKeyResponse::fromXContent, listener, emptySet()); + } + + /** + * Retrieve API Key(s) information.
+ * See + * the docs for more. + * + * @param request the request to retrieve API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the create API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetApiKeyResponse getApiKey(final GetApiKeyRequest request, final RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::getApiKey, options, + GetApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously retrieve API Key(s) information.
+ * See + * the docs for more. + * + * @param request the request to retrieve API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getApiKeyAsync(final GetApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::getApiKey, options, + GetApiKeyResponse::fromXContent, listener, emptySet()); + } + + /** + * Invalidate API Key(s).
+ * See + * the docs for more. + * + * @param request the request to invalidate API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response from the invalidate API key call + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public InvalidateApiKeyResponse invalidateApiKey(final InvalidateApiKeyRequest request, final RequestOptions options) + throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options, + InvalidateApiKeyResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously invalidates API key(s).
+ * See + * the docs for more. + * + * @param request the request to invalidate API key(s) + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void invalidateApiKeyAsync(final InvalidateApiKeyRequest request, final RequestOptions options, + final ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options, + InvalidateApiKeyResponse::fromXContent, listener, emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java index 290b9eb843eac..d4b5ceeae0d55 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java @@ -26,6 +26,7 @@ import org.elasticsearch.client.security.ChangePasswordRequest; import org.elasticsearch.client.security.ClearRealmCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheRequest; +import org.elasticsearch.client.security.CreateApiKeyRequest; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.DeletePrivilegesRequest; import org.elasticsearch.client.security.DeleteRoleMappingRequest; @@ -33,11 +34,13 @@ import org.elasticsearch.client.security.DeleteUserRequest; import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetRoleMappingsRequest; import org.elasticsearch.client.security.GetRolesRequest; import org.elasticsearch.client.security.GetUsersRequest; import org.elasticsearch.client.security.HasPrivilegesRequest; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; import org.elasticsearch.client.security.InvalidateTokenRequest; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutRoleMappingRequest; @@ -256,4 +259,36 @@ static Request putRole(final PutRoleRequest putRoleRequest) throws IOException { params.withRefreshPolicy(putRoleRequest.getRefreshPolicy()); return request; } + + static Request createApiKey(final CreateApiKeyRequest createApiKeyRequest) throws IOException { + final Request request = new Request(HttpPost.METHOD_NAME, "/_security/api_key"); + request.setEntity(createEntity(createApiKeyRequest, REQUEST_BODY_CONTENT_TYPE)); + final RequestConverters.Params params = new RequestConverters.Params(request); + params.withRefreshPolicy(createApiKeyRequest.getRefreshPolicy()); + return request; + } + + static Request getApiKey(final GetApiKeyRequest getApiKeyRequest) throws IOException { + final Request request = new Request(HttpGet.METHOD_NAME, "/_security/api_key"); + if (Strings.hasText(getApiKeyRequest.getId())) { + request.addParameter("id", getApiKeyRequest.getId()); + } + if (Strings.hasText(getApiKeyRequest.getName())) { + request.addParameter("name", getApiKeyRequest.getName()); + } + if (Strings.hasText(getApiKeyRequest.getUserName())) { + request.addParameter("username", getApiKeyRequest.getUserName()); + } + if (Strings.hasText(getApiKeyRequest.getRealmName())) { + request.addParameter("realm_name", getApiKeyRequest.getRealmName()); + } + return request; + } + + static Request invalidateApiKey(final InvalidateApiKeyRequest invalidateApiKeyRequest) throws IOException { + final Request request = new Request(HttpDelete.METHOD_NAME, "/_security/api_key"); + request.setEntity(createEntity(invalidateApiKeyRequest, REQUEST_BODY_CONTENT_TYPE)); + final RequestConverters.Params params = new RequestConverters.Params(request); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java new file mode 100644 index 0000000000000..ad5f0a9ba2cf6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyRequest.java @@ -0,0 +1,128 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Request to create API key + */ +public final class CreateApiKeyRequest implements Validatable, ToXContentObject { + + private final String name; + private final TimeValue expiration; + private final List roles; + private final RefreshPolicy refreshPolicy; + + /** + * Create API Key request constructor + * @param name name for the API key + * @param roles list of {@link Role}s + * @param expiration to specify expiration for the API key + */ + public CreateApiKeyRequest(String name, List roles, @Nullable TimeValue expiration, @Nullable final RefreshPolicy refreshPolicy) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + this.roles = Objects.requireNonNull(roles, "roles may not be null"); + this.expiration = expiration; + this.refreshPolicy = (refreshPolicy == null) ? RefreshPolicy.getDefault() : refreshPolicy; + } + + public String getName() { + return name; + } + + public TimeValue getExpiration() { + return expiration; + } + + public List getRoles() { + return roles; + } + + public RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + @Override + public int hashCode() { + return Objects.hash(name, refreshPolicy, roles, expiration); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CreateApiKeyRequest that = (CreateApiKeyRequest) o; + return Objects.equals(name, that.name) && Objects.equals(refreshPolicy, that.refreshPolicy) && Objects.equals(roles, that.roles) + && Objects.equals(expiration, that.expiration); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject().field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.getStringRep()); + } + builder.startObject("role_descriptors"); + for (Role role : roles) { + builder.startObject(role.getName()); + if (role.getApplicationPrivileges() != null) { + builder.field(Role.APPLICATIONS.getPreferredName(), role.getApplicationPrivileges()); + } + if (role.getClusterPrivileges() != null) { + builder.field(Role.CLUSTER.getPreferredName(), role.getClusterPrivileges()); + } + if (role.getGlobalPrivileges() != null) { + builder.field(Role.GLOBAL.getPreferredName(), role.getGlobalPrivileges()); + } + if (role.getIndicesPrivileges() != null) { + builder.field(Role.INDICES.getPreferredName(), role.getIndicesPrivileges()); + } + if (role.getMetadata() != null) { + builder.field(Role.METADATA.getPreferredName(), role.getMetadata()); + } + if (role.getRunAsPrivilege() != null) { + builder.field(Role.RUN_AS.getPreferredName(), role.getRunAsPrivilege()); + } + builder.endObject(); + } + builder.endObject(); + return builder.endObject(); + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java new file mode 100644 index 0000000000000..9c5037237407b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/CreateApiKeyResponse.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for create API key + */ +public final class CreateApiKeyResponse { + + private final String name; + private final String id; + private final SecureString key; + private final Instant expiration; + + public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) { + this.name = name; + this.id = id; + this.key = key; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + } + + public String getName() { + return name; + } + + public String getId() { + return id; + } + + public SecureString getKey() { + return key; + } + + @Nullable + public Instant getExpiration() { + return expiration; + } + + @Override + public int hashCode() { + return Objects.hash(id, name, key, expiration); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final CreateApiKeyResponse other = (CreateApiKeyResponse) obj; + return Objects.equals(id, other.id) + && Objects.equals(key, other.key) + && Objects.equals(name, other.name) + && Objects.equals(expiration, other.expiration); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("create_api_key_response", + args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]))); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareString(constructorArg(), new ParseField("api_key")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + } + + public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java new file mode 100644 index 0000000000000..6fa98ec549b07 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyRequest.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Request for get API key + */ +public final class GetApiKeyRequest implements Validatable, ToXContentObject { + + private final String realmName; + private final String userName; + private final String id; + private final String name; + + // pkg scope for testing + GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + throwValidationError("One of [api key id, api key name, username, realm name] must be specified"); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + throwValidationError( + "username or realm name must not be specified when the api key id or api key name is specified"); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + throwValidationError("only one of [api key id, api key name] can be specified"); + } + this.realmName = realmName; + this.userName = userName; + this.id = apiKeyId; + this.name = apiKeyName; + } + + private void throwValidationError(String message) { + throw new IllegalArgumentException(message); + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates get API key request for given realm name + * @param realmName realm name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmName(String realmName) { + return new GetApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates get API key request for given user name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingUserName(String userName) { + return new GetApiKeyRequest(null, userName, null, null); + } + + /** + * Creates get API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new GetApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates get API key request for given api key id + * @param apiKeyId api key id + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyId(String apiKeyId) { + return new GetApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates get API key request for given api key name + * @param apiKeyName api key name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyName(String apiKeyName) { + return new GetApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder; + } + +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java new file mode 100644 index 0000000000000..58e3e8effbb09 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetApiKeyResponse.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.support.ApiKey; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for get API keys.
+ * The result contains information about the API keys that were found. + */ +public final class GetApiKeyResponse { + + private final List foundApiKeysInfo; + + public GetApiKeyResponse(List foundApiKeysInfo) { + Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided"); + this.foundApiKeysInfo = Collections.unmodifiableList(foundApiKeysInfo); + } + + public static GetApiKeyResponse emptyResponse() { + return new GetApiKeyResponse(Collections.emptyList()); + } + + public List getApiKeyInfos() { + return foundApiKeysInfo; + } + + @Override + public int hashCode() { + return Objects.hash(foundApiKeysInfo); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final GetApiKeyResponse other = (GetApiKeyResponse) obj; + return Objects.equals(foundApiKeysInfo, other.foundApiKeysInfo); + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> { + return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List) args[0]); + }); + static { + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys")); + } + + public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]"; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java new file mode 100644 index 0000000000000..d3203354b7ab1 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyRequest.java @@ -0,0 +1,145 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Request for invalidating API key(s) so that it can no longer be used + */ +public final class InvalidateApiKeyRequest implements Validatable, ToXContentObject { + + private final String realmName; + private final String userName; + private final String id; + private final String name; + + // pkg scope for testing + InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + throwValidationError("One of [api key id, api key name, username, realm name] must be specified"); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + throwValidationError( + "username or realm name must not be specified when the api key id or api key name is specified"); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + throwValidationError("only one of [api key id, api key name] can be specified"); + } + this.realmName = realmName; + this.userName = userName; + this.id = apiKeyId; + this.name = apiKeyName; + } + + private void throwValidationError(String message) { + throw new IllegalArgumentException(message); + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates invalidate API key request for given realm name + * @param realmName realm name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmName(String realmName) { + return new InvalidateApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates invalidate API key request for given user name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingUserName(String userName) { + return new InvalidateApiKeyRequest(null, userName, null, null); + } + + /** + * Creates invalidate API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new InvalidateApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates invalidate API key request for given api key id + * @param apiKeyId api key id + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyId(String apiKeyId) { + return new InvalidateApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates invalidate API key request for given api key name + * @param apiKeyName api key name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyName(String apiKeyName) { + return new InvalidateApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (realmName != null) { + builder.field("realm_name", realmName); + } + if (userName != null) { + builder.field("username", userName); + } + if (id != null) { + builder.field("id", id); + } + if (name != null) { + builder.field("name", name); + } + return builder.endObject(); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java new file mode 100644 index 0000000000000..48df9d0f7f12b --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/InvalidateApiKeyResponse.java @@ -0,0 +1,121 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public final class InvalidateApiKeyResponse { + + private final List invalidatedApiKeys; + private final List previouslyInvalidatedApiKeys; + private final List errors; + + /** + * Constructor for API keys invalidation response + * @param invalidatedApiKeys list of invalidated API key ids + * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids + * @param errors list of encountered errors while invalidating API keys + */ + public InvalidateApiKeyResponse(List invalidatedApiKeys, List previouslyInvalidatedApiKeys, + @Nullable List errors) { + this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided"); + this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys, + "previously_invalidated_api_keys must be provided"); + if (null != errors) { + this.errors = errors; + } else { + this.errors = Collections.emptyList(); + } + } + + public static InvalidateApiKeyResponse emptyResponse() { + return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + public List getInvalidatedApiKeys() { + return invalidatedApiKeys; + } + + public List getPreviouslyInvalidatedApiKeys() { + return previouslyInvalidatedApiKeys; + } + + public List getErrors() { + return errors; + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("invalidate_api_key_response", + args -> { + return new InvalidateApiKeyResponse((List) args[0], (List) args[1], (List) args[3]); + }); + static { + PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys")); + PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys")); + // error count is parsed but ignored as we have list of errors + PARSER.declareInt(constructorArg(), new ParseField("error_count")); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error_details")); + } + + public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public int hashCode() { + return Objects.hash(invalidatedApiKeys, previouslyInvalidatedApiKeys, errors); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + InvalidateApiKeyResponse other = (InvalidateApiKeyResponse) obj; + return Objects.equals(invalidatedApiKeys, other.invalidatedApiKeys) + && Objects.equals(previouslyInvalidatedApiKeys, other.previouslyInvalidatedApiKeys) + && Objects.equals(errors, other.errors); + } + + @Override + public String toString() { + return "ApiKeysInvalidationResult [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys=" + + previouslyInvalidatedApiKeys + ", errors=" + errors + "]"; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java new file mode 100644 index 0000000000000..d021628f750cb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/support/ApiKey.java @@ -0,0 +1,152 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security.support; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * API key information + */ +public final class ApiKey { + + private final String name; + private final String id; + private final Instant creation; + private final Instant expiration; + private final boolean invalidated; + private final String username; + private final String realm; + + public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { + this.name = name; + this.id = id; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.creation = Instant.ofEpochMilli(creation.toEpochMilli()); + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + this.invalidated = invalidated; + this.username = username; + this.realm = realm; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * @return a instance of {@link Instant} when this API key was created. + */ + public Instant getCreation() { + return creation; + } + + /** + * @return a instance of {@link Instant} when this API key will expire. In case the API key does not expire then will return + * {@code null} + */ + public Instant getExpiration() { + return expiration; + } + + /** + * @return {@code true} if this API key has been invalidated else returns {@code false} + */ + public boolean isInvalidated() { + return invalidated; + } + + /** + * @return the username for which this API key was created. + */ + public String getUsername() { + return username; + } + + /** + * @return the realm name of the user for which this API key was created. + */ + public String getRealm() { + return realm; + } + + @Override + public int hashCode() { + return Objects.hash(name, id, creation, expiration, invalidated, username, realm); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ApiKey other = (ApiKey) obj; + return Objects.equals(name, other.name) + && Objects.equals(id, other.id) + && Objects.equals(creation, other.creation) + && Objects.equals(expiration, other.expiration) + && Objects.equals(invalidated, other.invalidated) + && Objects.equals(username, other.username) + && Objects.equals(realm, other.realm); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("api_key", args -> { + return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareLong(constructorArg(), new ParseField("creation")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); + PARSER.declareString(constructorArg(), new ParseField("username")); + PARSER.declareString(constructorArg(), new ParseField("realm")); + } + + public static ApiKey fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated=" + + invalidated + ", username=" + username + ", realm=" + realm + "]"; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java index ae04ffd258af9..0e36a0656c1f1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java @@ -24,6 +24,7 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.security.ChangePasswordRequest; +import org.elasticsearch.client.security.CreateApiKeyRequest; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.DeletePrivilegesRequest; import org.elasticsearch.client.security.DeleteRoleMappingRequest; @@ -31,10 +32,12 @@ import org.elasticsearch.client.security.DeleteUserRequest; import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; +import org.elasticsearch.client.security.GetApiKeyRequest; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetRoleMappingsRequest; import org.elasticsearch.client.security.GetRolesRequest; import org.elasticsearch.client.security.GetUsersRequest; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; import org.elasticsearch.client.security.PutPrivilegesRequest; import org.elasticsearch.client.security.PutRoleMappingRequest; import org.elasticsearch.client.security.PutRoleRequest; @@ -44,11 +47,14 @@ import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression; import org.elasticsearch.client.security.support.expressiondsl.fields.FieldRoleMapperExpression; import org.elasticsearch.client.security.user.User; -import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.ApplicationPrivilege; +import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.test.ESTestCase; @@ -61,6 +67,7 @@ import java.util.Map; import static org.elasticsearch.client.RequestConvertersTests.assertToXContentBody; +import static org.hamcrest.Matchers.equalTo; public class SecurityRequestConvertersTests extends ESTestCase { @@ -411,4 +418,47 @@ public void testPutRole() throws IOException { assertEquals(expectedParams, request.getParameters()); assertToXContentBody(putRoleRequest, request.getEntity()); } -} + + public void testCreateApiKey() throws IOException { + final String name = randomAlphaOfLengthBetween(4, 7); + final List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = randomBoolean() ? null : TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + final Map expectedParams; + if (refreshPolicy != RefreshPolicy.NONE) { + expectedParams = Collections.singletonMap("refresh", refreshPolicy.getValue()); + } else { + expectedParams = Collections.emptyMap(); + } + final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + final Request request = SecurityRequestConverters.createApiKey(createApiKeyRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertToXContentBody(createApiKeyRequest, request.getEntity()); + } + + public void testGetApiKey() throws IOException { + String realmName = randomAlphaOfLength(5); + String userName = randomAlphaOfLength(7); + final GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName(realmName, userName); + final Request request = SecurityRequestConverters.getApiKey(getApiKeyRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + Map mapOfParameters = new HashMap<>(); + mapOfParameters.put("realm_name", realmName); + mapOfParameters.put("username", userName); + assertThat(request.getParameters(), equalTo(mapOfParameters)); + } + + public void testInvalidateApiKey() throws IOException { + String realmName = randomAlphaOfLength(5); + String userName = randomAlphaOfLength(7); + final InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName(realmName, userName); + final Request request = SecurityRequestConverters.invalidateApiKey(invalidateApiKeyRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_security/api_key", request.getEndpoint()); + assertToXContentBody(invalidateApiKeyRequest, request.getEntity()); + } + } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index fe98b2540183f..0ee9621f64333 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -33,6 +33,8 @@ import org.elasticsearch.client.security.ClearRealmCacheResponse; import org.elasticsearch.client.security.ClearRolesCacheRequest; import org.elasticsearch.client.security.ClearRolesCacheResponse; +import org.elasticsearch.client.security.CreateApiKeyRequest; +import org.elasticsearch.client.security.CreateApiKeyResponse; import org.elasticsearch.client.security.CreateTokenRequest; import org.elasticsearch.client.security.CreateTokenResponse; import org.elasticsearch.client.security.DeletePrivilegesRequest; @@ -46,6 +48,8 @@ import org.elasticsearch.client.security.DisableUserRequest; import org.elasticsearch.client.security.EnableUserRequest; import org.elasticsearch.client.security.ExpressionRoleMapping; +import org.elasticsearch.client.security.GetApiKeyRequest; +import org.elasticsearch.client.security.GetApiKeyResponse; import org.elasticsearch.client.security.GetPrivilegesRequest; import org.elasticsearch.client.security.GetPrivilegesResponse; import org.elasticsearch.client.security.GetRoleMappingsRequest; @@ -58,6 +62,8 @@ import org.elasticsearch.client.security.GetUsersResponse; import org.elasticsearch.client.security.HasPrivilegesRequest; import org.elasticsearch.client.security.HasPrivilegesResponse; +import org.elasticsearch.client.security.InvalidateApiKeyRequest; +import org.elasticsearch.client.security.InvalidateApiKeyResponse; import org.elasticsearch.client.security.user.privileges.ApplicationResourcePrivileges; import org.elasticsearch.client.security.user.privileges.UserIndicesPrivileges; import org.elasticsearch.client.security.InvalidateTokenRequest; @@ -71,6 +77,7 @@ import org.elasticsearch.client.security.PutUserRequest; import org.elasticsearch.client.security.PutUserResponse; import org.elasticsearch.client.security.RefreshPolicy; +import org.elasticsearch.client.security.support.ApiKey; import org.elasticsearch.client.security.support.CertificateInfo; import org.elasticsearch.client.security.support.expressiondsl.RoleMapperExpression; import org.elasticsearch.client.security.support.expressiondsl.expressions.AnyRoleMapperExpression; @@ -79,12 +86,17 @@ import org.elasticsearch.client.security.user.privileges.ApplicationPrivilege; import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; +import org.elasticsearch.client.security.user.privileges.UserIndicesPrivileges; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.hamcrest.Matchers; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -97,15 +109,20 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; + import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase { @@ -336,7 +353,7 @@ public void onFailure(Exception e) { private void addUser(RestHighLevelClient client, String userName, String password) throws IOException { User user = new User(userName, Collections.singletonList(userName)); - PutUserRequest request = new PutUserRequest(user, password.toCharArray(), true, RefreshPolicy.NONE); + PutUserRequest request = PutUserRequest.withPassword(user, password.toCharArray(), true, RefreshPolicy.NONE); PutUserResponse response = client.security().putUser(request, RequestOptions.DEFAULT); assertTrue(response.isCreated()); } @@ -510,7 +527,7 @@ public void testEnableUser() throws Exception { RestHighLevelClient client = highLevelClient(); char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User enable_user = new User("enable_user", Collections.singletonList("superuser")); - PutUserRequest putUserRequest = new PutUserRequest(enable_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(enable_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); @@ -555,7 +572,7 @@ public void testDisableUser() throws Exception { RestHighLevelClient client = highLevelClient(); char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User disable_user = new User("disable_user", Collections.singletonList("superuser")); - PutUserRequest putUserRequest = new PutUserRequest(disable_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(disable_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); { @@ -1033,7 +1050,7 @@ public void testChangePassword() throws Exception { char[] password = new char[]{'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; char[] newPassword = new char[]{'n', 'e', 'w', 'p', 'a', 's', 's', 'w', 'o', 'r', 'd'}; User user = new User("change_password_user", Collections.singletonList("superuser"), Collections.emptyMap(), null, null); - PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.NONE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.NONE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); { @@ -1250,7 +1267,8 @@ public void testCreateToken() throws Exception { { // Setup user User token_user = new User("token_user", Collections.singletonList("kibana_user")); - PutUserRequest putUserRequest = new PutUserRequest(token_user, "password".toCharArray(), true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(token_user, "password".toCharArray(), true, + RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); } @@ -1328,27 +1346,27 @@ public void testInvalidateToken() throws Exception { // Setup users final char[] password = "password".toCharArray(); User user = new User("user", Collections.singletonList("kibana_user")); - PutUserRequest putUserRequest = new PutUserRequest(user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putUserRequest = PutUserRequest.withPassword(user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putUserResponse = client.security().putUser(putUserRequest, RequestOptions.DEFAULT); assertTrue(putUserResponse.isCreated()); User this_user = new User("this_user", Collections.singletonList("kibana_user")); - PutUserRequest putThisUserRequest = new PutUserRequest(this_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putThisUserRequest = PutUserRequest.withPassword(this_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putThisUserResponse = client.security().putUser(putThisUserRequest, RequestOptions.DEFAULT); assertTrue(putThisUserResponse.isCreated()); User that_user = new User("that_user", Collections.singletonList("kibana_user")); - PutUserRequest putThatUserRequest = new PutUserRequest(that_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putThatUserRequest = PutUserRequest.withPassword(that_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putThatUserResponse = client.security().putUser(putThatUserRequest, RequestOptions.DEFAULT); assertTrue(putThatUserResponse.isCreated()); User other_user = new User("other_user", Collections.singletonList("kibana_user")); - PutUserRequest putOtherUserRequest = new PutUserRequest(other_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putOtherUserRequest = PutUserRequest.withPassword(other_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putOtherUserResponse = client.security().putUser(putOtherUserRequest, RequestOptions.DEFAULT); assertTrue(putOtherUserResponse.isCreated()); User extra_user = new User("extra_user", Collections.singletonList("kibana_user")); - PutUserRequest putExtraUserRequest = new PutUserRequest(extra_user, password, true, RefreshPolicy.IMMEDIATE); + PutUserRequest putExtraUserRequest = PutUserRequest.withPassword(extra_user, password, true, RefreshPolicy.IMMEDIATE); PutUserResponse putExtraUserResponse = client.security().putUser(putExtraUserRequest, RequestOptions.DEFAULT); assertTrue(putExtraUserResponse.isCreated()); @@ -1748,4 +1766,363 @@ public void onFailure(Exception e) { } } + public void testCreateApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + { + final String name = randomAlphaOfLength(5); + // tag::create-api-key-request + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + // end::create-api-key-request + + // tag::create-api-key-execute + CreateApiKeyResponse createApiKeyResponse = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + // end::create-api-key-execute + + // tag::create-api-key-response + SecureString apiKey = createApiKeyResponse.getKey(); // <1> + Instant apiKeyExpiration = createApiKeyResponse.getExpiration(); // <2> + // end::create-api-key-response + assertThat(createApiKeyResponse.getName(), equalTo(name)); + assertNotNull(apiKey); + assertNotNull(apiKeyExpiration); + } + + { + final String name = randomAlphaOfLength(5); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + + ActionListener listener; + // tag::create-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(CreateApiKeyResponse createApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::create-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::create-api-key-execute-async + client.security().createApiKeyAsync(createApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::create-api-key-execute-async + + assertNotNull(future.get(30, TimeUnit.SECONDS)); + assertThat(future.get().getName(), equalTo(name)); + assertNotNull(future.get().getKey()); + assertNotNull(future.get().getExpiration()); + } + } + + public void testGetApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + // Create API Keys + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse1.getName(), equalTo("k1")); + assertNotNull(createApiKeyResponse1.getKey()); + + final ApiKey expectedApiKeyInfo = new ApiKey(createApiKeyResponse1.getName(), createApiKeyResponse1.getId(), Instant.now(), + Instant.now().plusMillis(expiration.getMillis()), false, "test_user", "default_file"); + { + // tag::get-api-key-id-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + // end::get-api-key-id-request + + // tag::get-api-key-execute + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + // end::get-api-key-execute + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-api-key-name-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyName(createApiKeyResponse1.getName()); + // end::get-api-key-name-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-realm-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmName("default_file"); + // end::get-realm-api-keys-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-user-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingUserName("test_user"); + // end::get-user-api-keys-request + + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + // tag::get-user-realm-api-keys-request + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingRealmAndUserName("default_file", "test_user"); + // end::get-user-realm-api-keys-request + + // tag::get-api-key-response + GetApiKeyResponse getApiKeyResponse = client.security().getApiKey(getApiKeyRequest, RequestOptions.DEFAULT); + // end::get-api-key-response + + assertThat(getApiKeyResponse.getApiKeyInfos(), is(notNullValue())); + assertThat(getApiKeyResponse.getApiKeyInfos().size(), is(1)); + verifyApiKey(getApiKeyResponse.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + + { + GetApiKeyRequest getApiKeyRequest = GetApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + + ActionListener listener; + // tag::get-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(GetApiKeyResponse getApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::get-api-key-execute-async + client.security().getApiKeyAsync(getApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::get-api-key-execute-async + + final GetApiKeyResponse response = future.get(30, TimeUnit.SECONDS); + assertNotNull(response); + + assertThat(response.getApiKeyInfos(), is(notNullValue())); + assertThat(response.getApiKeyInfos().size(), is(1)); + verifyApiKey(response.getApiKeyInfos().get(0), expectedApiKeyInfo); + } + } + + private void verifyApiKey(final ApiKey actual, final ApiKey expected) { + assertThat(actual.getId(), is(expected.getId())); + assertThat(actual.getName(), is(expected.getName())); + assertThat(actual.getUsername(), is(expected.getUsername())); + assertThat(actual.getRealm(), is(expected.getRealm())); + assertThat(actual.isInvalidated(), is(expected.isInvalidated())); + assertThat(actual.getExpiration(), is(greaterThan(Instant.now()))); + } + + public void testInvalidateApiKey() throws Exception { + RestHighLevelClient client = highLevelClient(); + + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = TimeValue.timeValueHours(24); + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + // Create API Keys + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("k1", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse1 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse1.getName(), equalTo("k1")); + assertNotNull(createApiKeyResponse1.getKey()); + + { + // tag::invalidate-api-key-id-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse1.getId()); + // end::invalidate-api-key-id-request + + // tag::invalidate-api-key-execute + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + // end::invalidate-api-key-execute + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse1.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k2", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse2 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse2.getName(), equalTo("k2")); + assertNotNull(createApiKeyResponse2.getKey()); + + // tag::invalidate-api-key-name-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyName(createApiKeyResponse2.getName()); + // end::invalidate-api-key-name-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse2.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k3", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse3 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse3.getName(), equalTo("k3")); + assertNotNull(createApiKeyResponse3.getKey()); + + // tag::invalidate-realm-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmName("default_file"); + // end::invalidate-realm-api-keys-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse3.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k4", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse4 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse4.getName(), equalTo("k4")); + assertNotNull(createApiKeyResponse4.getKey()); + + // tag::invalidate-user-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingUserName("test_user"); + // end::invalidate-user-api-keys-request + + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse4.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k5", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse5 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse5.getName(), equalTo("k5")); + assertNotNull(createApiKeyResponse5.getKey()); + + // tag::invalidate-user-realm-api-keys-request + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingRealmAndUserName("default_file", "test_user"); + // end::invalidate-user-realm-api-keys-request + + // tag::invalidate-api-key-response + InvalidateApiKeyResponse invalidateApiKeyResponse = client.security().invalidateApiKey(invalidateApiKeyRequest, + RequestOptions.DEFAULT); + // end::invalidate-api-key-response + + final List errors = invalidateApiKeyResponse.getErrors(); + final List invalidatedApiKeyIds = invalidateApiKeyResponse.getInvalidatedApiKeys(); + final List previouslyInvalidatedApiKeyIds = invalidateApiKeyResponse.getPreviouslyInvalidatedApiKeys(); + + assertTrue(errors.isEmpty()); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse5.getId()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(previouslyInvalidatedApiKeyIds.size(), equalTo(0)); + } + + { + createApiKeyRequest = new CreateApiKeyRequest("k6", roles, expiration, refreshPolicy); + CreateApiKeyResponse createApiKeyResponse6 = client.security().createApiKey(createApiKeyRequest, RequestOptions.DEFAULT); + assertThat(createApiKeyResponse6.getName(), equalTo("k6")); + assertNotNull(createApiKeyResponse6.getKey()); + + InvalidateApiKeyRequest invalidateApiKeyRequest = InvalidateApiKeyRequest.usingApiKeyId(createApiKeyResponse6.getId()); + + ActionListener listener; + // tag::invalidate-api-key-execute-listener + listener = new ActionListener() { + @Override + public void onResponse(InvalidateApiKeyResponse invalidateApiKeyResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::invalidate-api-key-execute-listener + + // Avoid unused variable warning + assertNotNull(listener); + + // Replace the empty listener by a blocking listener in test + final PlainActionFuture future = new PlainActionFuture<>(); + listener = future; + + // tag::invalidate-api-key-execute-async + client.security().invalidateApiKeyAsync(invalidateApiKeyRequest, RequestOptions.DEFAULT, listener); // <1> + // end::invalidate-api-key-execute-async + + final InvalidateApiKeyResponse response = future.get(30, TimeUnit.SECONDS); + assertNotNull(response); + final List invalidatedApiKeyIds = response.getInvalidatedApiKeys(); + List expectedInvalidatedApiKeyIds = Arrays.asList(createApiKeyResponse6.getId()); + assertTrue(response.getErrors().isEmpty()); + assertThat(invalidatedApiKeyIds, containsInAnyOrder(expectedInvalidatedApiKeyIds.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getPreviouslyInvalidatedApiKeys().size(), equalTo(0)); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java new file mode 100644 index 0000000000000..188493deeb78a --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyRequestTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.user.privileges.IndicesPrivileges; +import org.elasticsearch.client.security.user.privileges.Role; +import org.elasticsearch.client.security.user.privileges.Role.ClusterPrivilegeName; +import org.elasticsearch.client.security.user.privileges.Role.IndexPrivilegeName; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyRequestTests extends ESTestCase { + + public void test() throws IOException { + List roles = new ArrayList<>(); + roles.add(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + roles.add(Role.builder().name("r2").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-y").privileges(IndexPrivilegeName.ALL).build()).build()); + + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest("api-key", roles, null, null); + final XContentBuilder builder = XContentFactory.jsonBuilder(); + createApiKeyRequest.toXContent(builder, ToXContent.EMPTY_PARAMS); + final String output = Strings.toString(builder); + assertThat(output, equalTo( + "{\"name\":\"api-key\",\"role_descriptors\":{\"r1\":{\"applications\":[],\"cluster\":[\"all\"],\"indices\":[{\"names\":" + + "[\"ind-x\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}],\"metadata\":{},\"run_as\":[]}," + + "\"r2\":{\"applications\":[],\"cluster\":" + + "[\"all\"],\"indices\":[{\"names\":[\"ind-y\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":false}]," + + "\"metadata\":{},\"run_as\":[]}}}")); + } + + public void testEqualsHashCode() { + final String name = randomAlphaOfLength(5); + List roles = Collections.singletonList(Role.builder().name("r1").clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges(IndicesPrivileges.builder().indices("ind-x").privileges(IndexPrivilegeName.ALL).build()).build()); + final TimeValue expiration = null; + final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values()); + + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(name, roles, expiration, refreshPolicy); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyRequest, (original) -> { + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), original.getRefreshPolicy()); + }, CreateApiKeyRequestTests::mutateTestItem); + } + + private static CreateApiKeyRequest mutateTestItem(CreateApiKeyRequest original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), + original.getRefreshPolicy()); + case 1: + return new CreateApiKeyRequest(original.getName(), + Collections.singletonList(Role.builder().name(randomAlphaOfLength(6)).clusterPrivileges(ClusterPrivilegeName.ALL) + .indicesPrivileges( + IndicesPrivileges.builder().indices(randomAlphaOfLength(4)).privileges(IndexPrivilegeName.ALL).build()) + .build()), + original.getExpiration(), original.getRefreshPolicy()); + case 2: + return new CreateApiKeyRequest(original.getName(), original.getRoles(), TimeValue.timeValueSeconds(10000), + original.getRefreshPolicy()); + case 3: + List values = Arrays.stream(RefreshPolicy.values()).filter(rp -> rp != original.getRefreshPolicy()) + .collect(Collectors.toList()); + return new CreateApiKeyRequest(original.getName(), original.getRoles(), original.getExpiration(), randomFrom(values)); + default: + return new CreateApiKeyRequest(randomAlphaOfLength(5), original.getRoles(), original.getExpiration(), + original.getRefreshPolicy()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java new file mode 100644 index 0000000000000..4481d70c80b37 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/CreateApiKeyResponseTests.java @@ -0,0 +1,101 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.common.CharArrays; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + final String id = randomAlphaOfLengthBetween(4, 8); + final String name = randomAlphaOfLength(5); + final SecureString apiKey = UUIDs.randomBase64UUIDSecureString(); + final Instant expiration = randomBoolean() ? null : Instant.ofEpochMilli(10000); + + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + builder.startObject().field("id", id).field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + byte[] charBytes = CharArrays.toUtf8Bytes(apiKey.getChars()); + try { + builder.field("api_key").utf8Value(charBytes, 0, charBytes.length); + } finally { + Arrays.fill(charBytes, (byte) 0); + } + builder.endObject(); + BytesReference xContent = BytesReference.bytes(builder); + + final CreateApiKeyResponse response = CreateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(response.getId(), equalTo(id)); + assertThat(response.getName(), equalTo(name)); + assertThat(response.getKey(), equalTo(apiKey)); + if (expiration != null) { + assertThat(response.getExpiration(), equalTo(expiration)); + } + } + + public void testEqualsHashCode() { + final String id = randomAlphaOfLengthBetween(4, 8); + final String name = randomAlphaOfLength(5); + final SecureString apiKey = UUIDs.randomBase64UUIDSecureString(); + final Instant expiration = Instant.ofEpochMilli(10000); + CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyResponse(name, id, apiKey, expiration); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }, CreateApiKeyResponseTests::mutateTestItem); + } + + private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration()); + case 1: + return new CreateApiKeyResponse(original.getName(), randomAlphaOfLengthBetween(4, 8), original.getKey(), + original.getExpiration()); + case 2: + return new CreateApiKeyResponse(original.getName(), original.getId(), UUIDs.randomBase64UUIDSecureString(), + original.getExpiration()); + case 3: + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.ofEpochMilli(150000)); + default: + return new CreateApiKeyResponse(randomAlphaOfLength(7), original.getId(), original.getKey(), original.getExpiration()); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java new file mode 100644 index 0000000000000..79551e1e73e92 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyRequestTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Optional; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + Optional ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertFalse(ve.isPresent()); + request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertFalse(ve.isPresent()); + } + + public void testRequestValidationFailureScenarios() throws IOException { + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }; + + for (int i = 0; i < inputs.length; i++) { + final int caseNo = i; + IllegalArgumentException ve = expectThrows(IllegalArgumentException.class, + () -> new GetApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3])); + assertNotNull(ve); + assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo])); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java new file mode 100644 index 0000000000000..7aa92e4f212a4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetApiKeyResponseTests.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.security.support.ApiKey; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2)); + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + toXContent(response, builder); + BytesReference xContent = BytesReference.bytes(builder); + GetApiKeyResponse responseParsed = GetApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(responseParsed, equalTo(response)); + } + + private void toXContent(GetApiKeyResponse response, final XContentBuilder builder) throws IOException { + builder.startObject(); + builder.startArray("api_keys"); + for (ApiKey apiKey : response.getApiKeyInfos()) { + builder.startObject() + .field("id", apiKey.getId()) + .field("name", apiKey.getName()) + .field("creation", apiKey.getCreation().toEpochMilli()); + if (apiKey.getExpiration() != null) { + builder.field("expiration", apiKey.getExpiration().toEpochMilli()); + } + builder.field("invalidated", apiKey.isInvalidated()) + .field("username", apiKey.getUsername()) + .field("realm", apiKey.getRealm()); + builder.endObject(); + } + builder.endArray(); + builder.endObject(); + } + + public void testEqualsHashCode() { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1)); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> { + return new GetApiKeyResponse(original.getApiKeyInfos()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(response, (original) -> { + return new GetApiKeyResponse(original.getApiKeyInfos()); + }, GetApiKeyResponseTests::mutateTestItem); + } + + private static GetApiKeyResponse mutateTestItem(GetApiKeyResponse original) { + ApiKey apiKeyInfo = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + switch (randomIntBetween(0, 2)) { + case 0: + return new GetApiKeyResponse(Arrays.asList(apiKeyInfo)); + default: + return new GetApiKeyResponse(Arrays.asList(apiKeyInfo)); + } + } + + private static ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, + String username, String realm) { + return new ApiKey(name, id, creation, expiration, invalidated, username, realm); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java new file mode 100644 index 0000000000000..25ee4bb05bcc4 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyRequestTests.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Optional; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class InvalidateApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + Optional ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertThat(ve.isPresent(), is(false)); + } + + public void testRequestValidationFailureScenarios() throws IOException { + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[] expectedErrorMessages = new String[] { "One of [api key id, api key name, username, realm name] must be specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }; + + for (int i = 0; i < inputs.length; i++) { + final int caseNo = i; + IllegalArgumentException ve = expectThrows(IllegalArgumentException.class, + () -> new InvalidateApiKeyRequest(inputs[caseNo][0], inputs[caseNo][1], inputs[caseNo][2], inputs[caseNo][3])); + assertNotNull(ve); + assertThat(ve.getMessage(), equalTo(expectedErrorMessages[caseNo])); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java new file mode 100644 index 0000000000000..f5cd403536fc2 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/security/InvalidateApiKeyResponseTests.java @@ -0,0 +1,111 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.security; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class InvalidateApiKeyResponseTests extends ESTestCase { + + public void testFromXContent() throws IOException { + List invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))); + List previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5))); + List errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new, + () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4))))); + + final XContentType xContentType = randomFrom(XContentType.values()); + final XContentBuilder builder = XContentFactory.contentBuilder(xContentType); + builder.startObject().array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .field("error_count", errors.size()); + if (errors.isEmpty() == false) { + builder.field("error_details"); + builder.startArray(); + for (ElasticsearchException e : errors) { + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, ToXContent.EMPTY_PARAMS, e); + builder.endObject(); + } + builder.endArray(); + } + builder.endObject(); + BytesReference xContent = BytesReference.bytes(builder); + + final InvalidateApiKeyResponse response = InvalidateApiKeyResponse.fromXContent(createParser(xContentType.xContent(), xContent)); + assertThat(response.getInvalidatedApiKeys(), containsInAnyOrder(invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getPreviouslyInvalidatedApiKeys(), + containsInAnyOrder(previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY))); + assertThat(response.getErrors(), is(notNullValue())); + assertThat(response.getErrors().size(), is(errors.size())); + assertThat(response.getErrors().get(0).toString(), containsString("type=illegal_argument_exception")); + assertThat(response.getErrors().get(1).toString(), containsString("type=illegal_argument_exception")); + } + + public void testEqualsHashCode() { + List invalidatedApiKeys = Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))); + List previouslyInvalidatedApiKeys = Arrays.asList(randomArray(2, 3, String[]::new, () -> randomAlphaOfLength(5))); + List errors = Arrays.asList(randomArray(2, 5, ElasticsearchException[]::new, + () -> new ElasticsearchException(randomAlphaOfLength(5), new IllegalArgumentException(randomAlphaOfLength(4))))); + InvalidateApiKeyResponse invalidateApiKeyResponse = new InvalidateApiKeyResponse(invalidatedApiKeys, previouslyInvalidatedApiKeys, + errors); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> { + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + original.getErrors()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(invalidateApiKeyResponse, (original) -> { + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + original.getErrors()); + }, InvalidateApiKeyResponseTests::mutateTestItem); + } + + private static InvalidateApiKeyResponse mutateTestItem(InvalidateApiKeyResponse original) { + switch (randomIntBetween(0, 2)) { + case 0: + return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))), + original.getPreviouslyInvalidatedApiKeys(), original.getErrors()); + case 1: + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), Collections.emptyList(), original.getErrors()); + case 2: + return new InvalidateApiKeyResponse(original.getInvalidatedApiKeys(), original.getPreviouslyInvalidatedApiKeys(), + Collections.emptyList()); + default: + return new InvalidateApiKeyResponse(Arrays.asList(randomArray(2, 5, String[]::new, () -> randomAlphaOfLength(5))), + original.getPreviouslyInvalidatedApiKeys(), original.getErrors()); + } + } +} diff --git a/docs/java-rest/high-level/security/create-api-key.asciidoc b/docs/java-rest/high-level/security/create-api-key.asciidoc new file mode 100644 index 0000000000000..93c3fa16de1da --- /dev/null +++ b/docs/java-rest/high-level/security/create-api-key.asciidoc @@ -0,0 +1,40 @@ +-- +:api: create-api-key +:request: CreateApiKeyRequest +:response: CreateApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Create API Key API + +API Key can be created using this API. + +[id="{upid}-{api}-request"] +==== Create API Key Request + +A +{request}+ contains name for the API key, +list of role descriptors to define permissions and +optional expiration for the generated API key. +If expiration is not provided then by default the API +keys do not expire. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Create API Key Response + +The returned +{response}+ contains an id, +API key, name for the API key and optional +expiration. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- +<1> the API key that can be used to authenticate to Elasticsearch. +<2> expiration if the API keys expire \ No newline at end of file diff --git a/docs/java-rest/high-level/security/get-api-key.asciidoc b/docs/java-rest/high-level/security/get-api-key.asciidoc new file mode 100644 index 0000000000000..bb98b527d22ba --- /dev/null +++ b/docs/java-rest/high-level/security/get-api-key.asciidoc @@ -0,0 +1,67 @@ +-- +:api: get-api-key +:request: GetApiKeyRequest +:response: GetApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Get API Key information API + +API Key(s) information can be retrieved using this API. + +[id="{upid}-{api}-request"] +==== Get API Key Request +The +{request}+ supports retrieving API key information for + +. A specific API key + +. All API keys for a specific realm + +. All API keys for a specific user + +. All API keys for a specific user in a specific realm + +===== Retrieve a specific API key by its id +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-api-key-id-request] +-------------------------------------------------- + +===== Retrieve a specific API key by its name +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-api-key-name-request] +-------------------------------------------------- + +===== Retrieve all API keys for given realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-realm-api-keys-request] +-------------------------------------------------- + +===== Retrieve all API keys for a given user +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-user-api-keys-request] +-------------------------------------------------- + +===== Retrieve all API keys for given user in a realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[get-user-realm-api-keys-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Get API Key information API Response + +The returned +{response}+ contains the information regarding the API keys that were +requested. + +`api_keys`:: Available using `getApiKeyInfos`, contains list of API keys that were retrieved for this request. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- diff --git a/docs/java-rest/high-level/security/invalidate-api-key.asciidoc b/docs/java-rest/high-level/security/invalidate-api-key.asciidoc new file mode 100644 index 0000000000000..7f9c43b3165a8 --- /dev/null +++ b/docs/java-rest/high-level/security/invalidate-api-key.asciidoc @@ -0,0 +1,75 @@ +-- +:api: invalidate-api-key +:request: InvalidateApiKeyRequest +:response: InvalidateApiKeyResponse +-- + +[id="{upid}-{api}"] +=== Invalidate API Key API + +API Key(s) can be invalidated using this API. + +[id="{upid}-{api}-request"] +==== Invalidate API Key Request +The +{request}+ supports invalidating + +. A specific API key + +. All API keys for a specific realm + +. All API keys for a specific user + +. All API keys for a specific user in a specific realm + +===== Specific API key by API key id +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-api-key-id-request] +-------------------------------------------------- + +===== Specific API key by API key name +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-api-key-name-request] +-------------------------------------------------- + +===== All API keys for realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-realm-api-keys-request] +-------------------------------------------------- + +===== All API keys for user +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-user-api-keys-request] +-------------------------------------------------- + +===== All API key for user in realm +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[invalidate-user-realm-api-keys-request] +-------------------------------------------------- + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Invalidate API Key Response + +The returned +{response}+ contains the information regarding the API keys that the request +invalidated. + +`invalidatedApiKeys`:: Available using `getInvalidatedApiKeys` lists the API keys + that this request invalidated. + +`previouslyInvalidatedApiKeys`:: Available using `getPreviouslyInvalidatedApiKeys` lists the API keys + that this request attempted to invalidate + but were already invalid. + +`errors`:: Available using `getErrors` contains possible errors that were encountered while + attempting to invalidate API keys. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index b17b667155862..125d56c00f1ed 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -410,6 +410,9 @@ The Java High Level REST Client supports the following Security APIs: * <<{upid}-get-privileges>> * <<{upid}-put-privileges>> * <<{upid}-delete-privileges>> +* <<{upid}-create-api-key>> +* <<{upid}-get-api-key>> +* <<{upid}-invalidate-api-key>> include::security/put-user.asciidoc[] include::security/get-users.asciidoc[] @@ -434,6 +437,9 @@ include::security/delete-role-mapping.asciidoc[] include::security/create-token.asciidoc[] include::security/invalidate-token.asciidoc[] include::security/put-privileges.asciidoc[] +include::security/create-api-key.asciidoc[] +include::security/get-api-key.asciidoc[] +include::security/invalidate-api-key.asciidoc[] == Watcher APIs diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc index a9b6639359e24..c83edb69b3e62 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc @@ -280,6 +280,31 @@ example above), but the same goes for actual values: The stash should be reset at the beginning of each test file. +=== `transform_and_set` + +For some tests, it is necessary to extract a value and transform it from the previous `response`, in +order to reuse it in a subsequent `do` and other tests. +Currently, it only has support for `base64EncodeCredentials`, for unknown transformations it will not +do anything and stash the value as is. +For instance, when testing you may want to base64 encode username and password for +`Basic` authorization header: + +.... + - do: + index: + index: test + type: test + - transform_and_set: { login_creds: "#base64EncodeCredentials(user,password)" } # stash the base64 encoded credentials of `response.user` and `response.password` as `login_creds` + - do: + headers: + Authorization: Basic ${login_creds} # replace `$login_creds` with the stashed value + get: + index: test + type: test +.... + +Stashed values can be used as described in the `set` section + === `is_true` The specified key exists and has a true value (ie not `0`, `false`, `undefined`, `null` diff --git a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java index 59e5960b99d09..b5b35b477efbd 100644 --- a/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java +++ b/server/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java @@ -20,6 +20,9 @@ package org.elasticsearch.common; +import org.elasticsearch.common.settings.SecureString; + +import java.util.Arrays; import java.util.Base64; import java.util.Random; @@ -34,12 +37,37 @@ public String getBase64UUID() { return getBase64UUID(SecureRandomHolder.INSTANCE); } + /** + * Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID + * as defined here: http://www.ietf.org/rfc/rfc4122.txt + */ + public SecureString getBase64UUIDSecureString() { + byte[] uuidBytes = null; + byte[] encodedBytes = null; + try { + uuidBytes = getUUIDBytes(SecureRandomHolder.INSTANCE); + encodedBytes = Base64.getUrlEncoder().withoutPadding().encode(uuidBytes); + return new SecureString(CharArrays.utf8BytesToChars(encodedBytes)); + } finally { + if (uuidBytes != null) { + Arrays.fill(uuidBytes, (byte) 0); + } + if (encodedBytes != null) { + Arrays.fill(encodedBytes, (byte) 0); + } + } + } + /** * Returns a Base64 encoded version of a Version 4.0 compatible UUID * randomly initialized by the given {@link java.util.Random} instance * as defined here: http://www.ietf.org/rfc/rfc4122.txt */ public String getBase64UUID(Random random) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(getUUIDBytes(random)); + } + + private byte[] getUUIDBytes(Random random) { final byte[] randomBytes = new byte[16]; random.nextBytes(randomBytes); /* Set the version to version 4 (see http://www.ietf.org/rfc/rfc4122.txt) @@ -48,12 +76,12 @@ public String getBase64UUID(Random random) { * stamp (bits 4 through 7 of the time_hi_and_version field).*/ randomBytes[6] &= 0x0f; /* clear the 4 most significant bits for the version */ randomBytes[6] |= 0x40; /* set the version to 0100 / 0x40 */ - - /* Set the variant: + + /* Set the variant: * The high field of th clock sequence multiplexed with the variant. * We set only the MSB of the variant*/ randomBytes[8] &= 0x3f; /* clear the 2 most significant bits */ randomBytes[8] |= 0x80; /* set the variant (MSB is set)*/ - return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes); + return randomBytes; } } diff --git a/server/src/main/java/org/elasticsearch/common/UUIDs.java b/server/src/main/java/org/elasticsearch/common/UUIDs.java index 63fcaedde0f5c..a6a314c2cccb0 100644 --- a/server/src/main/java/org/elasticsearch/common/UUIDs.java +++ b/server/src/main/java/org/elasticsearch/common/UUIDs.java @@ -19,6 +19,8 @@ package org.elasticsearch.common; +import org.elasticsearch.common.settings.SecureString; + import java.util.Random; public class UUIDs { @@ -50,4 +52,9 @@ public static String randomBase64UUID() { return RANDOM_UUID_GENERATOR.getBase64UUID(); } + /** Returns a Base64 encoded {@link SecureString} of a Version 4.0 compatible UUID as defined here: http://www.ietf.org/rfc/rfc4122.txt, + * using a private {@code SecureRandom} instance */ + public static SecureString randomBase64UUIDSecureString() { + return RANDOM_UUID_GENERATOR.getBase64UUIDSecureString(); + } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 5adcdd3813a48..16f20bc570ffa 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -579,6 +579,23 @@ public Object readGenericValue() throws IOException { } } + /** + * Read an {@link Instant} from the stream with nanosecond resolution + */ + public final Instant readInstant() throws IOException { + return Instant.ofEpochSecond(readLong(), readInt()); + } + + /** + * Read an optional {@link Instant} from the stream. Returns null when + * no instant is present. + */ + @Nullable + public final Instant readOptionalInstant() throws IOException { + final boolean present = readBoolean(); + return present ? readInstant() : null; + } + @SuppressWarnings("unchecked") private List readArrayList() throws IOException { int size = readArraySize(); diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 651e0271f517b..da2f065d36038 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -55,6 +55,7 @@ import java.nio.file.FileSystemLoopException; import java.nio.file.NoSuchFileException; import java.nio.file.NotDirectoryException; +import java.time.Instant; import java.time.ZonedDateTime; import java.util.Collection; import java.util.Collections; @@ -560,6 +561,26 @@ public final void writeMap(final Map map, final Writer keyWriter } } + /** + * Writes an {@link Instant} to the stream with nanosecond resolution + */ + public final void writeInstant(Instant instant) throws IOException { + writeLong(instant.getEpochSecond()); + writeInt(instant.getNano()); + } + + /** + * Writes an {@link Instant} to the stream, which could possibly be null + */ + public final void writeOptionalInstant(@Nullable Instant instant) throws IOException { + if (instant == null) { + writeBoolean(false); + } else { + writeBoolean(true); + writeInstant(instant); + } + } + private static final Map, Writer> WRITERS; static { diff --git a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java index 0f1fe22c02010..02d534552100c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/set/Sets.java +++ b/server/src/main/java/org/elasticsearch/common/util/set/Sets.java @@ -144,4 +144,19 @@ public static Set union(Set left, Set right) { union.addAll(right); return union; } + + public static Set intersection(Set set1, Set set2) { + Objects.requireNonNull(set1); + Objects.requireNonNull(set2); + final Set left; + final Set right; + if (set1.size() < set2.size()) { + left = set1; + right = set2; + } else { + left = set2; + right = set1; + } + return left.stream().filter(o -> right.contains(o)).collect(Collectors.toSet()); + } } diff --git a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index e2cdaf3c7d5b8..837c0202faf92 100644 --- a/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -30,6 +30,7 @@ import java.io.ByteArrayInputStream; import java.io.EOFException; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -336,6 +337,37 @@ public void testSetOfLongs() throws IOException { assertThat(targetSet, equalTo(sourceSet)); } + public void testInstantSerialization() throws IOException { + final Instant instant = Instant.now(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeInstant(instant); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readInstant(); + assertEquals(instant, serialized); + } + } + } + + public void testOptionalInstantSerialization() throws IOException { + final Instant instant = Instant.now(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeOptionalInstant(instant); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readOptionalInstant(); + assertEquals(instant, serialized); + } + } + + final Instant missing = null; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeOptionalInstant(missing); + try (StreamInput in = out.bytes().streamInput()) { + final Instant serialized = in.readOptionalInstant(); + assertEquals(missing, serialized); + } + } + } + static final class WriteableString implements Writeable { final String string; diff --git a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java index 0c1869a6b4086..f4337daf4346c 100644 --- a/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/set/SetsTests.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -56,6 +57,17 @@ public void testSortedDifference() { } } + public void testIntersection() { + final int endExclusive = randomIntBetween(0, 256); + final Tuple, Set> sets = randomSets(endExclusive); + final Set intersection = Sets.intersection(sets.v1(), sets.v2()); + final Set expectedIntersection = IntStream.range(0, endExclusive) + .boxed() + .filter(i -> (sets.v1().contains(i) && sets.v2().contains(i))) + .collect(Collectors.toSet()); + assertThat(intersection, containsInAnyOrder(expectedIntersection.toArray(new Integer[0]))); + } + /** * Assert the difference between two sets is as expected. * diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java index 7fabacc3c9e68..2833140eff95c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java @@ -45,7 +45,9 @@ public final class Features { "stash_in_path", "stash_path_replace", "warnings", - "yaml")); + "yaml", + "transform_and_set" + )); private Features() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java index ff02d6d16aa4a..135a60cca3431 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java @@ -40,6 +40,7 @@ public interface ExecutableSection { List DEFAULT_EXECUTABLE_CONTEXTS = unmodifiableList(Arrays.asList( new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("transform_and_set"), TransformAndSetSection::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse), new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse), diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java new file mode 100644 index 0000000000000..7b0b915dd97df --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSection.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a transform_and_set section: + *

+ * + * In the following example,
+ * - transform_and_set: { login_creds: "#base64EncodeCredentials(user,password)" }
+ * user and password are from the response which are joined by ':' and Base64 encoded and then stashed as 'login_creds' + * + */ +public class TransformAndSetSection implements ExecutableSection { + public static TransformAndSetSection parse(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + + TransformAndSetSection transformAndStashSection = new TransformAndSetSection(parser.getTokenLocation()); + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + transformAndStashSection.addSet(currentFieldName, parser.text()); + } + } + + parser.nextToken(); + + if (transformAndStashSection.getStash().isEmpty()) { + throw new ParsingException(transformAndStashSection.location, "transform_and_set section must set at least a value"); + } + + return transformAndStashSection; + } + + private final Map transformStash = new HashMap<>(); + private final XContentLocation location; + + public TransformAndSetSection(XContentLocation location) { + this.location = location; + } + + public void addSet(String stashedField, String transformThis) { + transformStash.put(stashedField, transformThis); + } + + public Map getStash() { + return transformStash; + } + + @Override + public XContentLocation getLocation() { + return location; + } + + @Override + public void execute(ClientYamlTestExecutionContext executionContext) throws IOException { + for (Map.Entry entry : transformStash.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (value.startsWith("#base64EncodeCredentials(") && value.endsWith(")")) { + value = entry.getValue().substring("#base64EncodeCredentials(".length(), entry.getValue().lastIndexOf(")")); + String[] idAndPassword = value.split(","); + if (idAndPassword.length == 2) { + String credentials = executionContext.response(idAndPassword[0].trim()) + ":" + + executionContext.response(idAndPassword[1].trim()); + value = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + } else { + throw new IllegalArgumentException("base64EncodeCredentials requires a username/id and a password parameters"); + } + } + executionContext.stash().stashValue(key, value); + } + } + +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java new file mode 100644 index 0000000000000..a61f91de287e7 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TransformAndSetSectionTests.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.Stash; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class TransformAndSetSectionTests extends AbstractClientYamlTestFragmentParserTestCase { + + public void testParseSingleValue() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ key: value }" + ); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(1)); + assertThat(transformAndSet.getStash().get("key"), equalTo("value")); + } + + public void testParseMultipleValues() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ key1: value1, key2: value2 }" + ); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(2)); + assertThat(transformAndSet.getStash().get("key1"), equalTo("value1")); + assertThat(transformAndSet.getStash().get("key2"), equalTo("value2")); + } + + public void testTransformation() throws Exception { + parser = createParser(YamlXContent.yamlXContent, "{ login_creds: \"#base64EncodeCredentials(id,api_key)\" }"); + + TransformAndSetSection transformAndSet = TransformAndSetSection.parse(parser); + assertThat(transformAndSet, notNullValue()); + assertThat(transformAndSet.getStash(), notNullValue()); + assertThat(transformAndSet.getStash().size(), equalTo(1)); + assertThat(transformAndSet.getStash().get("login_creds"), equalTo("#base64EncodeCredentials(id,api_key)")); + + ClientYamlTestExecutionContext executionContext = mock(ClientYamlTestExecutionContext.class); + when(executionContext.response("id")).thenReturn("user"); + when(executionContext.response("api_key")).thenReturn("password"); + Stash stash = new Stash(); + when(executionContext.stash()).thenReturn(stash); + transformAndSet.execute(executionContext); + verify(executionContext).response("id"); + verify(executionContext).response("api_key"); + verify(executionContext).stash(); + assertThat(stash.getValue("$login_creds"), + equalTo(Base64.getEncoder().encodeToString("user:password".getBytes(StandardCharsets.UTF_8)))); + verifyNoMoreInteractions(executionContext); + } + + public void testParseSetSectionNoValues() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "{ }" + ); + + Exception e = expectThrows(ParsingException.class, () -> TransformAndSetSection.parse(parser)); + assertThat(e.getMessage(), is("transform_and_set section must set at least a value")); + } +} diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index b5bff78045511..713c3c1a5815f 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -73,6 +73,7 @@ project.copyRestSpec.from(xpackResources) { } integTestCluster { setting 'xpack.security.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' // Disable monitoring exporters for the docs tests setting 'xpack.monitoring.exporters._local.type', 'local' diff --git a/x-pack/docs/en/rest-api/security.asciidoc b/x-pack/docs/en/rest-api/security.asciidoc index 851bd2ba327b2..c59c44312ae60 100644 --- a/x-pack/docs/en/rest-api/security.asciidoc +++ b/x-pack/docs/en/rest-api/security.asciidoc @@ -51,6 +51,17 @@ without requiring basic authentication: * <> * <> +[float] +[[security-api-keys]] +=== API Keys + +You can use the following APIs to create, retrieve and invalidate API keys for access +without requiring basic authentication: + +* <> +* <> +* <> + [float] [[security-user-apis]] === Users @@ -88,3 +99,6 @@ include::security/get-users.asciidoc[] include::security/has-privileges.asciidoc[] include::security/invalidate-tokens.asciidoc[] include::security/ssl.asciidoc[] +include::security/create-api-keys.asciidoc[] +include::security/invalidate-api-keys.asciidoc[] +include::security/get-api-keys.asciidoc[] diff --git a/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc new file mode 100644 index 0000000000000..e4fa1be71d40e --- /dev/null +++ b/x-pack/docs/en/rest-api/security/create-api-keys.asciidoc @@ -0,0 +1,99 @@ +[role="xpack"] +[[security-api-create-api-key]] +=== Create API Key API + +Creates an API key for access without requiring basic authentication. + +==== Request + +`POST /_security/api_key` +`PUT /_security/api_key` + +==== Description + +The API keys are created by the {es} API key service, which is automatically enabled +when you configure TLS on the HTTP interface. See <>. Alternatively, +you can explicitly enable the `xpack.security.authc.api_key.enabled` setting. When +you are running in production mode, a bootstrap check prevents you from enabling +the API key service unless you also enable TLS on the HTTP interface. + +A successful create API key API call returns a JSON structure that contains +the unique id, the name to identify API key, the API key and the expiration if +applicable for the API key in milliseconds. + +NOTE: By default API keys never expire. You can specify expiration at the time of +creation for the API keys. + +==== Request Body + +The following parameters can be specified in the body of a POST or PUT request: + +`name`:: +(string) Specifies the name for this API key. + +`role_descriptors`:: +(array-of-role-descriptor) Optional array of role descriptor for this API key. The role descriptor +must be a subset of permissions of the authenticated user. The structure of role +descriptor is same as the request for create role API. For more details on role +see <>. +If the role descriptors are not provided then permissions of the authenticated user are applied. + +`expiration`:: +(string) Optional expiration time for the API key. By default API keys never expire. + +==== Examples + +The following example creates an API key: + +[source, js] +------------------------------------------------------------ +POST /_security/api_key +{ + "name": "my-api-key", + "expiration": "1d", <1> + "role_descriptors": { <2> + "role-a": { + "cluster": ["all"], + "index": [ + { + "names": ["index-a*"], + "privileges": ["read"] + } + ] + }, + "role-b": { + "cluster": ["all"], + "index": [ + { + "names": ["index-b*"], + "privileges": ["all"] + } + ] + } + } +} +------------------------------------------------------------ +// CONSOLE +<1> optional expiration for the API key being generated. If expiration is not + provided then the API keys do not expire. +<2> optional role descriptors for this API key, if not provided then permissions + of authenticated user are applied. + +A successful call returns a JSON structure that provides +API key information. + +[source,js] +-------------------------------------------------- +{ + "id":"VuaCfGcBCdbkQm-e5aOx", <1> + "name":"my-api-key", + "expiration":1544068612110, <2> + "api_key":"ui2lp2axTNmsyakw9tvNnw" <3> +} +-------------------------------------------------- +// TESTRESPONSE[s/VuaCfGcBCdbkQm-e5aOx/$body.id/] +// TESTRESPONSE[s/1544068612110/$body.expiration/] +// TESTRESPONSE[s/ui2lp2axTNmsyakw9tvNnw/$body.api_key/] +<1> unique id for this API key +<2> optional expiration in milliseconds for this API key +<3> generated API key diff --git a/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc new file mode 100644 index 0000000000000..ab2ef770cb124 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/get-api-keys.asciidoc @@ -0,0 +1,118 @@ +[role="xpack"] +[[security-api-get-api-key]] +=== Get API Key information API +++++ +Get API key information +++++ + +Retrieves information for one or more API keys. + +==== Request + +`GET /_security/api_key` + +==== Description + +The information for the API keys created by <> can be retrieved +using this API. + +==== Request Body + +The following parameters can be specified in the query parameters of a GET request and +pertain to retrieving api keys: + +`id` (optional):: +(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or + `username` are used. + +`name` (optional):: +(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or + `username` are used. + +`realm_name` (optional):: +(string) The name of an authentication realm. This parameter cannot be used with either `id` or `name`. + +`username` (optional):: +(string) The username of a user. This parameter cannot be used with either `id` or `name`. + +NOTE: While all parameters are optional, at least one of them is required. + +==== Examples + +The following example to retrieve the API key identified by specified `id`: + +[source,js] +-------------------------------------------------- +GET /_security/api_key?id=dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ== +-------------------------------------------------- +// NOTCONSOLE + +whereas the following example to retrieve the API key identified by specified `name`: + +[source,js] +-------------------------------------------------- +GET /_security/api_key?name=hadoop_myuser_key +-------------------------------------------------- +// NOTCONSOLE + +The following example retrieves all API keys for the `native1` realm: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?realm_name=native1 +-------------------------------------------------- +// NOTCONSOLE + +The following example retrieves all API keys for the user `myuser` in all realms: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?username=myuser +-------------------------------------------------- +// NOTCONSOLE + +Finally, the following example retrieves all API keys for the user `myuser` in + the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +GET /_xpack/api_key?username=myuser&realm_name=native1 +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that contains the information of one or more API keys that were retrieved. + +[source,js] +-------------------------------------------------- +{ + "api_keys": [ <1> + { + "id": "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==", <2> + "name": "hadoop_myuser_key", <3> + "creation": 1548550550158, <4> + "expiration": 1548551550158, <5> + "invalidated": false, <6> + "username": "myuser", <7> + "realm": "native1" <8> + }, + { + "id": "api-key-id-2", + "name": "api-key-name-2", + "creation": 1548550550158, + "invalidated": false, + "username": "user-y", + "realm": "realm-2" + } + ] +} +-------------------------------------------------- +// NOTCONSOLE + +<1> The list of API keys that were retrieved for this request. +<2> Id for the API key +<3> Name of the API key +<4> Creation time for the API key in milliseconds +<5> optional expiration time for the API key in milliseconds +<6> invalidation status for the API key, `true` if the key has been invalidated else `false` +<7> principal for which this API key was created +<8> realm name of the principal for which this API key was created diff --git a/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc new file mode 100644 index 0000000000000..4809e267ebd80 --- /dev/null +++ b/x-pack/docs/en/rest-api/security/invalidate-api-keys.asciidoc @@ -0,0 +1,140 @@ +[role="xpack"] +[[security-api-invalidate-api-key]] +=== Invalidate API Key API +++++ +Invalidate API key +++++ + +Invalidates one or more API keys. + +==== Request + +`DELETE /_security/api_key` + +==== Description + +The API keys created by <> can be invalidated +using this API. + +==== Request Body + +The following parameters can be specified in the body of a DELETE request and +pertain to invalidating api keys: + +`id` (optional):: +(string) An API key id. This parameter cannot be used with any of `name`, `realm_name` or + `username` are used. + +`name` (optional):: +(string) An API key name. This parameter cannot be used with any of `id`, `realm_name` or + `username` are used. + +`realm_name` (optional):: +(string) The name of an authentication realm. This parameter cannot be used with either `api_key_id` or `api_key_name`. + +`username` (optional):: +(string) The username of a user. This parameter cannot be used with either `api_key_id` or `api_key_name`. + +NOTE: While all parameters are optional, at least one of them is required. + +==== Examples + +The following example invalidates the API key identified by specified `id` immediately: + +[source,js] +-------------------------------------------------- +DELETE /_security/api_key +{ + "id" : "dGhpcyBpcyBub3QgYSByZWFsIHRva2VuIGJ1dCBpdCBpcyBvbmx5IHRlc3QgZGF0YS4gZG8gbm90IHRyeSB0byByZWFkIHRva2VuIQ==" +} +-------------------------------------------------- +// NOTCONSOLE + +whereas the following example invalidates the API key identified by specified `name` immediately: + +[source,js] +-------------------------------------------------- +DELETE /_security/api_key +{ + "name" : "hadoop_myuser_key" +} +-------------------------------------------------- +// NOTCONSOLE + +The following example invalidates all API keys for the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "realm_name" : "native1" +} +-------------------------------------------------- +// NOTCONSOLE + +The following example invalidates all API keys for the user `myuser` in all realms immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "username" : "myuser" +} +-------------------------------------------------- +// NOTCONSOLE + +Finally, the following example invalidates all API keys for the user `myuser` in + the `native1` realm immediately: + +[source,js] +-------------------------------------------------- +DELETE /_xpack/api_key +{ + "username" : "myuser", + "realm_name" : "native1" +} +-------------------------------------------------- +// NOTCONSOLE + +A successful call returns a JSON structure that contains the ids of the API keys that were invalidated, the ids +of the API keys that had already been invalidated, and potentially a list of errors encountered while invalidating +specific api keys. + +[source,js] +-------------------------------------------------- +{ + "invalidated_api_keys": [ <1> + "api-key-id-1" + ], + "previously_invalidated_api_keys": [ <2> + "api-key-id-2", + "api-key-id-3" + ], + "error_count": 2, <3> + "error_details": [ <4> + { + "type": "exception", + "reason": "error occurred while invalidating api keys", + "caused_by": { + "type": "illegal_argument_exception", + "reason": "invalid api key id" + } + }, + { + "type": "exception", + "reason": "error occurred while invalidating api keys", + "caused_by": { + "type": "illegal_argument_exception", + "reason": "invalid api key id" + } + } + ] +} +-------------------------------------------------- +// NOTCONSOLE + +<1> The ids of the API keys that were invalidated as part of this request. +<2> The ids of the API keys that were already invalidated. +<3> The number of errors that were encountered when invalidating the API keys. +<4> Details about these errors. This field is not present in the response when + `error_count` is 0. diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 7d50b39e89776..3fcdcc4e27796 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -133,6 +133,7 @@ integTestCluster { setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setting 'xpack.security.authc.token.enabled', 'true' + setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.key', nodeKey.name setting 'xpack.security.transport.ssl.certificate', nodeCert.name diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java index 884d12afe6650..06a686541005d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java @@ -14,13 +14,13 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.client.Client; import org.elasticsearch.client.FilterClient; import org.elasticsearch.cluster.ClusterState; @@ -37,14 +37,15 @@ import org.elasticsearch.license.RemoteClusterLicenseChecker; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.ccr.action.ShardChangesAction; +import org.elasticsearch.xpack.ccr.action.ShardFollowTask; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesAction; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesRequest; import org.elasticsearch.xpack.core.security.action.user.HasPrivilegesResponse; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import org.elasticsearch.xpack.core.security.support.Exceptions; import java.util.Arrays; @@ -334,7 +335,7 @@ public void hasPrivilegesToFollowIndices(final Client remoteClient, final String message.append(indices.length == 1 ? " index " : " indices "); message.append(Arrays.toString(indices)); - HasPrivilegesResponse.ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next(); + ResourcePrivileges resourcePrivileges = response.getIndexPrivileges().iterator().next(); for (Map.Entry entry : resourcePrivileges.getPrivileges().entrySet()) { if (entry.getValue() == false) { message.append(", privilege for action ["); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index b16409b6a3ccf..10ea252f5cd5f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -136,6 +136,9 @@ import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.SecuritySettings; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheAction; import org.elasticsearch.xpack.core.security.action.role.ClearRolesCacheAction; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction; @@ -314,6 +317,9 @@ public List getClientActions() { InvalidateTokenAction.INSTANCE, GetCertificateInfoAction.INSTANCE, RefreshTokenAction.INSTANCE, + CreateApiKeyAction.INSTANCE, + InvalidateApiKeyAction.INSTANCE, + GetApiKeyAction.INSTANCE, // upgrade IndexUpgradeInfoAction.INSTANCE, IndexUpgradeAction.INSTANCE, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index 288811fc1af9b..89e572bff8237 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -100,7 +100,7 @@ private XPackSettings() { public static final Setting RESERVED_REALM_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.reserved_realm.enabled", true, Setting.Property.NodeScope); - /** Setting for enabling or disabling the token service. Defaults to true */ + /** Setting for enabling or disabling the token service. Defaults to the value of https being enabled */ public static final Setting TOKEN_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.token.enabled", (s) -> { if (NetworkModule.HTTP_ENABLED.get(s)) { return XPackSettings.HTTP_SSL_ENABLED.getRaw(s); @@ -109,6 +109,10 @@ private XPackSettings() { } }, Setting.Property.NodeScope); + /** Setting for enabling or disabling the api key service. Defaults to the value of https being enabled */ + public static final Setting API_KEY_SERVICE_ENABLED_SETTING = Setting.boolSetting("xpack.security.authc.api_key.enabled", + XPackSettings.HTTP_SSL_ENABLED::getRaw, Setting.Property.NodeScope); + /** Setting for enabling or disabling FIPS mode. Defaults to false */ public static final Setting FIPS_MODE_ENABLED = Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope); @@ -197,6 +201,7 @@ public static List> getAllSettings() { settings.add(HTTP_SSL_ENABLED); settings.add(RESERVED_REALM_ENABLED_SETTING); settings.add(TOKEN_SERVICE_ENABLED_SETTING); + settings.add(API_KEY_SERVICE_ENABLED_SETTING); settings.add(SQL_ENABLED); settings.add(USER_SETTING); settings.add(ROLLUP_ENABLED); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java index c737ab75d81aa..0da07a52996ad 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java @@ -13,9 +13,11 @@ import org.elasticsearch.common.util.concurrent.ThreadContext.StoredContext; import org.elasticsearch.node.Node; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.Authentication.AuthenticationType; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; +import java.util.Collections; import java.util.Objects; import java.util.function.Consumer; @@ -71,7 +73,8 @@ public void setUser(User user, Version version) { } else { lookedUpBy = null; } - setAuthentication(new Authentication(user, authenticatedBy, lookedUpBy, version)); + setAuthentication( + new Authentication(user, authenticatedBy, lookedUpBy, version, AuthenticationType.INTERNAL, Collections.emptyMap())); } /** Writes the authentication to the thread context */ @@ -89,7 +92,7 @@ private void setAuthentication(Authentication authentication) { */ public void executeAsUser(User user, Consumer consumer, Version version) { final StoredContext original = threadContext.newStoredContext(true); - try (ThreadContext.StoredContext ctx = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setUser(user, version); consumer.accept(original); } @@ -102,9 +105,9 @@ public void executeAsUser(User user, Consumer consumer, Version v public void executeAfterRewritingAuthentication(Consumer consumer, Version version) { final StoredContext original = threadContext.newStoredContext(true); final Authentication authentication = Objects.requireNonNull(userSettings.getAuthentication()); - try (ThreadContext.StoredContext ctx = threadContext.stashContext()) { + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setAuthentication(new Authentication(authentication.getUser(), authentication.getAuthenticatedBy(), - authentication.getLookedUpBy(), version)); + authentication.getLookedUpBy(), version, authentication.getAuthenticationType(), authentication.getMetadata())); consumer.accept(original); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java new file mode 100644 index 0000000000000..bfe9f523062a0 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ApiKey.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * API key information + */ +public final class ApiKey implements ToXContentObject, Writeable { + + private final String name; + private final String id; + private final Instant creation; + private final Instant expiration; + private final boolean invalidated; + private final String username; + private final String realm; + + public ApiKey(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, String realm) { + this.name = name; + this.id = id; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.creation = Instant.ofEpochMilli(creation.toEpochMilli()); + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + this.invalidated = invalidated; + this.username = username; + this.realm = realm; + } + + public ApiKey(StreamInput in) throws IOException { + this.name = in.readString(); + this.id = in.readString(); + this.creation = in.readInstant(); + this.expiration = in.readOptionalInstant(); + this.invalidated = in.readBoolean(); + this.username = in.readString(); + this.realm = in.readString(); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public Instant getCreation() { + return creation; + } + + public Instant getExpiration() { + return expiration; + } + + public boolean isInvalidated() { + return invalidated; + } + + public String getUsername() { + return username; + } + + public String getRealm() { + return realm; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("id", id) + .field("name", name) + .field("creation", creation.toEpochMilli()); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + builder.field("invalidated", invalidated) + .field("username", username) + .field("realm", realm); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(id); + out.writeInstant(creation); + out.writeOptionalInstant(expiration); + out.writeBoolean(invalidated); + out.writeString(username); + out.writeString(realm); + } + + @Override + public int hashCode() { + return Objects.hash(name, id, creation, expiration, invalidated, username, realm); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + ApiKey other = (ApiKey) obj; + return Objects.equals(name, other.name) + && Objects.equals(id, other.id) + && Objects.equals(creation, other.creation) + && Objects.equals(expiration, other.expiration) + && Objects.equals(invalidated, other.invalidated) + && Objects.equals(username, other.username) + && Objects.equals(realm, other.realm); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("api_key", args -> { + return new ApiKey((String) args[0], (String) args[1], Instant.ofEpochMilli((Long) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]), (Boolean) args[4], (String) args[5], (String) args[6]); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareLong(constructorArg(), new ParseField("creation")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + PARSER.declareBoolean(constructorArg(), new ParseField("invalidated")); + PARSER.declareString(constructorArg(), new ParseField("username")); + PARSER.declareString(constructorArg(), new ParseField("realm")); + } + + public static ApiKey fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "ApiKey [name=" + name + ", id=" + id + ", creation=" + creation + ", expiration=" + expiration + ", invalidated=" + + invalidated + ", username=" + username + ", realm=" + realm + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java new file mode 100644 index 0000000000000..52d290e10ca3a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for the creation of an API key + */ +public final class CreateApiKeyAction extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/create"; + public static final CreateApiKeyAction INSTANCE = new CreateApiKeyAction(); + + private CreateApiKeyAction() { + super(NAME); + } + + @Override + public CreateApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return CreateApiKeyResponse::new; + } + + @Override + public CreateApiKeyRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new CreateApiKeyRequestBuilder(client); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java new file mode 100644 index 0000000000000..c3f7ece21fc79 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequest.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request class used for the creation of an API key. The request requires a name to be provided + * and optionally an expiration time and permission limitation can be provided. + */ +public final class CreateApiKeyRequest extends ActionRequest { + public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL; + + private String name; + private TimeValue expiration; + private List roleDescriptors = Collections.emptyList(); + private WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY; + + public CreateApiKeyRequest() {} + + /** + * Create API Key request constructor + * @param name name for the API key + * @param roleDescriptors list of {@link RoleDescriptor}s + * @param expiration to specify expiration for the API key + */ + public CreateApiKeyRequest(String name, List roleDescriptors, @Nullable TimeValue expiration) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + this.roleDescriptors = Objects.requireNonNull(roleDescriptors, "role descriptors may not be null"); + this.expiration = expiration; + } + + public CreateApiKeyRequest(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.expiration = in.readOptionalTimeValue(); + this.roleDescriptors = Collections.unmodifiableList(in.readList(RoleDescriptor::new)); + this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in); + } + + public String getName() { + return name; + } + + public void setName(String name) { + if (Strings.hasText(name)) { + this.name = name; + } else { + throw new IllegalArgumentException("name must not be null or empty"); + } + } + + public TimeValue getExpiration() { + return expiration; + } + + public void setExpiration(TimeValue expiration) { + this.expiration = expiration; + } + + public List getRoleDescriptors() { + return roleDescriptors; + } + + public void setRoleDescriptors(List roleDescriptors) { + this.roleDescriptors = Collections.unmodifiableList(Objects.requireNonNull(roleDescriptors, "role descriptors may not be null")); + } + + public WriteRequest.RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null"); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.isNullOrEmpty(name)) { + validationException = addValidationError("name is required", validationException); + } else { + if (name.length() > 256) { + validationException = addValidationError("name may not be more than 256 characters long", validationException); + } + if (name.equals(name.trim()) == false) { + validationException = addValidationError("name may not begin or end with whitespace", validationException); + } + if (name.startsWith("_")) { + validationException = addValidationError("name may not begin with an underscore", validationException); + } + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeOptionalTimeValue(expiration); + out.writeList(roleDescriptors); + refreshPolicy.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + this.name = in.readString(); + this.expiration = in.readOptionalTimeValue(); + this.roleDescriptors = Collections.unmodifiableList(in.readList(RoleDescriptor::new)); + this.refreshPolicy = WriteRequest.RefreshPolicy.readFrom(in); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java new file mode 100644 index 0000000000000..e089ec826da17 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilder.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Request builder for populating a {@link CreateApiKeyRequest} + */ +public final class CreateApiKeyRequestBuilder + extends ActionRequestBuilder { + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "api_key_request", false, (args, v) -> { + return new CreateApiKeyRequest((String) args[0], (List) args[1], + TimeValue.parseTimeValue((String) args[2], null, "expiration")); + }); + + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareNamedObjects(constructorArg(), (p, c, n) -> { + p.nextToken(); + return RoleDescriptor.parse(n, p, false); + }, new ParseField("role_descriptors")); + PARSER.declareString(optionalConstructorArg(), new ParseField("expiration")); + } + + public CreateApiKeyRequestBuilder(ElasticsearchClient client) { + super(client, CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest()); + } + + public CreateApiKeyRequestBuilder setName(String name) { + request.setName(name); + return this; + } + + public CreateApiKeyRequestBuilder setExpiration(TimeValue expiration) { + request.setExpiration(expiration); + return this; + } + + public CreateApiKeyRequestBuilder setRoleDescriptors(List roleDescriptors) { + request.setRoleDescriptors(roleDescriptors); + return this; + } + + public CreateApiKeyRequestBuilder setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + request.setRefreshPolicy(refreshPolicy); + return this; + } + + public CreateApiKeyRequestBuilder source(BytesReference source, XContentType xContentType) throws IOException { + final NamedXContentRegistry registry = NamedXContentRegistry.EMPTY; + try (InputStream stream = source.streamInput(); + XContentParser parser = xContentType.xContent().createParser(registry, LoggingDeprecationHandler.INSTANCE, stream)) { + CreateApiKeyRequest createApiKeyRequest = PARSER.parse(parser, null); + setName(createApiKeyRequest.getName()); + setRoleDescriptors(createApiKeyRequest.getRoleDescriptors()); + setExpiration(createApiKeyRequest.getExpiration()); + } + return this; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java new file mode 100644 index 0000000000000..a774413c3c4a2 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponse.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.CharArrays; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for the successful creation of an api key + */ +public final class CreateApiKeyResponse extends ActionResponse implements ToXContentObject { + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("create_api_key_response", + args -> new CreateApiKeyResponse((String) args[0], (String) args[1], new SecureString((String) args[2]), + (args[3] == null) ? null : Instant.ofEpochMilli((Long) args[3]))); + static { + PARSER.declareString(constructorArg(), new ParseField("name")); + PARSER.declareString(constructorArg(), new ParseField("id")); + PARSER.declareString(constructorArg(), new ParseField("api_key")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiration")); + } + + private final String name; + private final String id; + private final SecureString key; + private final Instant expiration; + + public CreateApiKeyResponse(String name, String id, SecureString key, Instant expiration) { + this.name = name; + this.id = id; + this.key = key; + // As we do not yet support the nanosecond precision when we serialize to JSON, + // here creating the 'Instant' of milliseconds precision. + // This Instant can then be used for date comparison. + this.expiration = (expiration != null) ? Instant.ofEpochMilli(expiration.toEpochMilli()): null; + } + + public CreateApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.name = in.readString(); + this.id = in.readString(); + byte[] bytes = null; + try { + bytes = in.readByteArray(); + this.key = new SecureString(CharArrays.utf8BytesToChars(bytes)); + } finally { + if (bytes != null) { + Arrays.fill(bytes, (byte) 0); + } + } + this.expiration = in.readOptionalInstant(); + } + + public String getName() { + return name; + } + + public String getId() { + return id; + } + + public SecureString getKey() { + return key; + } + + @Nullable + public Instant getExpiration() { + return expiration; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((expiration == null) ? 0 : expiration.hashCode()); + result = prime * result + Objects.hash(id, name, key); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final CreateApiKeyResponse other = (CreateApiKeyResponse) obj; + if (expiration == null) { + if (other.expiration != null) + return false; + } else if (!Objects.equals(expiration, other.expiration)) + return false; + return Objects.equals(id, other.id) + && Objects.equals(key, other.key) + && Objects.equals(name, other.name); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(name); + out.writeString(id); + byte[] bytes = null; + try { + bytes = CharArrays.toUtf8Bytes(key.getChars()); + out.writeByteArray(bytes); + } finally { + if (bytes != null) { + Arrays.fill(bytes, (byte) 0); + } + } + out.writeOptionalInstant(expiration); + } + + @Override + public void readFrom(StreamInput in) { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + public static CreateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("id", id) + .field("name", name); + if (expiration != null) { + builder.field("expiration", expiration.toEpochMilli()); + } + byte[] charBytes = CharArrays.toUtf8Bytes(key.getChars()); + try { + builder.field("api_key").utf8Value(charBytes, 0, charBytes.length); + } finally { + Arrays.fill(charBytes, (byte) 0); + } + return builder.endObject(); + } + + @Override + public String toString() { + return "CreateApiKeyResponse [name=" + name + ", id=" + id + ", expiration=" + expiration + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java new file mode 100644 index 0000000000000..6729a23618ee6 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for retrieving API key(s) + */ +public final class GetApiKeyAction extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/get"; + public static final GetApiKeyAction INSTANCE = new GetApiKeyAction(); + + private GetApiKeyAction() { + super(NAME); + } + + @Override + public GetApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return GetApiKeyResponse::new; + } + + @Override + public GetApiKeyRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new GetApiKeyRequestBuilder(client); + } +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java new file mode 100644 index 0000000000000..819a64151d20a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequest.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request for get API key + */ +public final class GetApiKeyRequest extends ActionRequest { + + private String realmName; + private String userName; + private String apiKeyId; + private String apiKeyName; + + public GetApiKeyRequest() { + this(null, null, null, null); + } + + public GetApiKeyRequest(StreamInput in) throws IOException { + super(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + apiKeyId = in.readOptionalString(); + apiKeyName = in.readOptionalString(); + } + + public GetApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String apiKeyId, + @Nullable String apiKeyName) { + this.realmName = realmName; + this.userName = userName; + this.apiKeyId = apiKeyId; + this.apiKeyName = apiKeyName; + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getApiKeyId() { + return apiKeyId; + } + + public String getApiKeyName() { + return apiKeyName; + } + + /** + * Creates get API key request for given realm name + * @param realmName realm name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmName(String realmName) { + return new GetApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates get API key request for given user name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingUserName(String userName) { + return new GetApiKeyRequest(null, userName, null, null); + } + + /** + * Creates get API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new GetApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates get API key request for given api key id + * @param apiKeyId api key id + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyId(String apiKeyId) { + return new GetApiKeyRequest(null, null, apiKeyId, null); + } + + /** + * Creates get api key request for given api key name + * @param apiKeyName api key name + * @return {@link GetApiKeyRequest} + */ + public static GetApiKeyRequest usingApiKeyName(String apiKeyName) { + return new GetApiKeyRequest(null, null, null, apiKeyName); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(apiKeyId) == false + && Strings.hasText(apiKeyName) == false) { + validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified", + validationException); + } + if (Strings.hasText(apiKeyId) || Strings.hasText(apiKeyName)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + validationException = addValidationError( + "username or realm name must not be specified when the api key id or api key name is specified", + validationException); + } + } + if (Strings.hasText(apiKeyId) && Strings.hasText(apiKeyName)) { + validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realmName); + out.writeOptionalString(userName); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + apiKeyId = in.readOptionalString(); + apiKeyName = in.readOptionalString(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestBuilder.java new file mode 100644 index 0000000000000..9a8b0b2910f55 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestBuilder.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +/** + * Request builder for populating a {@link GetApiKeyRequest} + */ +public class GetApiKeyRequestBuilder extends ActionRequestBuilder { + + protected GetApiKeyRequestBuilder(ElasticsearchClient client) { + super(client, GetApiKeyAction.INSTANCE, new GetApiKeyRequest()); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java new file mode 100644 index 0000000000000..97b8f380f6940 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponse.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for get API keys.
+ * The result contains information about the API keys that were found. + */ +public final class GetApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { + + private final ApiKey[] foundApiKeysInfo; + + public GetApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.foundApiKeysInfo = in.readArray(ApiKey::new, ApiKey[]::new); + } + + public GetApiKeyResponse(Collection foundApiKeysInfo) { + Objects.requireNonNull(foundApiKeysInfo, "found_api_keys_info must be provided"); + this.foundApiKeysInfo = foundApiKeysInfo.toArray(new ApiKey[0]); + } + + public static GetApiKeyResponse emptyResponse() { + return new GetApiKeyResponse(Collections.emptyList()); + } + + public ApiKey[] getApiKeyInfos() { + return foundApiKeysInfo; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .array("api_keys", (Object[]) foundApiKeysInfo); + return builder.endObject(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeArray(foundApiKeysInfo); + } + + @SuppressWarnings("unchecked") + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("get_api_key_response", args -> { + return (args[0] == null) ? GetApiKeyResponse.emptyResponse() : new GetApiKeyResponse((List) args[0]); + }); + static { + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ApiKey.fromXContent(p), new ParseField("api_keys")); + } + + public static GetApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "GetApiKeyResponse [foundApiKeysInfo=" + foundApiKeysInfo + "]"; + } + +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java new file mode 100644 index 0000000000000..9cac055ed9351 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Action for invalidating API key + */ +public final class InvalidateApiKeyAction + extends Action { + + public static final String NAME = "cluster:admin/xpack/security/api_key/invalidate"; + public static final InvalidateApiKeyAction INSTANCE = new InvalidateApiKeyAction(); + + private InvalidateApiKeyAction() { + super(NAME); + } + + @Override + public InvalidateApiKeyResponse newResponse() { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public Writeable.Reader getResponseReader() { + return InvalidateApiKeyResponse::new; + } + + @Override + public InvalidateApiKeyRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new InvalidateApiKeyRequestBuilder(client); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java new file mode 100644 index 0000000000000..1f6939fa5a95c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequest.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * Request for invalidating API key(s) so that it can no longer be used + */ +public final class InvalidateApiKeyRequest extends ActionRequest { + + private String realmName; + private String userName; + private String id; + private String name; + + public InvalidateApiKeyRequest() { + this(null, null, null, null); + } + + public InvalidateApiKeyRequest(StreamInput in) throws IOException { + super(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + id = in.readOptionalString(); + name = in.readOptionalString(); + } + + public InvalidateApiKeyRequest(@Nullable String realmName, @Nullable String userName, @Nullable String id, + @Nullable String name) { + this.realmName = realmName; + this.userName = userName; + this.id = id; + this.name = name; + } + + public String getRealmName() { + return realmName; + } + + public String getUserName() { + return userName; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + /** + * Creates invalidate api key request for given realm name + * @param realmName realm name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmName(String realmName) { + return new InvalidateApiKeyRequest(realmName, null, null, null); + } + + /** + * Creates invalidate API key request for given user name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingUserName(String userName) { + return new InvalidateApiKeyRequest(null, userName, null, null); + } + + /** + * Creates invalidate API key request for given realm and user name + * @param realmName realm name + * @param userName user name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingRealmAndUserName(String realmName, String userName) { + return new InvalidateApiKeyRequest(realmName, userName, null, null); + } + + /** + * Creates invalidate API key request for given api key id + * @param id api key id + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyId(String id) { + return new InvalidateApiKeyRequest(null, null, id, null); + } + + /** + * Creates invalidate api key request for given api key name + * @param name api key name + * @return {@link InvalidateApiKeyRequest} + */ + public static InvalidateApiKeyRequest usingApiKeyName(String name) { + return new InvalidateApiKeyRequest(null, null, null, name); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (Strings.hasText(realmName) == false && Strings.hasText(userName) == false && Strings.hasText(id) == false + && Strings.hasText(name) == false) { + validationException = addValidationError("One of [api key id, api key name, username, realm name] must be specified", + validationException); + } + if (Strings.hasText(id) || Strings.hasText(name)) { + if (Strings.hasText(realmName) || Strings.hasText(userName)) { + validationException = addValidationError( + "username or realm name must not be specified when the api key id or api key name is specified", + validationException); + } + } + if (Strings.hasText(id) && Strings.hasText(name)) { + validationException = addValidationError("only one of [api key id, api key name] can be specified", validationException); + } + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realmName); + out.writeOptionalString(userName); + out.writeOptionalString(id); + out.writeOptionalString(name); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + realmName = in.readOptionalString(); + userName = in.readOptionalString(); + id = in.readOptionalString(); + name = in.readOptionalString(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestBuilder.java new file mode 100644 index 0000000000000..cb71c91e5ecef --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestBuilder.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +/** + * Request builder for populating a {@link InvalidateApiKeyRequest} + */ +public final class InvalidateApiKeyRequestBuilder + extends ActionRequestBuilder { + + protected InvalidateApiKeyRequestBuilder(ElasticsearchClient client) { + super(client, InvalidateApiKeyAction.INSTANCE, new InvalidateApiKeyRequest()); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java new file mode 100644 index 0000000000000..e9580c93d9086 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponse.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * Response for invalidation of one or more API keys result.
+ * The result contains information about: + *

+ */ +public final class InvalidateApiKeyResponse extends ActionResponse implements ToXContentObject, Writeable { + + private final List invalidatedApiKeys; + private final List previouslyInvalidatedApiKeys; + private final List errors; + + public InvalidateApiKeyResponse(StreamInput in) throws IOException { + super(in); + this.invalidatedApiKeys = in.readList(StreamInput::readString); + this.previouslyInvalidatedApiKeys = in.readList(StreamInput::readString); + this.errors = in.readList(StreamInput::readException); + } + + /** + * Constructor for API keys invalidation response + * @param invalidatedApiKeys list of invalidated API key ids + * @param previouslyInvalidatedApiKeys list of previously invalidated API key ids + * @param errors list of encountered errors while invalidating API keys + */ + public InvalidateApiKeyResponse(List invalidatedApiKeys, List previouslyInvalidatedApiKeys, + @Nullable List errors) { + this.invalidatedApiKeys = Objects.requireNonNull(invalidatedApiKeys, "invalidated_api_keys must be provided"); + this.previouslyInvalidatedApiKeys = Objects.requireNonNull(previouslyInvalidatedApiKeys, + "previously_invalidated_api_keys must be provided"); + if (null != errors) { + this.errors = errors; + } else { + this.errors = Collections.emptyList(); + } + } + + public static InvalidateApiKeyResponse emptyResponse() { + return new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + public List getInvalidatedApiKeys() { + return invalidatedApiKeys; + } + + public List getPreviouslyInvalidatedApiKeys() { + return previouslyInvalidatedApiKeys; + } + + public List getErrors() { + return errors; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .array("invalidated_api_keys", invalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .array("previously_invalidated_api_keys", previouslyInvalidatedApiKeys.toArray(Strings.EMPTY_ARRAY)) + .field("error_count", errors.size()); + if (errors.isEmpty() == false) { + builder.field("error_details"); + builder.startArray(); + for (ElasticsearchException e : errors) { + builder.startObject(); + ElasticsearchException.generateThrowableXContent(builder, params, e); + builder.endObject(); + } + builder.endArray(); + } + return builder.endObject(); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable"); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringCollection(invalidatedApiKeys); + out.writeStringCollection(previouslyInvalidatedApiKeys); + out.writeCollection(errors, StreamOutput::writeException); + } + + static ConstructingObjectParser PARSER = new ConstructingObjectParser<>("invalidate_api_key_response", + args -> { + return new InvalidateApiKeyResponse((List) args[0], (List) args[1], (List) args[3]); + }); + static { + PARSER.declareStringArray(constructorArg(), new ParseField("invalidated_api_keys")); + PARSER.declareStringArray(constructorArg(), new ParseField("previously_invalidated_api_keys")); + // we parse error_count but ignore it while constructing response + PARSER.declareInt(constructorArg(), new ParseField("error_count")); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error_details")); + } + + public static InvalidateApiKeyResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public String toString() { + return "InvalidateApiKeyResponse [invalidatedApiKeys=" + invalidatedApiKeys + ", previouslyInvalidatedApiKeys=" + + previouslyInvalidatedApiKeys + ", errors=" + errors + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java index 93c9d6bca9b64..27079eebcc36b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/GetRolesResponse.java @@ -37,7 +37,7 @@ public void readFrom(StreamInput in) throws IOException { int size = in.readVInt(); roles = new RoleDescriptor[size]; for (int i = 0; i < size; i++) { - roles[i] = RoleDescriptor.readFrom(in); + roles[i] = new RoleDescriptor(in); } } @@ -46,7 +46,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeVInt(roles.length); for (RoleDescriptor role : roles) { - RoleDescriptor.writeTo(role, out); + role.writeTo(out); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java index 71aafbb76187a..2885e702944ed 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import java.io.IOException; import java.util.Collection; @@ -49,7 +50,7 @@ public HasPrivilegesResponse(String username, boolean completeMatch, Map sorted(Collection resources) { - final Set set = new TreeSet<>(Comparator.comparing(o -> o.resource)); + final Set set = new TreeSet<>(Comparator.comparing(o -> o.getResource())); set.addAll(resources); return set; } @@ -116,11 +117,11 @@ public void readFrom(StreamInput in) throws IOException { private static Set readResourcePrivileges(StreamInput in) throws IOException { final int count = in.readVInt(); - final Set set = new TreeSet<>(Comparator.comparing(o -> o.resource)); + final Set set = new TreeSet<>(Comparator.comparing(o -> o.getResource())); for (int i = 0; i < count; i++) { final String index = in.readString(); final Map privileges = in.readMap(StreamInput::readString, StreamInput::readBoolean); - set.add(new ResourcePrivileges(index, privileges)); + set.add(ResourcePrivileges.builder(index).addPrivileges(privileges).build()); } return set; } @@ -144,8 +145,8 @@ public void writeTo(StreamOutput out) throws IOException { private static void writeResourcePrivileges(StreamOutput out, Set privileges) throws IOException { out.writeVInt(privileges.size()); for (ResourcePrivileges priv : privileges) { - out.writeString(priv.resource); - out.writeMap(priv.privileges, StreamOutput::writeString, StreamOutput::writeBoolean); + out.writeString(priv.getResource()); + out.writeMap(priv.getPrivileges(), StreamOutput::writeString, StreamOutput::writeBoolean); } } @@ -181,60 +182,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - private void appendResources(XContentBuilder builder, String field, Set privileges) + private void appendResources(XContentBuilder builder, String field, Set privileges) throws IOException { builder.startObject(field); - for (HasPrivilegesResponse.ResourcePrivileges privilege : privileges) { + for (ResourcePrivileges privilege : privileges) { builder.field(privilege.getResource()); builder.map(privilege.getPrivileges()); } builder.endObject(); } - - public static class ResourcePrivileges { - private final String resource; - private final Map privileges; - - public ResourcePrivileges(String resource, Map privileges) { - this.resource = Objects.requireNonNull(resource); - this.privileges = Collections.unmodifiableMap(privileges); - } - - public String getResource() { - return resource; - } - - public Map getPrivileges() { - return privileges; - } - - @Override - public String toString() { - return getClass().getSimpleName() + "{" + - "resource='" + resource + '\'' + - ", privileges=" + privileges + - '}'; - } - - @Override - public int hashCode() { - int result = resource.hashCode(); - result = 31 * result + privileges.hashCode(); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - final ResourcePrivileges other = (ResourcePrivileges) o; - return this.resource.equals(other.resource) && this.privileges.equals(other.privileges); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index b9dbe0a948ff2..de75d90eca51f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.util.Base64; +import java.util.Collections; +import java.util.Map; import java.util.Objects; // TODO(hub-cap) Clean this up after moving User over - This class can re-inherit its field AUTHENTICATION_KEY in AuthenticationField. @@ -28,16 +30,25 @@ public class Authentication implements ToXContentObject { private final RealmRef authenticatedBy; private final RealmRef lookedUpBy; private final Version version; + private final AuthenticationType type; + private final Map metadata; public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy) { this(user, authenticatedBy, lookedUpBy, Version.CURRENT); } public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version) { + this(user, authenticatedBy, lookedUpBy, version, AuthenticationType.REALM, Collections.emptyMap()); + } + + public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version, + AuthenticationType type, Map metadata) { this.user = Objects.requireNonNull(user); this.authenticatedBy = Objects.requireNonNull(authenticatedBy); this.lookedUpBy = lookedUpBy; this.version = version; + this.type = type; + this.metadata = metadata; } public Authentication(StreamInput in) throws IOException { @@ -49,6 +60,13 @@ public Authentication(StreamInput in) throws IOException { this.lookedUpBy = null; } this.version = in.getVersion(); + if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + type = AuthenticationType.values()[in.readVInt()]; + metadata = in.readMap(); + } else { + type = AuthenticationType.REALM; + metadata = Collections.emptyMap(); + } } public User getUser() { @@ -67,8 +85,15 @@ public Version getVersion() { return version; } - public static Authentication readFromContext(ThreadContext ctx) - throws IOException, IllegalArgumentException { + public AuthenticationType getAuthenticationType() { + return type; + } + + public Map getMetadata() { + return metadata; + } + + public static Authentication readFromContext(ThreadContext ctx) throws IOException, IllegalArgumentException { Authentication authentication = ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY); if (authentication != null) { assert ctx.getHeader(AuthenticationField.AUTHENTICATION_KEY) != null; @@ -107,8 +132,7 @@ public static Authentication decode(String header) throws IOException { * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an * {@link IllegalStateException} will be thrown */ - public void writeToContext(ThreadContext ctx) - throws IOException, IllegalArgumentException { + public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumentException { ensureContextDoesNotContainAuthentication(ctx); String header = encode(); ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, this); @@ -141,28 +165,28 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + out.writeVInt(type.ordinal()); + out.writeMap(metadata); + } } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Authentication that = (Authentication) o; - - if (!user.equals(that.user)) return false; - if (!authenticatedBy.equals(that.authenticatedBy)) return false; - if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false; - return version.equals(that.version); + return user.equals(that.user) && + authenticatedBy.equals(that.authenticatedBy) && + Objects.equals(lookedUpBy, that.lookedUpBy) && + version.equals(that.version) && + type == that.type && + metadata.equals(that.metadata); } @Override public int hashCode() { - int result = user.hashCode(); - result = 31 * result + authenticatedBy.hashCode(); - result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0); - result = 31 * result + version.hashCode(); - return result; + return Objects.hash(user, authenticatedBy, lookedUpBy, version, type, metadata); } @Override @@ -246,5 +270,13 @@ public int hashCode() { return result; } } + + public enum AuthenticationType { + REALM, + API_KEY, + TOKEN, + ANONYMOUS, + INTERNAL + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java index 736b9378e3876..a7c51b206db9c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandler.java @@ -77,10 +77,12 @@ private static Integer authSchemePriority(final String headerValue) { return 0; } else if (headerValue.regionMatches(true, 0, "bearer", 0, "bearer".length())) { return 1; - } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) { + } else if (headerValue.regionMatches(true, 0, "apikey", 0, "apikey".length())) { return 2; - } else { + } else if (headerValue.regionMatches(true, 0, "basic", 0, "basic".length())) { return 3; + } else { + return 4; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java index 7472a510c38b2..dc506881b1db1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java @@ -43,7 +43,7 @@ * A holder for a Role that contains user-readable information about the Role * without containing the actual Role object. */ -public class RoleDescriptor implements ToXContentObject { +public class RoleDescriptor implements ToXContentObject, Writeable { public static final String ROLE_TYPE = "role"; @@ -110,6 +110,27 @@ public RoleDescriptor(String name, Collections.singletonMap("enabled", true); } + public RoleDescriptor(StreamInput in) throws IOException { + this.name = in.readString(); + this.clusterPrivileges = in.readStringArray(); + int size = in.readVInt(); + this.indicesPrivileges = new IndicesPrivileges[size]; + for (int i = 0; i < size; i++) { + indicesPrivileges[i] = new IndicesPrivileges(in); + } + this.runAs = in.readStringArray(); + this.metadata = in.readMap(); + this.transientMetadata = in.readMap(); + + if (in.getVersion().onOrAfter(Version.V_6_4_0)) { + this.applicationPrivileges = in.readArray(ApplicationResourcePrivileges::new, ApplicationResourcePrivileges[]::new); + this.conditionalClusterPrivileges = ConditionalClusterPrivileges.readArray(in); + } else { + this.applicationPrivileges = ApplicationResourcePrivileges.NONE; + this.conditionalClusterPrivileges = ConditionalClusterPrivileges.EMPTY_ARRAY; + } + } + public String getName() { return this.name; } @@ -264,21 +285,20 @@ public static RoleDescriptor readFrom(StreamInput in) throws IOException { runAs, metadata, transientMetadata); } - public static void writeTo(RoleDescriptor descriptor, StreamOutput out) throws IOException { - out.writeString(descriptor.name); - out.writeStringArray(descriptor.clusterPrivileges); - out.writeVInt(descriptor.indicesPrivileges.length); - for (IndicesPrivileges group : descriptor.indicesPrivileges) { + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeStringArray(clusterPrivileges); + out.writeVInt(indicesPrivileges.length); + for (IndicesPrivileges group : indicesPrivileges) { group.writeTo(out); } - out.writeStringArray(descriptor.runAs); - out.writeMap(descriptor.metadata); - if (out.getVersion().onOrAfter(Version.V_5_2_0)) { - out.writeMap(descriptor.transientMetadata); - } + out.writeStringArray(runAs); + out.writeMap(metadata); + out.writeMap(transientMetadata); if (out.getVersion().onOrAfter(Version.V_6_4_0)) { - out.writeArray(ApplicationResourcePrivileges::write, descriptor.applicationPrivileges); - ConditionalClusterPrivileges.writeArray(out, descriptor.getConditionalClusterPrivileges()); + out.writeArray(ApplicationResourcePrivileges::write, applicationPrivileges); + ConditionalClusterPrivileges.writeArray(out, getConditionalClusterPrivileges()); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java index 6df9ad834c1e5..8cdf099e676d8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/IndicesAccessControl.java @@ -6,11 +6,13 @@ package org.elasticsearch.xpack.core.security.authz.accesscontrol; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.core.security.authz.IndicesAndAliasesResolverField; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -22,7 +24,7 @@ public class IndicesAccessControl { public static final IndicesAccessControl ALLOW_ALL = new IndicesAccessControl(true, Collections.emptyMap()); public static final IndicesAccessControl ALLOW_NO_INDICES = new IndicesAccessControl(true, Collections.singletonMap(IndicesAndAliasesResolverField.NO_INDEX_PLACEHOLDER, - new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), null))); + new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), DocumentPermissions.allowAll()))); private final boolean granted; private final Map indexPermissions; @@ -55,12 +57,12 @@ public static class IndexAccessControl { private final boolean granted; private final FieldPermissions fieldPermissions; - private final Set queries; + private final DocumentPermissions documentPermissions; - public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, Set queries) { + public IndexAccessControl(boolean granted, FieldPermissions fieldPermissions, DocumentPermissions documentPermissions) { this.granted = granted; - this.fieldPermissions = fieldPermissions; - this.queries = queries; + this.fieldPermissions = (fieldPermissions == null) ? FieldPermissions.DEFAULT : fieldPermissions; + this.documentPermissions = (documentPermissions == null) ? DocumentPermissions.allowAll() : documentPermissions; } /** @@ -82,8 +84,33 @@ public FieldPermissions getFieldPermissions() { * then this means that there are no document level restrictions */ @Nullable - public Set getQueries() { - return queries; + public DocumentPermissions getDocumentPermissions() { + return documentPermissions; + } + + /** + * Returns a instance of {@link IndexAccessControl}, where the privileges for {@code this} object are constrained by the privileges + * contained in the provided parameter.
+ * Allowed fields for this index permission would be an intersection of allowed fields.
+ * Allowed documents for this index permission would be an intersection of allowed documents.
+ * + * @param limitedByIndexAccessControl {@link IndexAccessControl} + * @return {@link IndexAccessControl} + * @see FieldPermissions#limitFieldPermissions(FieldPermissions) + * @see DocumentPermissions#limitDocumentPermissions(DocumentPermissions) + */ + public IndexAccessControl limitIndexAccessControl(IndexAccessControl limitedByIndexAccessControl) { + final boolean granted; + if (this.granted == limitedByIndexAccessControl.granted) { + granted = this.granted; + } else { + granted = false; + } + FieldPermissions fieldPermissions = getFieldPermissions().limitFieldPermissions( + limitedByIndexAccessControl.fieldPermissions); + DocumentPermissions documentPermissions = getDocumentPermissions() + .limitDocumentPermissions(limitedByIndexAccessControl.getDocumentPermissions()); + return new IndexAccessControl(granted, fieldPermissions, documentPermissions); } @Override @@ -91,11 +118,38 @@ public String toString() { return "IndexAccessControl{" + "granted=" + granted + ", fieldPermissions=" + fieldPermissions + - ", queries=" + queries + + ", documentPermissions=" + documentPermissions + '}'; } } + /** + * Returns a instance of {@link IndicesAccessControl}, where the privileges for {@code this} + * object are constrained by the privileges contained in the provided parameter.
+ * + * @param limitedByIndicesAccessControl {@link IndicesAccessControl} + * @return {@link IndicesAccessControl} + */ + public IndicesAccessControl limitIndicesAccessControl(IndicesAccessControl limitedByIndicesAccessControl) { + final boolean granted; + if (this.granted == limitedByIndicesAccessControl.granted) { + granted = this.granted; + } else { + granted = false; + } + Set indexes = indexPermissions.keySet(); + Set otherIndexes = limitedByIndicesAccessControl.indexPermissions.keySet(); + Set commonIndexes = Sets.intersection(indexes, otherIndexes); + + Map indexPermissions = new HashMap<>(commonIndexes.size()); + for (String index : commonIndexes) { + IndexAccessControl indexAccessControl = getIndexPermissions(index); + IndexAccessControl limitedByIndexAccessControl = limitedByIndicesAccessControl.getIndexPermissions(index); + indexPermissions.put(index, indexAccessControl.limitIndexAccessControl(limitedByIndexAccessControl)); + } + return new IndicesAccessControl(granted, indexPermissions); + } + @Override public String toString() { return "IndicesAccessControl{" + diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index 997db63c47423..1169cfff5794c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.core.security.authz.accesscontrol; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanQuery; @@ -18,64 +18,35 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.BitSetProducer; -import org.apache.lucene.search.join.ToChildBlockJoinQuery; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.SparseFixedBitSet; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.EngineException; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.BoostingQueryBuilder; -import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.GeoShapeQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.index.query.Rewriteable; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; -import org.elasticsearch.index.search.NestedHelper; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.script.TemplateScript; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.support.Exceptions; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Function; -import static org.apache.lucene.search.BooleanClause.Occur.FILTER; -import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; - /** * An {@link IndexSearcherWrapper} implementation that is used for field and document level security. *

@@ -107,7 +78,7 @@ public SecurityIndexSearcherWrapper(Function querySh } @Override - protected DirectoryReader wrap(DirectoryReader reader) { + protected DirectoryReader wrap(final DirectoryReader reader) { if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) { return reader; } @@ -120,47 +91,22 @@ protected DirectoryReader wrap(DirectoryReader reader) { throw new IllegalStateException(LoggerMessageFormat.format("couldn't extract shardId from reader [{}]", reader)); } - IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName()); + final IndicesAccessControl.IndexAccessControl permissions = indicesAccessControl.getIndexPermissions(shardId.getIndexName()); // No permissions have been defined for an index, so don't intercept the index reader for access control if (permissions == null) { return reader; } - if (permissions.getQueries() != null) { - BooleanQuery.Builder filter = new BooleanQuery.Builder(); - for (BytesReference bytesReference : permissions.getQueries()) { - QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId); - String templateResult = evaluateTemplate(bytesReference.utf8ToString()); - try (XContentParser parser = XContentFactory.xContent(templateResult) - .createParser(queryShardContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, templateResult)) { - QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser); - verifyRoleQuery(queryBuilder); - failIfQueryUsesClient(queryBuilder, queryShardContext); - Query roleQuery = queryShardContext.toFilter(queryBuilder).query(); - filter.add(roleQuery, SHOULD); - if (queryShardContext.getMapperService().hasNested()) { - NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService()); - if (nestedHelper.mightMatchNestedDocs(roleQuery)) { - roleQuery = new BooleanQuery.Builder() - .add(roleQuery, FILTER) - .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER) - .build(); - } - // If access is allowed on root doc then also access is allowed on all nested docs of that root document: - BitSetProducer rootDocs = queryShardContext.bitsetFilter( - Queries.newNonNestedFilter(queryShardContext.indexVersionCreated())); - ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs); - filter.add(includeNestedDocs, SHOULD); - } - } + DirectoryReader wrappedReader = reader; + DocumentPermissions documentPermissions = permissions.getDocumentPermissions(); + if (documentPermissions != null && documentPermissions.hasDocumentLevelPermissions()) { + BooleanQuery filterQuery = documentPermissions.filter(getUser(), scriptService, shardId, queryShardContextProvider); + if (filterQuery != null) { + wrappedReader = DocumentSubsetReader.wrap(wrappedReader, bitsetFilterCache, new ConstantScoreQuery(filterQuery)); } - - // at least one of the queries should match - filter.setMinimumNumberShouldMatch(1); - reader = DocumentSubsetReader.wrap(reader, bitsetFilterCache, new ConstantScoreQuery(filter.build())); } - return permissions.getFieldPermissions().filter(reader); + return permissions.getFieldPermissions().filter(wrappedReader); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); @@ -255,48 +201,6 @@ static void intersectScorerAndRoleBits(Scorer scorer, SparseFixedBitSet roleBits } } - String evaluateTemplate(String querySource) throws IOException { - // EMPTY is safe here because we never use namedObject - try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY, - LoggingDeprecationHandler.INSTANCE, querySource)) { - XContentParser.Token token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - token = parser.nextToken(); - if (token != XContentParser.Token.FIELD_NAME) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - if ("template".equals(parser.currentName())) { - token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("Unexpected token [" + token + "]"); - } - Script script = Script.parse(parser); - // Add the user details to the params - Map params = new HashMap<>(); - if (script.getParams() != null) { - params.putAll(script.getParams()); - } - User user = getUser(); - Map userModel = new HashMap<>(); - userModel.put("username", user.principal()); - userModel.put("full_name", user.fullName()); - userModel.put("email", user.email()); - userModel.put("roles", Arrays.asList(user.roles())); - userModel.put("metadata", Collections.unmodifiableMap(user.metadata())); - params.put("_user", userModel); - // Always enforce mustache script lang: - script = new Script(script.getType(), - script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(), script.getOptions(), params); - TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams()); - return compiledTemplate.execute(); - } else { - return querySource; - } - } - } - protected IndicesAccessControl getIndicesAccessControl() { IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); if (indicesAccessControl == null) { @@ -310,65 +214,4 @@ protected User getUser(){ return authentication.getUser(); } - /** - * Checks whether the role query contains queries we know can't be used as DLS role query. - */ - static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException { - if (queryBuilder instanceof TermsQueryBuilder) { - TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder; - if (termsQueryBuilder.termsLookup() != null) { - throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query"); - } - } else if (queryBuilder instanceof GeoShapeQueryBuilder) { - GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder; - if (geoShapeQueryBuilder.shape() == null) { - throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query"); - } - } else if (queryBuilder.getName().equals("percolate")) { - // actually only if percolate query is referring to an existing document then this is problematic, - // a normal percolate query does work. However we can't check that here as this query builder is inside - // another module. So we don't allow the entire percolate query. I don't think users would ever use - // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls. - throw new IllegalArgumentException("percolate query isn't support as part of a role query"); - } else if (queryBuilder.getName().equals("has_child")) { - throw new IllegalArgumentException("has_child query isn't support as part of a role query"); - } else if (queryBuilder.getName().equals("has_parent")) { - throw new IllegalArgumentException("has_parent query isn't support as part of a role query"); - } else if (queryBuilder instanceof BoolQueryBuilder) { - BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; - List clauses = new ArrayList<>(); - clauses.addAll(boolQueryBuilder.filter()); - clauses.addAll(boolQueryBuilder.must()); - clauses.addAll(boolQueryBuilder.mustNot()); - clauses.addAll(boolQueryBuilder.should()); - for (QueryBuilder clause : clauses) { - verifyRoleQuery(clause); - } - } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { - verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); - } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { - verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query()); - } else if (queryBuilder instanceof BoostingQueryBuilder) { - verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery()); - verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery()); - } - } - - /** - * Fall back validation that verifies that queries during rewrite don't use - * the client to make remote calls. In the case of DLS this can cause a dead - * lock if DLS is also applied on these remote calls. For example in the - * case of terms query with lookup, this can cause recursive execution of - * the DLS query until the get thread pool has been exhausted: - * https://github.com/elastic/x-plugins/issues/3145 - */ - static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original) - throws IOException { - QueryRewriteContext copy = new QueryRewriteContext( - original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis); - Rewriteable.rewrite(queryBuilder, copy); - if (copy.hasAsyncActions()) { - throw new IllegalStateException("role queries are not allowed to execute additional requests"); - } - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java index 073e92f7faf44..0cd4e8a8b0ddc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ApplicationPermission.java @@ -12,10 +12,12 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.support.Automatons; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -83,6 +85,40 @@ public boolean grants(ApplicationPrivilege other, String resource) { return matched; } + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder(); + for (String checkResource : checkForResources) { + for (String checkPrivilegeName : checkForPrivilegeNames) { + final Set nameSet = Collections.singleton(checkPrivilegeName); + final ApplicationPrivilege checkPrivilege = ApplicationPrivilege.get(applicationName, nameSet, storedPrivileges); + assert checkPrivilege.getApplication().equals(applicationName) : "Privilege " + checkPrivilege + " should have application " + + applicationName; + assert checkPrivilege.name().equals(nameSet) : "Privilege " + checkPrivilege + " should have name " + nameSet; + + if (grants(checkPrivilege, checkResource)) { + resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.TRUE); + } else { + resourcePrivilegesMapBuilder.addResourcePrivilege(checkResource, checkPrivilegeName, Boolean.FALSE); + } + } + } + return resourcePrivilegesMapBuilder.build(); + } + @Override public String toString() { return getClass().getSimpleName() + "{privileges=" + permissions + "}"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java index 3af016959d4ed..687798971399f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ClusterPermission.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.authz.permission; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; @@ -33,6 +34,10 @@ public ClusterPrivilege privilege() { public abstract boolean check(String action, TransportRequest request); + public boolean grants(ClusterPrivilege clusterPrivilege) { + return Operations.subsetOf(clusterPrivilege.getAutomaton(), this.privilege().getAutomaton()); + } + public abstract List> privileges(); /** @@ -111,5 +116,10 @@ public List> privileges() { public boolean check(String action, TransportRequest request) { return children.stream().anyMatch(p -> p.check(action, request)); } + + @Override + public boolean grants(ClusterPrivilege clusterPrivilege) { + return children.stream().anyMatch(p -> p.grants(clusterPrivilege)); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java new file mode 100644 index 0000000000000..08d754b4e5357 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java @@ -0,0 +1,262 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.BoostingQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.GeoShapeQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.Rewriteable; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.index.search.NestedHelper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.xpack.core.security.authz.support.SecurityQueryTemplateEvaluator; +import org.elasticsearch.xpack.core.security.user.User; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.lucene.search.BooleanClause.Occur.FILTER; +import static org.apache.lucene.search.BooleanClause.Occur.SHOULD; + +/** + * Stores document level permissions in the form queries that match all the accessible documents.
+ * The document level permissions may be limited by another set of queries in that case the limited + * queries are used as an additional filter. + */ +public final class DocumentPermissions { + private final Set queries; + private final Set limitedByQueries; + + private static DocumentPermissions ALLOW_ALL = new DocumentPermissions(); + + DocumentPermissions() { + this.queries = null; + this.limitedByQueries = null; + } + + DocumentPermissions(Set queries) { + this(queries, null); + } + + DocumentPermissions(Set queries, Set scopedByQueries) { + if (queries == null && scopedByQueries == null) { + throw new IllegalArgumentException("one of the queries or scoped queries must be provided"); + } + this.queries = (queries != null) ? Collections.unmodifiableSet(queries) : queries; + this.limitedByQueries = (scopedByQueries != null) ? Collections.unmodifiableSet(scopedByQueries) : scopedByQueries; + } + + public Set getQueries() { + return queries; + } + + public Set getLimitedByQueries() { + return limitedByQueries; + } + + /** + * @return {@code true} if either queries or scoped queries are present for document level security else returns {@code false} + */ + public boolean hasDocumentLevelPermissions() { + return queries != null || limitedByQueries != null; + } + + /** + * Creates a {@link BooleanQuery} to be used as filter to restrict access to documents.
+ * Document permission queries are used to create an boolean query.
+ * If the document permissions are limited, then there is an additional filter added restricting access to documents only allowed by the + * limited queries. + * + * @param user authenticated {@link User} + * @param scriptService {@link ScriptService} for evaluating query templates + * @param shardId {@link ShardId} + * @param queryShardContextProvider {@link QueryShardContext} + * @return {@link BooleanQuery} for the filter + * @throws IOException thrown if there is an exception during parsing + */ + public BooleanQuery filter(User user, ScriptService scriptService, ShardId shardId, + Function queryShardContextProvider) throws IOException { + if (hasDocumentLevelPermissions()) { + BooleanQuery.Builder filter; + if (queries != null && limitedByQueries != null) { + filter = new BooleanQuery.Builder(); + BooleanQuery.Builder scopedFilter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, scopedFilter); + filter.add(scopedFilter.build(), FILTER); + + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter); + } else if (queries != null) { + filter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, queries, filter); + } else if (limitedByQueries != null) { + filter = new BooleanQuery.Builder(); + buildRoleQuery(user, scriptService, shardId, queryShardContextProvider, limitedByQueries, filter); + } else { + return null; + } + return filter.build(); + } + return null; + } + + private static void buildRoleQuery(User user, ScriptService scriptService, ShardId shardId, + Function queryShardContextProvider, Set queries, + BooleanQuery.Builder filter) throws IOException { + for (BytesReference bytesReference : queries) { + QueryShardContext queryShardContext = queryShardContextProvider.apply(shardId); + String templateResult = SecurityQueryTemplateEvaluator.evaluateTemplate(bytesReference.utf8ToString(), scriptService, user); + try (XContentParser parser = XContentFactory.xContent(templateResult).createParser(queryShardContext.getXContentRegistry(), + LoggingDeprecationHandler.INSTANCE, templateResult)) { + QueryBuilder queryBuilder = queryShardContext.parseInnerQueryBuilder(parser); + verifyRoleQuery(queryBuilder); + failIfQueryUsesClient(queryBuilder, queryShardContext); + Query roleQuery = queryShardContext.toQuery(queryBuilder).query(); + filter.add(roleQuery, SHOULD); + if (queryShardContext.getMapperService().hasNested()) { + NestedHelper nestedHelper = new NestedHelper(queryShardContext.getMapperService()); + if (nestedHelper.mightMatchNestedDocs(roleQuery)) { + roleQuery = new BooleanQuery.Builder().add(roleQuery, FILTER) + .add(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated()), FILTER).build(); + } + // If access is allowed on root doc then also access is allowed on all nested docs of that root document: + BitSetProducer rootDocs = queryShardContext + .bitsetFilter(Queries.newNonNestedFilter(queryShardContext.indexVersionCreated())); + ToChildBlockJoinQuery includeNestedDocs = new ToChildBlockJoinQuery(roleQuery, rootDocs); + filter.add(includeNestedDocs, SHOULD); + } + } + } + // at least one of the queries should match + filter.setMinimumNumberShouldMatch(1); + } + + /** + * Checks whether the role query contains queries we know can't be used as DLS role query. + */ + static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException { + if (queryBuilder instanceof TermsQueryBuilder) { + TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) queryBuilder; + if (termsQueryBuilder.termsLookup() != null) { + throw new IllegalArgumentException("terms query with terms lookup isn't supported as part of a role query"); + } + } else if (queryBuilder instanceof GeoShapeQueryBuilder) { + GeoShapeQueryBuilder geoShapeQueryBuilder = (GeoShapeQueryBuilder) queryBuilder; + if (geoShapeQueryBuilder.shape() == null) { + throw new IllegalArgumentException("geoshape query referring to indexed shapes isn't support as part of a role query"); + } + } else if (queryBuilder.getName().equals("percolate")) { + // actually only if percolate query is referring to an existing document then this is problematic, + // a normal percolate query does work. However we can't check that here as this query builder is inside + // another module. So we don't allow the entire percolate query. I don't think users would ever use + // a percolate query as role query, so this restriction shouldn't prohibit anyone from using dls. + throw new IllegalArgumentException("percolate query isn't support as part of a role query"); + } else if (queryBuilder.getName().equals("has_child")) { + throw new IllegalArgumentException("has_child query isn't support as part of a role query"); + } else if (queryBuilder.getName().equals("has_parent")) { + throw new IllegalArgumentException("has_parent query isn't support as part of a role query"); + } else if (queryBuilder instanceof BoolQueryBuilder) { + BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder; + List clauses = new ArrayList<>(); + clauses.addAll(boolQueryBuilder.filter()); + clauses.addAll(boolQueryBuilder.must()); + clauses.addAll(boolQueryBuilder.mustNot()); + clauses.addAll(boolQueryBuilder.should()); + for (QueryBuilder clause : clauses) { + verifyRoleQuery(clause); + } + } else if (queryBuilder instanceof ConstantScoreQueryBuilder) { + verifyRoleQuery(((ConstantScoreQueryBuilder) queryBuilder).innerQuery()); + } else if (queryBuilder instanceof FunctionScoreQueryBuilder) { + verifyRoleQuery(((FunctionScoreQueryBuilder) queryBuilder).query()); + } else if (queryBuilder instanceof BoostingQueryBuilder) { + verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).negativeQuery()); + verifyRoleQuery(((BoostingQueryBuilder) queryBuilder).positiveQuery()); + } + } + + /** + * Fall back validation that verifies that queries during rewrite don't use + * the client to make remote calls. In the case of DLS this can cause a dead + * lock if DLS is also applied on these remote calls. For example in the + * case of terms query with lookup, this can cause recursive execution of + * the DLS query until the get thread pool has been exhausted: + * https://github.com/elastic/x-plugins/issues/3145 + */ + static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original) + throws IOException { + QueryRewriteContext copy = new QueryRewriteContext( + original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis); + Rewriteable.rewrite(queryBuilder, copy); + if (copy.hasAsyncActions()) { + throw new IllegalStateException("role queries are not allowed to execute additional requests"); + } + } + + /** + * Create {@link DocumentPermissions} for given set of queries + * @param queries set of queries + * @return {@link DocumentPermissions} + */ + public static DocumentPermissions filteredBy(Set queries) { + if (queries == null || queries.isEmpty()) { + throw new IllegalArgumentException("null or empty queries not permitted"); + } + return new DocumentPermissions(queries); + } + + /** + * Create {@link DocumentPermissions} with no restriction. The {@link #getQueries()} + * will return {@code null} in this case and {@link #hasDocumentLevelPermissions()} + * will be {@code false} + * @return {@link DocumentPermissions} + */ + public static DocumentPermissions allowAll() { + return ALLOW_ALL; + } + + /** + * Create a document permissions, where the permissions for {@code this} are + * limited by the queries from other document permissions.
+ * + * @param limitedByDocumentPermissions {@link DocumentPermissions} used to limit the document level access + * @return instance of {@link DocumentPermissions} + */ + public DocumentPermissions limitDocumentPermissions( + DocumentPermissions limitedByDocumentPermissions) { + assert limitedByQueries == null + && limitedByDocumentPermissions.limitedByQueries == null : "nested scoping for document permissions is not permitted"; + if (queries == null && limitedByDocumentPermissions.queries == null) { + return DocumentPermissions.allowAll(); + } + return new DocumentPermissions(queries, limitedByDocumentPermissions.queries); + } + + @Override + public String toString() { + return "DocumentPermissions [queries=" + queries + ", scopedByQueries=" + limitedByQueries + "]"; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index e7dd9d2be4c88..53ea785913557 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -93,13 +93,15 @@ public FieldPermissions(FieldPermissionsDefinition fieldPermissionsDefinition) { long ramBytesUsed = BASE_FIELD_PERM_DEF_BYTES; - for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) { - ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE; - if (group.getGrantedFields() != null) { - ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields()); - } - if (group.getExcludedFields() != null) { - ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields()); + if (fieldPermissionsDefinition != null) { + for (FieldGrantExcludeGroup group : fieldPermissionsDefinition.getFieldGrantExcludeGroups()) { + ramBytesUsed += BASE_FIELD_GROUP_BYTES + BASE_HASHSET_ENTRY_SIZE; + if (group.getGrantedFields() != null) { + ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getGrantedFields()); + } + if (group.getExcludedFields() != null) { + ramBytesUsed += RamUsageEstimator.shallowSizeOf(group.getExcludedFields()); + } } } ramBytesUsed += permittedFieldsAutomaton.ramBytesUsed(); @@ -170,6 +172,28 @@ private static boolean containsAllField(String[] fields) { return fields != null && Arrays.stream(fields).anyMatch(AllFieldMapper.NAME::equals); } + /** + * Returns a field permissions instance where it is limited by the given field permissions.
+ * If the current and the other field permissions have field level security then it takes + * an intersection of permitted fields.
+ * If none of the permissions have field level security enabled, then returns permissions + * instance where all fields are allowed. + * + * @param limitedBy {@link FieldPermissions} used to limit current field permissions + * @return {@link FieldPermissions} + */ + public FieldPermissions limitFieldPermissions(FieldPermissions limitedBy) { + if (hasFieldLevelSecurity() && limitedBy != null && limitedBy.hasFieldLevelSecurity()) { + Automaton permittedFieldsAutomaton = Automatons.intersectAndMinimize(getIncludeAutomaton(), limitedBy.getIncludeAutomaton()); + return new FieldPermissions(null, permittedFieldsAutomaton); + } else if (limitedBy != null && limitedBy.hasFieldLevelSecurity()) { + return new FieldPermissions(limitedBy.getFieldPermissionsDefinition(), limitedBy.getIncludeAutomaton()); + } else if (hasFieldLevelSecurity()) { + return new FieldPermissions(getFieldPermissionsDefinition(), getIncludeAutomaton()); + } + return FieldPermissions.DEFAULT; + } + /** * Returns true if this field permission policy allows access to the field and false if not. * fieldName can be a wildcard. @@ -195,7 +219,6 @@ public DirectoryReader filter(DirectoryReader reader) throws IOException { return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton); } - // for testing only Automaton getIncludeAutomaton() { return originalAutomaton; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 4c2a479721a2a..da8c3701ed300 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.cluster.metadata.AliasOrIndex; @@ -23,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -123,6 +125,49 @@ public boolean check(String action) { return false; } + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkResourcePrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + final ResourcePrivilegesMap.Builder resourcePrivilegesMapBuilder = ResourcePrivilegesMap.builder(); + final Map predicateCache = new HashMap<>(); + for (String forIndexPattern : checkForIndexPatterns) { + final Automaton checkIndexAutomaton = IndicesPermission.Group.buildIndexMatcherAutomaton(allowRestrictedIndices, + forIndexPattern); + Automaton allowedIndexPrivilegesAutomaton = null; + for (Group group : groups) { + final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group, + g -> IndicesPermission.Group.buildIndexMatcherAutomaton(g.allowRestrictedIndices(), g.indices())); + if (Operations.subsetOf(checkIndexAutomaton, groupIndexAutomaton)) { + if (allowedIndexPrivilegesAutomaton != null) { + allowedIndexPrivilegesAutomaton = Automatons + .unionAndMinimize(Arrays.asList(allowedIndexPrivilegesAutomaton, group.privilege().getAutomaton())); + } else { + allowedIndexPrivilegesAutomaton = group.privilege().getAutomaton(); + } + } + } + for (String privilege : checkForPrivileges) { + IndexPrivilege indexPrivilege = IndexPrivilege.get(Collections.singleton(privilege)); + if (allowedIndexPrivilegesAutomaton != null + && Operations.subsetOf(indexPrivilege.getAutomaton(), allowedIndexPrivilegesAutomaton)) { + resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.TRUE); + } else { + resourcePrivilegesMapBuilder.addResourcePrivilege(forIndexPattern, privilege, Boolean.FALSE); + } + } + } + return resourcePrivilegesMapBuilder.build(); + } + public Automaton allowedActionsMatcher(String index) { List automatonList = new ArrayList<>(); for (Group group : groups) { @@ -207,7 +252,8 @@ public Map authorize(String act } else { fieldPermissions = FieldPermissions.DEFAULT; } - indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions, roleQueries)); + indexPermissions.put(index, new IndicesAccessControl.IndexAccessControl(entry.getValue(), fieldPermissions, + (roleQueries != null) ? DocumentPermissions.filteredBy(roleQueries) : DocumentPermissions.allowAll())); } return unmodifiableMap(indexPermissions); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java new file mode 100644 index 0000000000000..809b95965340e --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; +import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; + +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; + +/** + * A {@link Role} limited by another role.
+ * The effective permissions returned on {@link #authorize(String, Set, MetaData, FieldPermissionsCache)} call would be limited by the + * provided role. + */ +public final class LimitedRole extends Role { + private final Role limitedBy; + + LimitedRole(String[] names, ClusterPermission cluster, IndicesPermission indices, ApplicationPermission application, + RunAsPermission runAs, Role limitedBy) { + super(names, cluster, indices, application, runAs); + assert limitedBy != null : "limiting role is required"; + this.limitedBy = limitedBy; + } + + public Role limitedBy() { + return limitedBy; + } + + @Override + public IndicesAccessControl authorize(String action, Set requestedIndicesOrAliases, MetaData metaData, + FieldPermissionsCache fieldPermissionsCache) { + IndicesAccessControl indicesAccessControl = super.authorize(action, requestedIndicesOrAliases, metaData, fieldPermissionsCache); + IndicesAccessControl limitedByIndicesAccessControl = limitedBy.authorize(action, requestedIndicesOrAliases, metaData, + fieldPermissionsCache); + + return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl); + } + + /** + * @return A predicate that will match all the indices that this role and the limited by role has the privilege for executing the given + * action on. + */ + @Override + public Predicate allowedIndicesMatcher(String action) { + Predicate predicate = indices().allowedIndicesMatcher(action); + predicate = predicate.and(limitedBy.indices().allowedIndicesMatcher(action)); + return predicate; + } + + /** + * Check if indices permissions allow for the given action, also checks whether the limited by role allows the given actions + * + * @param action indices action + * @return {@code true} if action is allowed else returns {@code false} + */ + @Override + public boolean checkIndicesAction(String action) { + return super.checkIndicesAction(action) && limitedBy.checkIndicesAction(action); + } + + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not.
+ * This one takes intersection of resource privileges with the resource privileges from the limited-by role. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + @Override + public ResourcePrivilegesMap checkIndicesPrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + ResourcePrivilegesMap resourcePrivilegesMap = super.indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, + checkForPrivileges); + ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.indices().checkResourcePrivileges(checkForIndexPatterns, + allowRestrictedIndices, checkForPrivileges); + return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole); + } + + /** + * Check if cluster permissions allow for the given action, also checks whether the limited by role allows the given actions + * + * @param action cluster action + * @param request {@link TransportRequest} + * @return {@code true} if action is allowed else returns {@code false} + */ + @Override + public boolean checkClusterAction(String action, TransportRequest request) { + return super.checkClusterAction(action, request) && limitedBy.checkClusterAction(action, request); + } + + /** + * Check if cluster permissions grants the given cluster privilege, also checks whether the limited by role grants the given cluster + * privilege + * + * @param clusterPrivilege cluster privilege + * @return {@code true} if cluster privilege is allowed else returns {@code false} + */ + @Override + public boolean grants(ClusterPrivilege clusterPrivilege) { + return super.grants(clusterPrivilege) && limitedBy.grants(clusterPrivilege); + } + + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not.
+ * This one takes intersection of resource privileges with the resource privileges from the limited-by role. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + @Override + public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + ResourcePrivilegesMap resourcePrivilegesMap = super.application().checkResourcePrivileges(applicationName, checkForResources, + checkForPrivilegeNames, storedPrivileges); + ResourcePrivilegesMap resourcePrivilegesMapForLimitedRole = limitedBy.application().checkResourcePrivileges(applicationName, + checkForResources, checkForPrivilegeNames, storedPrivileges); + return ResourcePrivilegesMap.intersection(resourcePrivilegesMap, resourcePrivilegesMapForLimitedRole); + } + + /** + * Create a new role defined by given role and the limited role. + * + * @param fromRole existing role {@link Role} + * @param limitedByRole restrict the newly formed role to the permissions defined by this limited {@link Role} + * @return {@link LimitedRole} + */ + public static LimitedRole createLimitedRole(Role fromRole, Role limitedByRole) { + Objects.requireNonNull(limitedByRole, "limited by role is required to create limited role"); + return new LimitedRole(fromRole.names(), fromRole.cluster(), fromRole.indices(), fromRole.application(), fromRole.runAs(), + limitedByRole); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java new file mode 100644 index 0000000000000..3c64cc4afa8a1 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivileges.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; + +/** + * A generic structure to encapsulate resource to privileges map. + */ +public final class ResourcePrivileges { + + private final String resource; + private final Map privileges; + + ResourcePrivileges(String resource, Map privileges) { + this.resource = Objects.requireNonNull(resource); + this.privileges = Collections.unmodifiableMap(privileges); + } + + public String getResource() { + return resource; + } + + public Map getPrivileges() { + return privileges; + } + + public boolean isAllowed(String privilege) { + return Boolean.TRUE.equals(privileges.get(privilege)); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "{" + "resource='" + resource + '\'' + ", privileges=" + privileges + '}'; + } + + @Override + public int hashCode() { + int result = resource.hashCode(); + result = 31 * result + privileges.hashCode(); + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + final ResourcePrivileges other = (ResourcePrivileges) o; + return this.resource.equals(other.resource) && this.privileges.equals(other.privileges); + } + + public static Builder builder(String resource) { + return new Builder(resource); + } + + public static final class Builder { + private final String resource; + private Map privileges = new HashMap<>(); + + private Builder(String resource) { + this.resource = resource; + } + + public Builder addPrivilege(String privilege, Boolean allowed) { + this.privileges.compute(privilege, (k, v) -> ((v == null) ? allowed : v && allowed)); + return this; + } + + public Builder addPrivileges(Map privileges) { + for (Entry entry : privileges.entrySet()) { + addPrivilege(entry.getKey(), entry.getValue()); + } + return this; + } + + public ResourcePrivileges build() { + return new ResourcePrivileges(resource, privileges); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java new file mode 100644 index 0000000000000..814a6ed29d39f --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/ResourcePrivilegesMap.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.permission; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * A generic structure to encapsulate resources to {@link ResourcePrivileges}. Also keeps track of whether the resource privileges allow + * permissions to all resources. + */ +public final class ResourcePrivilegesMap { + + private final boolean allAllowed; + private final Map resourceToResourcePrivileges; + + public ResourcePrivilegesMap(boolean allAllowed, Map resToResPriv) { + this.allAllowed = allAllowed; + this.resourceToResourcePrivileges = Collections.unmodifiableMap(Objects.requireNonNull(resToResPriv)); + } + + public boolean allAllowed() { + return allAllowed; + } + + public Map getResourceToResourcePrivileges() { + return resourceToResourcePrivileges; + } + + @Override + public int hashCode() { + return Objects.hash(allAllowed, resourceToResourcePrivileges); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final ResourcePrivilegesMap other = (ResourcePrivilegesMap) obj; + return allAllowed == other.allAllowed && Objects.equals(resourceToResourcePrivileges, other.resourceToResourcePrivileges); + } + + @Override + public String toString() { + return "ResourcePrivilegesMap [allAllowed=" + allAllowed + ", resourceToResourcePrivileges=" + resourceToResourcePrivileges + "]"; + } + + public static final class Builder { + private boolean allowAll = true; + private Map resourceToResourcePrivilegesBuilder = new LinkedHashMap<>(); + + public Builder addResourcePrivilege(String resource, String privilege, Boolean allowed) { + assert resource != null && privilege != null + && allowed != null : "resource, privilege and permission(allowed or denied) are required"; + ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder); + builder.addPrivilege(privilege, allowed); + allowAll = allowAll && allowed; + return this; + } + + public Builder addResourcePrivilege(String resource, Map privilegePermissions) { + assert resource != null && privilegePermissions != null : "resource, privilege permissions(allowed or denied) are required"; + ResourcePrivileges.Builder builder = resourceToResourcePrivilegesBuilder.computeIfAbsent(resource, ResourcePrivileges::builder); + builder.addPrivileges(privilegePermissions); + allowAll = allowAll && privilegePermissions.values().stream().allMatch(b -> Boolean.TRUE.equals(b)); + return this; + } + + public Builder addResourcePrivilegesMap(ResourcePrivilegesMap resourcePrivilegesMap) { + resourcePrivilegesMap.getResourceToResourcePrivileges().entrySet().stream() + .forEach(e -> this.addResourcePrivilege(e.getKey(), e.getValue().getPrivileges())); + return this; + } + + public ResourcePrivilegesMap build() { + Map result = resourceToResourcePrivilegesBuilder.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().build())); + return new ResourcePrivilegesMap(allowAll, result); + } + } + + public static Builder builder() { + return new Builder(); + } + + /** + * Takes an intersection of resource privileges and returns a new instance of {@link ResourcePrivilegesMap}. If one of the resource + * privileges map does not allow access to a resource then the resulting map would also not allow access. + * + * @param left an instance of {@link ResourcePrivilegesMap} + * @param right an instance of {@link ResourcePrivilegesMap} + * @return a new instance of {@link ResourcePrivilegesMap}, an intersection of resource privileges. + */ + public static ResourcePrivilegesMap intersection(final ResourcePrivilegesMap left, final ResourcePrivilegesMap right) { + Objects.requireNonNull(left); + Objects.requireNonNull(right); + final ResourcePrivilegesMap.Builder builder = ResourcePrivilegesMap.builder(); + for (Entry leftResPrivsEntry : left.getResourceToResourcePrivileges().entrySet()) { + final ResourcePrivileges leftResPrivs = leftResPrivsEntry.getValue(); + final ResourcePrivileges rightResPrivs = right.getResourceToResourcePrivileges().get(leftResPrivsEntry.getKey()); + builder.addResourcePrivilege(leftResPrivsEntry.getKey(), leftResPrivs.getPrivileges()); + builder.addResourcePrivilege(leftResPrivsEntry.getKey(), rightResPrivs.getPrivileges()); + } + return builder.build(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java index f01869b4ea8dc..c63e8049193af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java @@ -10,9 +10,11 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege; +import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivilege; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; @@ -20,13 +22,15 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Predicate; -public final class Role { +public class Role { public static final Role EMPTY = Role.builder("__empty").build(); @@ -44,6 +48,7 @@ public final class Role { this.runAs = Objects.requireNonNull(runAs); } + public String[] names() { return names; } @@ -76,6 +81,79 @@ public static Builder builder(RoleDescriptor rd, FieldPermissionsCache fieldPerm return new Builder(rd, fieldPermissionsCache); } + /** + * @return A predicate that will match all the indices that this role + * has the privilege for executing the given action on. + */ + public Predicate allowedIndicesMatcher(String action) { + return indices().allowedIndicesMatcher(action); + } + + /** + * Check if indices permissions allow for the given action + * + * @param action indices action + * @return {@code true} if action is allowed else returns {@code false} + */ + public boolean checkIndicesAction(String action) { + return indices().check(action); + } + + + /** + * For given index patterns and index privileges determines allowed privileges and creates an instance of {@link ResourcePrivilegesMap} + * holding a map of resource to {@link ResourcePrivileges} where resource is index pattern and the map of index privilege to whether it + * is allowed or not. + * + * @param checkForIndexPatterns check permission grants for the set of index patterns + * @param allowRestrictedIndices if {@code true} then checks permission grants even for restricted indices by index matching + * @param checkForPrivileges check permission grants for the set of index privileges + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkIndicesPrivileges(Set checkForIndexPatterns, boolean allowRestrictedIndices, + Set checkForPrivileges) { + return indices().checkResourcePrivileges(checkForIndexPatterns, allowRestrictedIndices, checkForPrivileges); + } + + /** + * Check if cluster permissions allow for the given action + * + * @param action cluster action + * @param request {@link TransportRequest} + * @return {@code true} if action is allowed else returns {@code false} + */ + public boolean checkClusterAction(String action, TransportRequest request) { + return cluster().check(action, request); + } + + /** + * Check if cluster permissions grants the given cluster privilege + * + * @param clusterPrivilege cluster privilege + * @return {@code true} if cluster privilege is allowed else returns {@code false} + */ + public boolean grants(ClusterPrivilege clusterPrivilege) { + return cluster().grants(clusterPrivilege); + } + + /** + * For a given application, checks for the privileges for resources and returns an instance of {@link ResourcePrivilegesMap} holding a + * map of resource to {@link ResourcePrivileges} where the resource is application resource and the map of application privilege to + * whether it is allowed or not. + * + * @param applicationName checks privileges for the provided application name + * @param checkForResources check permission grants for the set of resources + * @param checkForPrivilegeNames check permission grants for the set of privilege names + * @param storedPrivileges stored {@link ApplicationPrivilegeDescriptor} for an application against which the access checks are + * performed + * @return an instance of {@link ResourcePrivilegesMap} + */ + public ResourcePrivilegesMap checkApplicationResourcePrivileges(final String applicationName, Set checkForResources, + Set checkForPrivilegeNames, + Collection storedPrivileges) { + return application().checkResourcePrivileges(applicationName, checkForResources, checkForPrivilegeNames, storedPrivileges); + } + /** * Returns whether at least one group encapsulated by this indices permissions is authorized to execute the * specified action with the requested indices/aliases. At the same time if field and/or document level security @@ -211,4 +289,5 @@ static Tuple> convertApplicationPrivilege(Stri ), Sets.newHashSet(arp.getResources())); } } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java new file mode 100644 index 0000000000000..951c4acf10d0d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluator.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.authz.support; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.script.TemplateScript; +import org.elasticsearch.xpack.core.security.user.User; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Helper class that helps to evaluate the query source template. + */ +public final class SecurityQueryTemplateEvaluator { + + private SecurityQueryTemplateEvaluator() { + } + + /** + * If the query source is a template, then parses the script, compiles the + * script with user details parameters and then executes it to return the + * query string. + *

+ * Note: This method always enforces "mustache" script language for the + * template. + * + * @param querySource query string template to be evaluated. + * @param scriptService {@link ScriptService} + * @param user {@link User} details for user defined parameters in the + * script. + * @return resultant query string after compiling and executing the script. + * If the source does not contain template then it will return the query + * source without any modifications. + * @throws IOException thrown when there is any error parsing the query + * string. + */ + public static String evaluateTemplate(final String querySource, final ScriptService scriptService, final User user) throws IOException { + // EMPTY is safe here because we never use namedObject + try (XContentParser parser = XContentFactory.xContent(querySource).createParser(NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, querySource)) { + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + token = parser.nextToken(); + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + if ("template".equals(parser.currentName())) { + token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("Unexpected token [" + token + "]"); + } + Script script = Script.parse(parser); + // Add the user details to the params + Map params = new HashMap<>(); + if (script.getParams() != null) { + params.putAll(script.getParams()); + } + Map userModel = new HashMap<>(); + userModel.put("username", user.principal()); + userModel.put("full_name", user.fullName()); + userModel.put("email", user.email()); + userModel.put("roles", Arrays.asList(user.roles())); + userModel.put("metadata", Collections.unmodifiableMap(user.metadata())); + params.put("_user", userModel); + // Always enforce mustache script lang: + script = new Script(script.getType(), script.getType() == ScriptType.STORED ? null : "mustache", script.getIdOrCode(), + script.getOptions(), params); + TemplateScript compiledTemplate = scriptService.compile(script, TemplateScript.CONTEXT).newInstance(script.getParams()); + return compiledTemplate.execute(); + } else { + return querySource; + } + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java index 709f12ecf498e..55026c553ccb0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/client/SecurityClient.java @@ -10,6 +10,16 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequestBuilder; +import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.GetApiKeyAction; +import org.elasticsearch.xpack.core.security.action.GetApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.GetApiKeyResponse; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyAction; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyRequest; +import org.elasticsearch.xpack.core.security.action.InvalidateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction; import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesRequestBuilder; import org.elasticsearch.xpack.core.security.action.privilege.GetPrivilegesAction; @@ -337,6 +347,27 @@ public void invalidateToken(InvalidateTokenRequest request, ActionListener listener) { + client.execute(CreateApiKeyAction.INSTANCE, request, listener); + } + + public void invalidateApiKey(InvalidateApiKeyRequest request, ActionListener listener) { + client.execute(InvalidateApiKeyAction.INSTANCE, request, listener); + } + + public void getApiKey(GetApiKeyRequest request, ActionListener listener) { + client.execute(GetApiKeyAction.INSTANCE, request, listener); + } + public SamlAuthenticateRequestBuilder prepareSamlAuthenticate(byte[] xmlContent, List validIds) { final SamlAuthenticateRequestBuilder builder = new SamlAuthenticateRequestBuilder(client); builder.saml(xmlContent); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java index 87a0099580b5f..7e6fd7ca46283 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/Automatons.java @@ -26,6 +26,7 @@ import static org.apache.lucene.util.automaton.MinimizationOperations.minimize; import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; import static org.apache.lucene.util.automaton.Operations.concatenate; +import static org.apache.lucene.util.automaton.Operations.intersection; import static org.apache.lucene.util.automaton.Operations.minus; import static org.apache.lucene.util.automaton.Operations.union; import static org.elasticsearch.common.Strings.collectionToDelimitedString; @@ -173,6 +174,11 @@ public static Automaton minusAndMinimize(Automaton a1, Automaton a2) { return minimize(res, maxDeterminizedStates); } + public static Automaton intersectAndMinimize(Automaton a1, Automaton a2) { + Automaton res = intersection(a1, a2); + return minimize(res, maxDeterminizedStates); + } + public static Predicate predicate(String... patterns) { return predicate(Arrays.asList(patterns)); } diff --git a/x-pack/plugin/core/src/main/resources/security-index-template.json b/x-pack/plugin/core/src/main/resources/security-index-template.json index 3723aff9054de..183ffff4ea534 100644 --- a/x-pack/plugin/core/src/main/resources/security-index-template.json +++ b/x-pack/plugin/core/src/main/resources/security-index-template.json @@ -152,6 +152,40 @@ "type" : "date", "format" : "epoch_millis" }, + "api_key_hash" : { + "type" : "keyword", + "index": false, + "doc_values": false + }, + "api_key_invalidated" : { + "type" : "boolean" + }, + "role_descriptors" : { + "type" : "object", + "enabled": false + }, + "limited_by_role_descriptors" : { + "type" : "object", + "enabled": false + }, + "version" : { + "type" : "integer" + }, + "creator" : { + "type" : "object", + "properties" : { + "principal" : { + "type": "keyword" + }, + "metadata" : { + "type" : "object", + "dynamic" : true + }, + "realm" : { + "type" : "keyword" + } + } + }, "rules" : { "type" : "object", "dynamic" : true diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java new file mode 100644 index 0000000000000..fb4f87089e8e7 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestBuilderTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class CreateApiKeyRequestBuilderTests extends ESTestCase { + + public void testParserAndCreateApiRequestBuilder() throws IOException { + boolean withExpiration = randomBoolean(); + final String json = "{ \"name\" : \"my-api-key\", " + + ((withExpiration) ? " \"expiration\": \"1d\", " : "") + +" \"role_descriptors\": { \"role-a\": {\"cluster\":[\"a-1\", \"a-2\"]," + + " \"index\": [{\"names\": [\"indx-a\"], \"privileges\": [\"read\"] }] }, " + + " \"role-b\": {\"cluster\":[\"b\"]," + + " \"index\": [{\"names\": [\"indx-b\"], \"privileges\": [\"read\"] }] } " + + "} }"; + final BytesArray source = new BytesArray(json); + final NodeClient mockClient = mock(NodeClient.class); + final CreateApiKeyRequest request = new CreateApiKeyRequestBuilder(mockClient).source(source, XContentType.JSON).request(); + final List actualRoleDescriptors = request.getRoleDescriptors(); + assertThat(request.getName(), equalTo("my-api-key")); + assertThat(actualRoleDescriptors.size(), is(2)); + for (RoleDescriptor rd : actualRoleDescriptors) { + String[] clusters = null; + IndicesPrivileges indicesPrivileges = null; + if (rd.getName().equals("role-a")) { + clusters = new String[] { "a-1", "a-2" }; + indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-a").privileges("read").build(); + } else if (rd.getName().equals("role-b")){ + clusters = new String[] { "b" }; + indicesPrivileges = RoleDescriptor.IndicesPrivileges.builder().indices("indx-b").privileges("read").build(); + } else { + fail("unexpected role name"); + } + assertThat(rd.getClusterPrivileges(), arrayContainingInAnyOrder(clusters)); + assertThat(rd.getIndicesPrivileges(), + arrayContainingInAnyOrder(indicesPrivileges)); + } + if (withExpiration) { + assertThat(request.getExpiration(), equalTo(TimeValue.parseTimeValue("1d", "expiration"))); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java new file mode 100644 index 0000000000000..654d56b42130e --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyRequestTests.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +public class CreateApiKeyRequestTests extends ESTestCase { + + public void testNameValidation() { + final String name = randomAlphaOfLengthBetween(1, 256); + CreateApiKeyRequest request = new CreateApiKeyRequest(); + + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name is required")); + + request.setName(name); + ve = request.validate(); + assertNull(ve); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> request.setName("")); + assertThat(e.getMessage(), containsString("name must not be null or empty")); + + e = expectThrows(IllegalArgumentException.class, () -> request.setName(null)); + assertThat(e.getMessage(), containsString("name must not be null or empty")); + + request.setName(randomAlphaOfLength(257)); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not be more than 256 characters long")); + + request.setName(" leading space"); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName("trailing space "); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName(" leading and trailing space "); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin or end with whitespace")); + + request.setName("inner space"); + ve = request.validate(); + assertNull(ve); + + request.setName("_foo"); + ve = request.validate(); + assertNotNull(ve); + assertThat(ve.validationErrors().size(), is(1)); + assertThat(ve.validationErrors().get(0), containsString("name may not begin with an underscore")); + } + + public void testSerialization() throws IOException { + final String name = randomAlphaOfLengthBetween(1, 256); + final TimeValue expiration = randomBoolean() ? null : + TimeValue.parseTimeValue(randomTimeValue(), "test serialization of create api key"); + final WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values()); + final int numDescriptors = randomIntBetween(0, 4); + final List descriptorList = new ArrayList<>(); + for (int i = 0; i < numDescriptors; i++) { + descriptorList.add(new RoleDescriptor("role_" + i, new String[] { "all" }, null, null)); + } + + final CreateApiKeyRequest request = new CreateApiKeyRequest(); + request.setName(name); + request.setExpiration(expiration); + + if (refreshPolicy != request.getRefreshPolicy() || randomBoolean()) { + request.setRefreshPolicy(refreshPolicy); + } + if (descriptorList.isEmpty() == false || randomBoolean()) { + request.setRoleDescriptors(descriptorList); + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + request.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + final CreateApiKeyRequest serialized = new CreateApiKeyRequest(in); + assertEquals(name, serialized.getName()); + assertEquals(expiration, serialized.getExpiration()); + assertEquals(refreshPolicy, serialized.getRefreshPolicy()); + assertEquals(descriptorList, serialized.getRoleDescriptors()); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java new file mode 100644 index 0000000000000..20ff4bc251d15 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/CreateApiKeyResponseTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static org.hamcrest.Matchers.equalTo; + +public class CreateApiKeyResponseTests extends AbstractXContentTestCase { + + @Override + protected CreateApiKeyResponse doParseInstance(XContentParser parser) throws IOException { + return CreateApiKeyResponse.fromXContent(parser); + } + + @Override + protected CreateApiKeyResponse createTestInstance() { + final String name = randomAlphaOfLengthBetween(1, 256); + final SecureString key = new SecureString(UUIDs.randomBase64UUID().toCharArray()); + final Instant expiration = randomBoolean() ? Instant.now().plus(7L, ChronoUnit.DAYS) : null; + final String id = randomAlphaOfLength(100); + return new CreateApiKeyResponse(name, id, key, expiration); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public void testSerialization() throws IOException { + final CreateApiKeyResponse response = createTestInstance(); + try (BytesStreamOutput out = new BytesStreamOutput()) { + response.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + CreateApiKeyResponse serialized = new CreateApiKeyResponse(in); + assertThat(serialized, equalTo(response)); + } + } + } + + public void testEqualsHashCode() { + CreateApiKeyResponse createApiKeyResponse = createTestInstance(); + + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }); + EqualsHashCodeTestUtils.checkEqualsAndHashCode(createApiKeyResponse, (original) -> { + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), original.getExpiration()); + }, CreateApiKeyResponseTests::mutateTestItem); + } + + private static CreateApiKeyResponse mutateTestItem(CreateApiKeyResponse original) { + switch (randomIntBetween(0, 3)) { + case 0: + return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration()); + case 1: + return new CreateApiKeyResponse(original.getName(), randomAlphaOfLength(5), original.getKey(), original.getExpiration()); + case 2: + return new CreateApiKeyResponse(original.getName(), original.getId(), new SecureString(UUIDs.randomBase64UUID().toCharArray()), + original.getExpiration()); + case 3: + return new CreateApiKeyResponse(original.getName(), original.getId(), original.getKey(), Instant.now()); + default: + return new CreateApiKeyResponse(randomAlphaOfLength(5), original.getId(), original.getKey(), original.getExpiration()); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java new file mode 100644 index 0000000000000..27be0d88eb82c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyRequestTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class GetApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + GetApiKeyRequest request = GetApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + ActionRequestValidationException ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = GetApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertNull(ve); + } + + public void testRequestValidationFailureScenarios() throws IOException { + class Dummy extends ActionRequest { + String realm; + String user; + String apiKeyId; + String apiKeyName; + + Dummy(String[] a) { + realm = a[0]; + user = a[1]; + apiKeyId = a[2]; + apiKeyName = a[3]; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realm); + out.writeOptionalString(user); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + } + + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified" }, + { "only one of [api key id, api key name] can be specified" } }; + + for (int caseNo = 0; caseNo < inputs.length; caseNo++) { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) { + Dummy d = new Dummy(inputs[caseNo]); + d.writeTo(osso); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + InputStreamStreamInput issi = new InputStreamStreamInput(bis); + + GetApiKeyRequest request = new GetApiKeyRequest(issi); + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size()); + assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo])); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java new file mode 100644 index 0000000000000..c278c135edaf8 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/GetApiKeyResponseTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class GetApiKeyResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + boolean withExpiration = randomBoolean(); + ApiKey apiKeyInfo = createApiKeyInfo(randomAlphaOfLength(4), randomAlphaOfLength(5), Instant.now(), + (withExpiration) ? Instant.now() : null, false, randomAlphaOfLength(4), randomAlphaOfLength(5)); + GetApiKeyResponse response = new GetApiKeyResponse(Collections.singletonList(apiKeyInfo)); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + GetApiKeyResponse serialized = new GetApiKeyResponse(input); + assertThat(serialized.getApiKeyInfos(), equalTo(response.getApiKeyInfos())); + } + } + } + + public void testToXContent() throws IOException { + ApiKey apiKeyInfo1 = createApiKeyInfo("name1", "id-1", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), false, + "user-a", "realm-x"); + ApiKey apiKeyInfo2 = createApiKeyInfo("name2", "id-2", Instant.ofEpochMilli(100000L), Instant.ofEpochMilli(10000000L), true, + "user-b", "realm-y"); + GetApiKeyResponse response = new GetApiKeyResponse(Arrays.asList(apiKeyInfo1, apiKeyInfo2)); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), equalTo( + "{" + + "\"api_keys\":[" + + "{\"id\":\"id-1\",\"name\":\"name1\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":false," + + "\"username\":\"user-a\",\"realm\":\"realm-x\"}," + + "{\"id\":\"id-2\",\"name\":\"name2\",\"creation\":100000,\"expiration\":10000000,\"invalidated\":true," + + "\"username\":\"user-b\",\"realm\":\"realm-y\"}" + + "]" + + "}")); + } + + private ApiKey createApiKeyInfo(String name, String id, Instant creation, Instant expiration, boolean invalidated, String username, + String realm) { + return new ApiKey(name, id, creation, expiration, invalidated, username, realm); + } +} + diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java new file mode 100644 index 0000000000000..3d7fd90234286 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyRequestTests.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class InvalidateApiKeyRequestTests extends ESTestCase { + + public void testRequestValidation() { + InvalidateApiKeyRequest request = InvalidateApiKeyRequest.usingApiKeyId(randomAlphaOfLength(5)); + ActionRequestValidationException ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingApiKeyName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingRealmName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingUserName(randomAlphaOfLength(5)); + ve = request.validate(); + assertNull(ve); + request = InvalidateApiKeyRequest.usingRealmAndUserName(randomAlphaOfLength(5), randomAlphaOfLength(7)); + ve = request.validate(); + assertNull(ve); + } + + public void testRequestValidationFailureScenarios() throws IOException { + class Dummy extends ActionRequest { + String realm; + String user; + String apiKeyId; + String apiKeyName; + + Dummy(String[] a) { + realm = a[0]; + user = a[1]; + apiKeyId = a[2]; + apiKeyName = a[3]; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(realm); + out.writeOptionalString(user); + out.writeOptionalString(apiKeyId); + out.writeOptionalString(apiKeyName); + } + } + + String[][] inputs = new String[][] { + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), + randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), "user", "api-kid", "api-kname" }, + { "realm", randomFrom(new String[] { null, "" }), "api-kid", "api-kname" }, + { "realm", "user", "api-kid", randomFrom(new String[] { null, "" }) }, + { randomFrom(new String[] { null, "" }), randomFrom(new String[] { null, "" }), "api-kid", "api-kname" } }; + String[][] expectedErrorMessages = new String[][] { { "One of [api key id, api key name, username, realm name] must be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified", + "only one of [api key id, api key name] can be specified" }, + { "username or realm name must not be specified when the api key id or api key name is specified" }, + { "only one of [api key id, api key name] can be specified" } }; + + + for (int caseNo = 0; caseNo < inputs.length; caseNo++) { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + OutputStreamStreamOutput osso = new OutputStreamStreamOutput(bos)) { + Dummy d = new Dummy(inputs[caseNo]); + d.writeTo(osso); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + InputStreamStreamInput issi = new InputStreamStreamInput(bis); + + InvalidateApiKeyRequest request = new InvalidateApiKeyRequest(issi); + ActionRequestValidationException ve = request.validate(); + assertNotNull(ve); + assertEquals(expectedErrorMessages[caseNo].length, ve.validationErrors().size()); + assertThat(ve.validationErrors(), containsInAnyOrder(expectedErrorMessages[caseNo])); + } + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java new file mode 100644 index 0000000000000..f4606a4f20f1b --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/InvalidateApiKeyResponseTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.security.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class InvalidateApiKeyResponseTests extends ESTestCase { + + public void testSerialization() throws IOException { + InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"), + Arrays.asList("api-key-id-2", "api-key-id-3"), + Arrays.asList(new ElasticsearchException("error1"), + new ElasticsearchException("error2"))); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input); + assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys())); + assertThat(serialized.getPreviouslyInvalidatedApiKeys(), + equalTo(response.getPreviouslyInvalidatedApiKeys())); + assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size())); + assertThat(serialized.getErrors().get(0).toString(), containsString("error1")); + assertThat(serialized.getErrors().get(1).toString(), containsString("error2")); + } + } + + response = new InvalidateApiKeyResponse(Arrays.asList(generateRandomStringArray(20, 15, false)), + Arrays.asList(generateRandomStringArray(20, 15, false)), + Collections.emptyList()); + try (BytesStreamOutput output = new BytesStreamOutput()) { + response.writeTo(output); + try (StreamInput input = output.bytes().streamInput()) { + InvalidateApiKeyResponse serialized = new InvalidateApiKeyResponse(input); + assertThat(serialized.getInvalidatedApiKeys(), equalTo(response.getInvalidatedApiKeys())); + assertThat(serialized.getPreviouslyInvalidatedApiKeys(), + equalTo(response.getPreviouslyInvalidatedApiKeys())); + assertThat(serialized.getErrors().size(), equalTo(response.getErrors().size())); + } + } + } + + public void testToXContent() throws IOException { + InvalidateApiKeyResponse response = new InvalidateApiKeyResponse(Arrays.asList("api-key-id-1"), + Arrays.asList("api-key-id-2", "api-key-id-3"), + Arrays.asList(new ElasticsearchException("error1", new IllegalArgumentException("msg - 1")), + new ElasticsearchException("error2", new IllegalArgumentException("msg - 2")))); + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertThat(Strings.toString(builder), + equalTo("{" + + "\"invalidated_api_keys\":[\"api-key-id-1\"]," + + "\"previously_invalidated_api_keys\":[\"api-key-id-2\",\"api-key-id-3\"]," + + "\"error_count\":2," + + "\"error_details\":[" + + "{\"type\":\"exception\"," + + "\"reason\":\"error1\"," + + "\"caused_by\":{" + + "\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"msg - 1\"}" + + "}," + + "{\"type\":\"exception\"," + + "\"reason\":\"error2\"," + + "\"caused_by\":" + + "{\"type\":\"illegal_argument_exception\"," + + "\"reason\":\"msg - 2\"}" + + "}" + + "]" + + "}")); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java index 0481e01e74ac3..a605917f01c2d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesResponseTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.protocol.AbstractHlrcStreamableXContentTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xpack.core.security.authz.permission.ResourcePrivileges; import org.hamcrest.Matchers; import java.io.IOException; @@ -59,16 +60,17 @@ public void testSerializationV63() throws IOException { } public void testToXContent() throws Exception { - final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false, - Collections.singletonMap("manage", true), - Arrays.asList( - new HasPrivilegesResponse.ResourcePrivileges("staff", - MapBuilder.newMapBuilder(new LinkedHashMap<>()) - .put("read", true).put("index", true).put("delete", false).put("manage", false).map()), - new HasPrivilegesResponse.ResourcePrivileges("customers", - MapBuilder.newMapBuilder(new LinkedHashMap<>()) - .put("read", true).put("index", true).put("delete", true).put("manage", false).map()) - ), Collections.emptyMap()); + final HasPrivilegesResponse response = new HasPrivilegesResponse("daredevil", false, Collections.singletonMap("manage", true), + Arrays.asList( + ResourcePrivileges.builder("staff") + .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<>()).put("read", true) + .put("index", true).put("delete", false).put("manage", false).map()) + .build(), + ResourcePrivileges.builder("customers") + .addPrivileges(MapBuilder.newMapBuilder(new LinkedHashMap<>()).put("read", true) + .put("index", true).put("delete", true).put("manage", false).map()) + .build()), + Collections.emptyMap()); final XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); response.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -120,9 +122,9 @@ public HasPrivilegesResponse convertHlrcToInternal(org.elasticsearch.client.secu ); } - private static List toResourcePrivileges(Map> map) { + private static List toResourcePrivileges(Map> map) { return map.entrySet().stream() - .map(e -> new HasPrivilegesResponse.ResourcePrivileges(e.getKey(), e.getValue())) + .map(e -> ResourcePrivileges.builder(e.getKey()).addPrivileges(e.getValue()).build()) .collect(Collectors.toList()); } @@ -146,23 +148,23 @@ private HasPrivilegesResponse randomResponse() { for (String priv : randomArray(1, 6, String[]::new, () -> randomAlphaOfLengthBetween(3, 12))) { cluster.put(priv, randomBoolean()); } - final Collection index = randomResourcePrivileges(); - final Map> application = new HashMap<>(); + final Collection index = randomResourcePrivileges(); + final Map> application = new HashMap<>(); for (String app : randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(3, 6).toLowerCase(Locale.ROOT))) { application.put(app, randomResourcePrivileges()); } return new HasPrivilegesResponse(username, randomBoolean(), cluster, index, application); } - private Collection randomResourcePrivileges() { - final Collection list = new ArrayList<>(); + private Collection randomResourcePrivileges() { + final Collection list = new ArrayList<>(); // Use hash set to force a unique set of resources for (String resource : Sets.newHashSet(randomArray(1, 3, String[]::new, () -> randomAlphaOfLengthBetween(2, 6)))) { final Map privileges = new HashMap<>(); for (String priv : randomArray(1, 5, String[]::new, () -> randomAlphaOfLengthBetween(3, 8))) { privileges.put(priv, randomBoolean()); } - list.add(new HasPrivilegesResponse.ResourcePrivileges(resource, privileges)); + list.add(ResourcePrivileges.builder(resource).addPrivileges(privileges).build()); } return list; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java index 15593f0b82ea5..24f9d16324f2d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/DefaultAuthenticationFailureHandlerTests.java @@ -111,8 +111,9 @@ public void testSortsWWWAuthenticateHeaderValues() { final String basicAuthScheme = "Basic realm=\"" + XPackField.SECURITY + "\" charset=\"UTF-8\""; final String bearerAuthScheme = "Bearer realm=\"" + XPackField.SECURITY + "\""; final String negotiateAuthScheme = randomFrom("Negotiate", "Negotiate Ijoijksdk"); + final String apiKeyAuthScheme = "ApiKey"; final Map> failureResponeHeaders = new HashMap<>(); - final List supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme); + final List supportedSchemes = Arrays.asList(basicAuthScheme, bearerAuthScheme, negotiateAuthScheme, apiKeyAuthScheme); Collections.shuffle(supportedSchemes, random()); failureResponeHeaders.put("WWW-Authenticate", supportedSchemes); final DefaultAuthenticationFailureHandler failuerHandler = new DefaultAuthenticationFailureHandler(failureResponeHeaders); @@ -123,7 +124,7 @@ public void testSortsWWWAuthenticateHeaderValues() { assertThat(ese, is(notNullValue())); assertThat(ese.getHeader("WWW-Authenticate"), is(notNullValue())); assertThat(ese.getMessage(), equalTo("error attempting to authenticate request")); - assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, basicAuthScheme); + assertWWWAuthenticateWithSchemes(ese, negotiateAuthScheme, bearerAuthScheme, apiKeyAuthScheme, basicAuthScheme); } private void assertWWWAuthenticateWithSchemes(final ElasticsearchSecurityException ese, final String... schemes) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java index 72b6c5b628ee7..2b6c269018fae 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperIntegrationTests.java @@ -8,6 +8,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -16,12 +17,14 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Accountable; import org.elasticsearch.client.Client; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -36,14 +39,21 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractBuilderTestCase; import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationField; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; +import org.elasticsearch.xpack.core.security.user.User; import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; @@ -53,7 +63,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -public class SecurityIndexSearcherWrapperIntegrationTests extends ESTestCase { +public class SecurityIndexSearcherWrapperIntegrationTests extends AbstractBuilderTestCase { public void testDLS() throws Exception { ShardId shardId = new ShardId("_index", "_na_", 0); @@ -64,9 +74,12 @@ public void testDLS() throws Exception { .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0])); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + final Authentication authentication = mock(Authentication.class); + when(authentication.getUser()).thenReturn(mock(User.class)); + threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new FieldPermissions(), - singleton(new BytesArray("{\"match_all\" : {}}"))); + DocumentPermissions.filteredBy(singleton(new BytesArray("{\"match_all\" : {}}")))); IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -159,4 +172,116 @@ protected IndicesAccessControl getIndicesAccessControl() { directoryReader.close(); directory.close(); } + + public void testDLSWithLimitedPermissions() throws Exception { + ShardId shardId = new ShardId("_index", "_na_", 0); + MapperService mapperService = mock(MapperService.class); + ScriptService scriptService = mock(ScriptService.class); + when(mapperService.documentMapper(any(String.class))).thenReturn(null); + when(mapperService.simpleMatchToFullName(anyString())) + .then(invocationOnMock -> Collections.singletonList((String) invocationOnMock.getArguments()[0])); + + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + final Authentication authentication = mock(Authentication.class); + when(authentication.getUser()).thenReturn(mock(User.class)); + threadContext.putTransient(AuthenticationField.AUTHENTICATION_KEY, authentication); + final boolean noFilteredIndexPermissions = randomBoolean(); + boolean restrictiveLimitedIndexPermissions = false; + if (noFilteredIndexPermissions == false) { + restrictiveLimitedIndexPermissions = randomBoolean(); + } + Set queries = new HashSet<>(); + queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv22\"] } }")); + queries.add(new BytesArray("{\"terms\" : { \"f2\" : [\"fv32\"] } }")); + IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new + FieldPermissions(), + DocumentPermissions.filteredBy(queries)); + queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv21\", \"fv31\"] } }")); + if (restrictiveLimitedIndexPermissions) { + queries = singleton(new BytesArray("{\"terms\" : { \"f1\" : [\"fv11\", \"fv31\"] } }")); + } + IndicesAccessControl.IndexAccessControl limitedIndexAccessControl = new IndicesAccessControl.IndexAccessControl(true, new + FieldPermissions(), + DocumentPermissions.filteredBy(queries)); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), Settings.EMPTY); + Client client = mock(Client.class); + when(client.settings()).thenReturn(Settings.EMPTY); + final long nowInMillis = randomNonNegativeLong(); + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, null, null, mapperService, null, + null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null); + QueryShardContext queryShardContext = spy(realQueryShardContext); + IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(settings, new BitsetFilterCache.Listener() { + @Override + public void onCache(ShardId shardId, Accountable accountable) { + } + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + } + }); + + XPackLicenseState licenseState = mock(XPackLicenseState.class); + when(licenseState.isDocumentAndFieldLevelSecurityAllowed()).thenReturn(true); + SecurityIndexSearcherWrapper wrapper = new SecurityIndexSearcherWrapper(s -> queryShardContext, + bitsetFilterCache, threadContext, licenseState, scriptService) { + + @Override + protected IndicesAccessControl getIndicesAccessControl() { + IndicesAccessControl indicesAccessControl = new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)); + if (noFilteredIndexPermissions) { + return indicesAccessControl; + } + IndicesAccessControl limitedByIndicesAccessControl = new IndicesAccessControl(true, + singletonMap("_index", limitedIndexAccessControl)); + return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl); + } + }; + + Directory directory = newDirectory(); + IndexWriter iw = new IndexWriter( + directory, + new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(NoMergePolicy.INSTANCE) + ); + + Document doc1 = new Document(); + doc1.add(new StringField("f1", "fv11", Store.NO)); + doc1.add(new StringField("f2", "fv12", Store.NO)); + iw.addDocument(doc1); + Document doc2 = new Document(); + doc2.add(new StringField("f1", "fv21", Store.NO)); + doc2.add(new StringField("f2", "fv22", Store.NO)); + iw.addDocument(doc2); + Document doc3 = new Document(); + doc3.add(new StringField("f1", "fv31", Store.NO)); + doc3.add(new StringField("f2", "fv32", Store.NO)); + iw.addDocument(doc3); + iw.commit(); + iw.close(); + + DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId); + DirectoryReader wrappedDirectoryReader = wrapper.wrap(directoryReader); + IndexSearcher indexSearcher = wrapper.wrap(new IndexSearcher(wrappedDirectoryReader)); + + ScoreDoc[] hits = indexSearcher.search(new MatchAllDocsQuery(), 1000).scoreDocs; + Set actualDocIds = new HashSet<>(); + for (ScoreDoc doc : hits) { + actualDocIds.add(doc.doc); + } + + if (noFilteredIndexPermissions) { + assertThat(actualDocIds, containsInAnyOrder(1, 2)); + } else { + if (restrictiveLimitedIndexPermissions) { + assertThat(actualDocIds, containsInAnyOrder(2)); + } else { + assertThat(actualDocIds, containsInAnyOrder(1, 2)); + } + } + + bitsetFilterCache.close(); + directoryReader.close(); + directory.close(); + } + } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java index 078de0fe7cb14..8747b8537ca21 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java @@ -28,21 +28,16 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SparseFixedBitSet; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -52,59 +47,35 @@ import org.elasticsearch.index.mapper.ParentFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.BoostingQueryBuilder; -import org.elasticsearch.index.query.ConstantScoreQueryBuilder; -import org.elasticsearch.index.query.GeoShapeQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.indices.TermsLookup; -import org.elasticsearch.join.query.HasChildQueryBuilder; -import org.elasticsearch.join.query.HasParentQueryBuilder; import org.elasticsearch.license.XPackLicenseState; -import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.script.TemplateScript; import org.elasticsearch.search.aggregations.LeafBucketCollector; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.xpack.core.security.authz.accesscontrol.DocumentSubsetReader.DocumentSubsetDirectoryReader; +import org.elasticsearch.xpack.core.security.authz.permission.DocumentPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDefinition; -import org.elasticsearch.xpack.core.security.user.User; import org.junit.After; import org.junit.Before; -import org.mockito.ArgumentCaptor; import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.IdentityHashMap; -import java.util.Map; import java.util.Set; import static java.util.Collections.singletonMap; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper.intersectScorerAndRoleBits; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class SecurityIndexSearcherWrapperUnitTests extends ESTestCase { @@ -139,7 +110,7 @@ public void setup() throws Exception { IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(shardId); - Directory directory = new RAMDirectory(); + Directory directory = new MMapDirectory(createTempDir()); IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig()); writer.close(); @@ -159,7 +130,7 @@ public void testDefaultMetaFields() throws Exception { @Override protected IndicesAccessControl getIndicesAccessControl() { IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(true, - new FieldPermissions(fieldPermissionDef(new String[]{}, null)), null); + new FieldPermissions(fieldPermissionDef(new String[]{}, null)), DocumentPermissions.allowAll()); return new IndicesAccessControl(true, singletonMap("_index", indexAccessControl)); } }; @@ -444,66 +415,6 @@ public void testIndexSearcherWrapperDenseWithDeletions() throws IOException { doTestIndexSearcherWrapper(false, true); } - public void testTemplating() throws Exception { - User user = new User("_username", new String[]{"role1", "role2"}, "_full_name", "_email", - Collections.singletonMap("key", "value"), true); - securityIndexSearcherWrapper = - new SecurityIndexSearcherWrapper(null, null, threadContext, licenseState, scriptService) { - - @Override - protected User getUser() { - return user; - } - }; - - TemplateScript.Factory compiledTemplate = templateParams -> - new TemplateScript(templateParams) { - @Override - public String execute() { - return "rendered_text"; - } - }; - - when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenReturn(compiledTemplate); - - XContentBuilder builder = jsonBuilder(); - String query = Strings.toString(new TermQueryBuilder("field", "{{_user.username}}").toXContent(builder, ToXContent.EMPTY_PARAMS)); - Script script = new Script(ScriptType.INLINE, "mustache", query, Collections.singletonMap("custom", "value")); - builder = jsonBuilder().startObject().field("template"); - script.toXContent(builder, ToXContent.EMPTY_PARAMS); - String querySource = Strings.toString(builder.endObject()); - - securityIndexSearcherWrapper.evaluateTemplate(querySource); - ArgumentCaptor