From ccdc95d9f86e0fdbacb8401c3e2126b85628b932 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Fri, 26 Mar 2021 12:52:23 -0400 Subject: [PATCH] [ML] Allow datafeed and job configs for datafeed preview API (#70836) Previously, a datafeed and job must already exist for the `_preview` API to work. With this change, users can get an accurate preview of the data that will be sent to the anomaly detection job without creating either of them. closes https://github.com/elastic/elasticsearch/issues/70264 --- .../client/MLRequestConverters.java | 18 ++- .../client/ml/PreviewDatafeedRequest.java | 60 ++++++- .../client/MLRequestConvertersTests.java | 18 ++- .../ml/PreviewDatafeedRequestTests.java | 7 +- .../apis/preview-datafeed.asciidoc | 122 +++++++++++++- .../core/ml/action/PreviewDatafeedAction.java | 124 ++++++++++++++- .../core/ml/datafeed/DatafeedConfig.java | 4 + .../PreviewDatafeedActionRequestTests.java | 56 ++++++- .../ml/qa/ml-with-security/build.gradle | 2 + .../TransportPreviewDatafeedAction.java | 69 +++++--- .../datafeeds/RestPreviewDatafeedAction.java | 11 +- .../api/ml.preview_datafeed.json | 14 +- .../test/ml/preview_datafeed.yml | 150 ++++++++++++++++++ 13 files changed, 595 insertions(+), 60 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 9f8a82c33dff7..2106a97889f04 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -326,14 +326,18 @@ static Request getDatafeedStats(GetDatafeedStatsRequest getDatafeedStatsRequest) return request; } - static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) { - String endpoint = new EndpointBuilder() + static Request previewDatafeed(PreviewDatafeedRequest previewDatafeedRequest) throws IOException { + EndpointBuilder builder = new EndpointBuilder() .addPathPartAsIs("_ml") - .addPathPartAsIs("datafeeds") - .addPathPart(previewDatafeedRequest.getDatafeedId()) - .addPathPartAsIs("_preview") - .build(); - return new Request(HttpGet.METHOD_NAME, endpoint); + .addPathPartAsIs("datafeeds"); + String endpoint = previewDatafeedRequest.getDatafeedId() != null ? + builder.addPathPart(previewDatafeedRequest.getDatafeedId()).addPathPartAsIs("_preview").build() : + builder.addPathPartAsIs("_preview").build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + if (previewDatafeedRequest.getDatafeedId() == null) { + request.setEntity(createEntity(previewDatafeedRequest, REQUEST_BODY_CONTENT_TYPE)); + } + return request; } static Request deleteForecast(DeleteForecastRequest deleteForecastRequest) { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java index b5a14a217f6cf..1e3dfd819a82a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/PreviewDatafeedRequest.java @@ -10,6 +10,9 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.client.ml.datafeed.DatafeedConfig; +import org.elasticsearch.client.ml.job.config.Job; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -24,11 +27,17 @@ */ public class PreviewDatafeedRequest extends ActionRequest implements ToXContentObject { + private static final ParseField DATAFEED_CONFIG = new ParseField("datafeed_config"); + private static final ParseField JOB_CONFIG = new ParseField("job_config"); + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "open_datafeed_request", true, a -> new PreviewDatafeedRequest((String) a[0])); + "preview_datafeed_request", + a -> new PreviewDatafeedRequest((String) a[0], (DatafeedConfig.Builder) a[1], (Job.Builder) a[2])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.ID); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DatafeedConfig.PARSER, DATAFEED_CONFIG); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Job.PARSER, JOB_CONFIG); } public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws IOException { @@ -36,6 +45,16 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws } private final String datafeedId; + private final DatafeedConfig datafeedConfig; + private final Job jobConfig; + + private PreviewDatafeedRequest(@Nullable String datafeedId, + @Nullable DatafeedConfig.Builder datafeedConfig, + @Nullable Job.Builder jobConfig) { + this.datafeedId = datafeedId; + this.datafeedConfig = datafeedConfig == null ? null : datafeedConfig.build(); + this.jobConfig = jobConfig == null ? null : jobConfig.build(); + } /** * Create a new request with the desired datafeedId @@ -44,12 +63,33 @@ public static PreviewDatafeedRequest fromXContent(XContentParser parser) throws */ public PreviewDatafeedRequest(String datafeedId) { this.datafeedId = Objects.requireNonNull(datafeedId, "[datafeed_id] must not be null"); + this.datafeedConfig = null; + this.jobConfig = null; + } + + /** + * Create a new request to preview the provided datafeed config and optional job config + * @param datafeedConfig The datafeed to preview + * @param jobConfig The associated job config (required if the datafeed does not refer to an existing job) + */ + public PreviewDatafeedRequest(DatafeedConfig datafeedConfig, Job jobConfig) { + this.datafeedId = null; + this.datafeedConfig = datafeedConfig; + this.jobConfig = jobConfig; } public String getDatafeedId() { return datafeedId; } + public DatafeedConfig getDatafeedConfig() { + return datafeedConfig; + } + + public Job getJobConfig() { + return jobConfig; + } + @Override public ActionRequestValidationException validate() { return null; @@ -58,7 +98,15 @@ public ActionRequestValidationException validate() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + if (datafeedId != null) { + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + } + if (datafeedConfig != null) { + builder.field(DATAFEED_CONFIG.getPreferredName(), datafeedConfig); + } + if (jobConfig != null) { + builder.field(JOB_CONFIG.getPreferredName(), jobConfig); + } builder.endObject(); return builder; } @@ -70,7 +118,7 @@ public String toString() { @Override public int hashCode() { - return Objects.hash(datafeedId); + return Objects.hash(datafeedId, datafeedConfig, jobConfig); } @Override @@ -84,6 +132,8 @@ public boolean equals(Object other) { } PreviewDatafeedRequest that = (PreviewDatafeedRequest) other; - return Objects.equals(datafeedId, that.datafeedId); + return Objects.equals(datafeedId, that.datafeedId) + && Objects.equals(datafeedConfig, that.datafeedConfig) + && Objects.equals(jobConfig, that.jobConfig); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index 1eb267d0cc095..533492a9c6e83 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -91,6 +91,7 @@ import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.Detector; import org.elasticsearch.client.ml.job.config.Job; +import org.elasticsearch.client.ml.job.config.JobTests; import org.elasticsearch.client.ml.job.config.JobUpdate; import org.elasticsearch.client.ml.job.config.JobUpdateTests; import org.elasticsearch.client.ml.job.config.MlFilter; @@ -390,11 +391,24 @@ public void testGetDatafeedStats() { assertEquals(Boolean.toString(true), request.getParameters().get("allow_no_match")); } - public void testPreviewDatafeed() { + public void testPreviewDatafeed() throws IOException { PreviewDatafeedRequest datafeedRequest = new PreviewDatafeedRequest("datafeed_1"); Request request = MLRequestConverters.previewDatafeed(datafeedRequest); - assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); assertEquals("/_ml/datafeeds/" + datafeedRequest.getDatafeedId() + "/_preview", request.getEndpoint()); + assertThat(request.getEntity(), is(nullValue())); + + datafeedRequest = new PreviewDatafeedRequest( + DatafeedConfigTests.createRandom(), + randomBoolean() ? null : JobTests.createRandomizedJob() + ); + request = MLRequestConverters.previewDatafeed(datafeedRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_ml/datafeeds/_preview", request.getEndpoint()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) { + PreviewDatafeedRequest parsedDatafeedRequest = PreviewDatafeedRequest.PARSER.apply(parser, null); + assertThat(parsedDatafeedRequest, equalTo(datafeedRequest)); + } } public void testDeleteForecast() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java index 72fb42de09195..31e3bc1c4bda8 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/PreviewDatafeedRequestTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.client.ml; import org.elasticsearch.client.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.client.ml.job.config.JobTests; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; @@ -17,7 +18,9 @@ public class PreviewDatafeedRequestTests extends AbstractXContentTestCase/_preview` +`GET _ml/datafeeds//_preview` + + +`POST _ml/datafeeds//_preview` + + +`GET _ml/datafeeds/_preview` + + +`POST _ml/datafeeds/_preview` [[ml-preview-datafeed-prereqs]] == {api-prereq-title} @@ -25,9 +31,10 @@ Previews a {dfeed}. [[ml-preview-datafeed-desc]] == {api-description-title} -The preview {dfeeds} API returns the first "page" of results from the `search` -that is created by using the current {dfeed} settings. This preview shows the -structure of the data that will be passed to the anomaly detection engine. +The preview {dfeeds} API returns the first "page" of search results from a +{dfeed}. You can preview an existing {dfeed} or provide configuration details +for the {dfeed} and {anomaly-job} in the API. The preview shows the structure of +the data that will be passed to the anomaly detection engine. IMPORTANT: When {es} {security-features} are enabled, the {dfeed} query is previewed using the credentials of the user calling the preview {dfeed} API. @@ -43,12 +50,32 @@ supply the credentials. == {api-path-parms-title} ``:: -(Required, string) +(Optional, string) include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] ++ +NOTE: If you provide the `` as a path parameter, you cannot +provide {dfeed} or {anomaly-job} configuration details in the request body. + +[[ml-preview-datafeed-request-body]] +== {api-request-body-title} + +`datafeed_config`:: +(Optional, object) The {dfeed} definition to preview. For valid definitions, see +the <>. + +`job_config`:: +(Optional, object) The configuration details for the {anomaly-job} that is +associated with the {dfeed}. If the `datafeed_config` object does not include a +`job_id` that references an existing {anomaly-job}, you must supply this +`job_config` object. If you include both a `job_id` and a `job_config`, the +latter information is used. You cannot specify a `job_config` object unless you also supply a `datafeed_config` object. For valid definitions, see the +<>. [[ml-preview-datafeed-example]] == {api-examples-title} +This is an example of providing the ID of an existing {dfeed}: + [source,console] -------------------------------------------------- GET _ml/datafeeds/datafeed-high_sum_total_sales/_preview @@ -87,3 +114,88 @@ The data that is returned for this example is as follows: ... ] ---- + +The following example provides {dfeed} and {anomaly-job} configuration +details in the API: + +[source,console] +-------------------------------------------------- +POST _ml/datafeeds/_preview +{ + "datafeed_config": { + "indices" : [ + "kibana_sample_data_ecommerce" + ], + "query" : { + "bool" : { + "filter" : [ + { + "term" : { + "_index" : "kibana_sample_data_ecommerce" + } + } + ] + } + }, + "scroll_size" : 1000 + }, + "job_config": { + "description" : "Find customers spending an unusually high amount in an hour", + "analysis_config" : { + "bucket_span" : "1h", + "detectors" : [ + { + "detector_description" : "High total sales", + "function" : "high_sum", + "field_name" : "taxful_total_price", + "over_field_name" : "customer_full_name.keyword" + } + ], + "influencers" : [ + "customer_full_name.keyword", + "category.keyword" + ] + }, + "analysis_limits" : { + "model_memory_limit" : "10mb" + }, + "data_description" : { + "time_field" : "order_date", + "time_format" : "epoch_ms" + } + } +} +-------------------------------------------------- +// TEST[skip:set up Kibana sample data] + +The data that is returned for this example is as follows: + +[source,console-result] +---- +[ + { + "order_date" : 1574294659000, + "category.keyword" : "Men's Clothing", + "customer_full_name.keyword" : "Sultan Al Benson", + "taxful_total_price" : 35.96875 + }, + { + "order_date" : 1574294918000, + "category.keyword" : [ + "Women's Accessories", + "Women's Clothing" + ], + "customer_full_name.keyword" : "Pia Webb", + "taxful_total_price" : 83.0 + }, + { + "order_date" : 1574295782000, + "category.keyword" : [ + "Women's Accessories", + "Women's Shoes" + ], + "customer_full_name.keyword" : "Brigitte Graham", + "taxful_total_price" : 72.0 + } +] +---- diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java index 4f203308fbe0f..6f2aec6ab75c2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedAction.java @@ -6,20 +6,25 @@ */ package org.elasticsearch.xpack.core.ml.action; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; @@ -37,26 +42,68 @@ private PreviewDatafeedAction() { public static class Request extends ActionRequest implements ToXContentObject { - private String datafeedId; + private static final String BLANK_ID = ""; - public Request() { + public static final ParseField DATAFEED_CONFIG = new ParseField("datafeed_config"); + public static final ParseField JOB_CONFIG = new ParseField("job_config"); + + private static final ObjectParser PARSER = new ObjectParser<>( + "preview_datafeed_action", + Request.Builder::new + ); + static { + PARSER.declareObject(Builder::setDatafeedBuilder, DatafeedConfig.STRICT_PARSER, DATAFEED_CONFIG); + PARSER.declareObject(Builder::setJobBuilder, Job.STRICT_PARSER, JOB_CONFIG); + } + + public static Request fromXContent(XContentParser parser) { + return PARSER.apply(parser, null).build(); + } + + private final String datafeedId; + private final DatafeedConfig datafeedConfig; + private final Job.Builder jobConfig; + + private Request() { + this.datafeedId = null; + this.datafeedConfig = null; + this.jobConfig = null; } public Request(StreamInput in) throws IOException { super(in); datafeedId = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_13_0)) { + datafeedConfig = in.readOptionalWriteable(DatafeedConfig::new); + jobConfig = in.readOptionalWriteable(Job.Builder::new); + } else { + datafeedConfig = null; + jobConfig = null; + } } public Request(String datafeedId) { - setDatafeedId(datafeedId); + this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID); + this.datafeedConfig = null; + this.jobConfig = null; + } + + public Request(DatafeedConfig datafeedConfig, Job.Builder jobConfig) { + this.datafeedId = BLANK_ID; + this.datafeedConfig = ExceptionsHelper.requireNonNull(datafeedConfig, DATAFEED_CONFIG.getPreferredName()); + this.jobConfig = jobConfig; } public String getDatafeedId() { return datafeedId; } - public final void setDatafeedId(String datafeedId) { - this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); + public DatafeedConfig getDatafeedConfig() { + return datafeedConfig; + } + + public Job.Builder getJobConfig() { + return jobConfig; } @Override @@ -68,19 +115,31 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(datafeedId); + if (out.getVersion().onOrAfter(Version.V_7_13_0)) { + out.writeOptionalWriteable(datafeedConfig); + out.writeOptionalWriteable(jobConfig); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + if (datafeedId.equals("") == false) { + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + } + if (datafeedConfig != null) { + builder.field(DATAFEED_CONFIG.getPreferredName(), datafeedConfig); + } + if (jobConfig != null) { + builder.field(JOB_CONFIG.getPreferredName(), jobConfig); + } builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(datafeedId); + return Objects.hash(datafeedId, datafeedConfig, jobConfig); } @Override @@ -92,7 +151,56 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Objects.equals(datafeedId, other.datafeedId); + return Objects.equals(datafeedId, other.datafeedId) + && Objects.equals(datafeedConfig, other.datafeedConfig) + && Objects.equals(jobConfig, other.jobConfig); + } + + public static class Builder { + private String datafeedId; + private DatafeedConfig.Builder datafeedBuilder; + private Job.Builder jobBuilder; + + public Builder setDatafeedId(String datafeedId) { + this.datafeedId = datafeedId; + return this; + } + + public Builder setDatafeedBuilder(DatafeedConfig.Builder datafeedBuilder) { + this.datafeedBuilder = datafeedBuilder; + return this; + } + + public Builder setJobBuilder(Job.Builder jobBuilder) { + this.jobBuilder = jobBuilder; + return this; + } + + public Request build() { + if (datafeedBuilder != null) { + datafeedBuilder.setId("preview_id"); + if (datafeedBuilder.getJobId() == null && jobBuilder == null) { + throw new IllegalArgumentException("[datafeed_config.job_id] must be set or a [job_config] must be provided"); + } + if (datafeedBuilder.getJobId() == null) { + datafeedBuilder.setJobId("preview_job_id"); + } + } + if (jobBuilder != null) { + jobBuilder.setId("preview_job_id"); + if (datafeedBuilder == null) { + throw new IllegalArgumentException("[datafeed_config] must be present when a [job_config] is provided"); + } + } + if (datafeedId != null && (datafeedBuilder != null || jobBuilder != null)) { + throw new IllegalArgumentException( + "[datafeed_id] cannot be supplied when either [job_config] or [datafeed_config] is present" + ); + } + return datafeedId != null ? + new Request(datafeedId) : + new Request(datafeedBuilder == null ? null : datafeedBuilder.build(), jobBuilder); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index fa8d65c4fc006..0affe363929b8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -744,6 +744,10 @@ public Builder setJobId(String jobId) { return this; } + public String getJobId() { + return jobId; + } + public Builder setHeaders(Map headers) { this.headers = ExceptionsHelper.requireNonNull(headers, HEADERS.getPreferredName()); return this; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java index 1aa66995557c5..53859b422f051 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PreviewDatafeedActionRequestTests.java @@ -6,19 +6,73 @@ */ package org.elasticsearch.xpack.core.ml.action; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.action.PreviewDatafeedAction.Request; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; + +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class PreviewDatafeedActionRequestTests extends AbstractWireSerializingTestCase { @Override protected Request createTestInstance() { - return new Request(randomAlphaOfLength(10)); + String jobId = randomAlphaOfLength(10); + return randomBoolean() ? + new Request(randomAlphaOfLength(10)) : + new Request( + DatafeedConfigTests.createRandomizedDatafeedConfig(jobId), + randomBoolean() ? JobTests.buildJobBuilder(jobId) : null + ); } @Override protected Writeable.Reader instanceReader() { return Request::new; } + + public void testCtor() { + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new Request((String) null)); + assertThat(ex.getMessage(), equalTo("[datafeed_id] must not be null.")); + } + + public void testValidation() { + String jobId = randomAlphaOfLength(10); + Request.Builder requestBuilder = new Request.Builder() + .setDatafeedId(randomAlphaOfLength(10)) + .setDatafeedBuilder(new DatafeedConfig.Builder(DatafeedConfigTests.createRandomizedDatafeedConfig(jobId))) + .setJobBuilder(randomBoolean() ? JobTests.buildJobBuilder(jobId) : null); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, requestBuilder::build); + assertThat(ex.getMessage(), + containsString("[datafeed_id] cannot be supplied when either [job_config] or [datafeed_config] is present")); + + requestBuilder.setJobBuilder(null) + .setDatafeedId(null) + .setDatafeedBuilder(new DatafeedConfig.Builder()); + + ex = expectThrows(IllegalArgumentException.class, requestBuilder::build); + assertThat(ex.getMessage(), + containsString("[datafeed_config.job_id] must be set or a [job_config] must be provided")); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index db969f6ff8dcf..feb10c7baee47 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -191,6 +191,8 @@ tasks.named("yamlRestTest").configure { 'ml/post_data/Test open and close with non-existent job id', 'ml/post_data/Test POST data with invalid parameters', 'ml/preview_datafeed/Test preview missing datafeed', + 'ml/preview_datafeed/Test preview with datafeed_id and job config', + 'ml/preview_datafeed/Test preview with datafeed id and config', 'ml/revert_model_snapshot/Test revert model with invalid snapshotId', 'ml/start_data_frame_analytics/Test start given missing source index', 'ml/start_data_frame_analytics/Test start outlier_detection given source index has no fields', diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java index 8c0b95e733fe2..23f6f510bb30f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPreviewDatafeedAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats; import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.ml.datafeed.DatafeedTimingStatsReporter; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; @@ -33,6 +34,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.Date; import java.util.Optional; import java.util.stream.Collectors; @@ -65,35 +67,50 @@ public TransportPreviewDatafeedAction(Settings settings, ThreadPool threadPool, @Override protected void doExecute(Task task, PreviewDatafeedAction.Request request, ActionListener listener) { - datafeedConfigProvider.getDatafeedConfig(request.getDatafeedId(), ActionListener.wrap( - datafeedConfigBuilder -> { - DatafeedConfig datafeedConfig = datafeedConfigBuilder.build(); + ActionListener datafeedConfigActionListener = ActionListener.wrap( + datafeedConfig -> { + if (request.getJobConfig() != null) { + previewDatafeed(datafeedConfig, request.getJobConfig().build(new Date()), listener); + return; + } jobConfigProvider.getJob(datafeedConfig.getJobId(), ActionListener.wrap( - jobBuilder -> { - DatafeedConfig.Builder previewDatafeed = buildPreviewDatafeed(datafeedConfig); - useSecondaryAuthIfAvailable(securityContext, () -> { - previewDatafeed.setHeaders(filterSecurityHeaders(threadPool.getThreadContext().getHeaders())); - - // NB: this is using the client from the transport layer, NOT the internal client. - // This is important because it means the datafeed search will fail if the user - // requesting the preview doesn't have permission to search the relevant indices. - DataExtractorFactory.create( - client, - previewDatafeed.build(), - jobBuilder.build(), - xContentRegistry, - // Fake DatafeedTimingStatsReporter that does not have access to results index - new DatafeedTimingStatsReporter(new DatafeedTimingStats(datafeedConfig.getJobId()), (ts, refreshPolicy) -> { - }), - listener.delegateFailure((l, dataExtractorFactory) -> { - DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); - threadPool.generic().execute(() -> previewDatafeed(dataExtractor, l)); - })); - }); - }, + jobBuilder -> previewDatafeed(datafeedConfig, jobBuilder.build(), listener), listener::onFailure)); }, - listener::onFailure)); + listener::onFailure + ); + if (request.getDatafeedConfig() != null) { + datafeedConfigActionListener.onResponse(request.getDatafeedConfig()); + } else { + datafeedConfigProvider.getDatafeedConfig( + request.getDatafeedId(), + ActionListener.wrap(builder -> datafeedConfigActionListener.onResponse(builder.build()), listener::onFailure)); + } + } + + private void previewDatafeed( + DatafeedConfig datafeedConfig, + Job job, + ActionListener listener + ) { + DatafeedConfig.Builder previewDatafeed = buildPreviewDatafeed(datafeedConfig); + useSecondaryAuthIfAvailable(securityContext, () -> { + previewDatafeed.setHeaders(filterSecurityHeaders(threadPool.getThreadContext().getHeaders())); + // NB: this is using the client from the transport layer, NOT the internal client. + // This is important because it means the datafeed search will fail if the user + // requesting the preview doesn't have permission to search the relevant indices. + DataExtractorFactory.create( + client, + previewDatafeed.build(), + job, + xContentRegistry, + // Fake DatafeedTimingStatsReporter that does not have access to results index + new DatafeedTimingStatsReporter(new DatafeedTimingStats(datafeedConfig.getJobId()), (ts, refreshPolicy) -> {}), + listener.delegateFailure((l, dataExtractorFactory) -> { + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(0, Long.MAX_VALUE); + threadPool.generic().execute(() -> previewDatafeed(dataExtractor, l)); + })); + }); } /** Visible for testing */ diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java index 7734bcd33fe8b..ce9b9f5f86b5c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPreviewDatafeedAction.java @@ -18,6 +18,7 @@ import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.xpack.ml.MachineLearning.BASE_PATH; import static org.elasticsearch.xpack.ml.MachineLearning.PRE_V7_BASE_PATH; @@ -27,7 +28,10 @@ public class RestPreviewDatafeedAction extends BaseRestHandler { public List routes() { return org.elasticsearch.common.collect.List.of( Route.builder(GET, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview") - .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview", RestApiVersion.V_7).build() + .replaces(GET, PRE_V7_BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview", RestApiVersion.V_7).build(), + new Route(GET, BASE_PATH + "datafeeds/_preview"), + new Route(POST, BASE_PATH + "datafeeds/{" + DatafeedConfig.ID + "}/_preview"), + new Route(POST, BASE_PATH + "datafeeds/_preview") ); } @@ -38,8 +42,9 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); - PreviewDatafeedAction.Request request = new PreviewDatafeedAction.Request(datafeedId); + PreviewDatafeedAction.Request request = restRequest.hasContentOrSourceParam() ? + PreviewDatafeedAction.Request.fromXContent(restRequest.contentOrSourceParamParser()) : + new PreviewDatafeedAction.Request(restRequest.param(DatafeedConfig.ID.getPreferredName())); return channel -> client.execute(PreviewDatafeedAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json index 5cd94bc2cf528..2077cc5d396bc 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.preview_datafeed.json @@ -14,7 +14,8 @@ { "path":"/_ml/datafeeds/{datafeed_id}/_preview", "methods":[ - "GET" + "GET", + "POST" ], "parts":{ "datafeed_id":{ @@ -22,8 +23,19 @@ "description":"The ID of the datafeed to preview" } } + }, + { + "path":"/_ml/datafeeds/_preview", + "methods":[ + "GET", + "POST" + ] } ] + }, + "body":{ + "description":"The datafeed config and job config with which to execute the preview", + "required":false } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/preview_datafeed.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/preview_datafeed.yml index df2eabc38228a..77a9fc71379e5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/preview_datafeed.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/preview_datafeed.yml @@ -109,6 +109,60 @@ setup: - match: { 3.airline: foo } - match: { 3.responsetime: 42.0 } + - do: + ml.preview_datafeed: + body: > + { + "datafeed_config": { + "job_id":"preview-datafeed-job", + "indexes":"airline-data" + } + } + - length: { $body: 4 } + - match: { 0.time: 1487376000000 } + - match: { 0.airline: foo } + - match: { 0.responsetime: 1.0 } + - match: { 1.time: 1487377800000 } + - match: { 1.airline: foo } + - match: { 1.responsetime: 1.0 } + - match: { 2.time: 1487379600000 } + - match: { 2.airline: bar } + - match: { 2.responsetime: 42.0 } + - match: { 3.time: 1487379660000 } + - match: { 3.airline: foo } + - match: { 3.responsetime: 42.0 } + + - do: + ml.preview_datafeed: + body: > + { + "datafeed_config": { + "job_id":"preview-datafeed-job", + "indexes":"airline-data" + }, + "job_config": { + "analysis_config": { + "bucket_span": "1h", + "detectors": [{"function":"sum","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description": { + "time_field":"time" + } + } + } + - length: { $body: 4 } + - match: { 0.time: 1487376000000 } + - match: { 0.airline: foo } + - match: { 0.responsetime: 1.0 } + - match: { 1.time: 1487377800000 } + - match: { 1.airline: foo } + - match: { 1.responsetime: 1.0 } + - match: { 2.time: 1487379600000 } + - match: { 2.airline: bar } + - match: { 2.responsetime: 42.0 } + - match: { 3.time: 1487379660000 } + - match: { 3.airline: foo } + - match: { 3.responsetime: 42.0 } --- "Test preview aggregation datafeed with doc_count": @@ -180,6 +234,65 @@ setup: - match: { 2.airline: foo } - match: { 2.responsetime: 42.0 } - match: { 2.doc_count: 1 } + - do: + ml.preview_datafeed: + body: > + { + "datafeed_config": { + "indexes":"airline-data", + "aggregations": { + "buckets": { + "histogram": { + "field": "time", + "interval": 3600000 + }, + "aggregations": { + "time": { + "max": { + "field": "time" + } + }, + "airline": { + "terms": { + "field": "airline", + "size": 100 + }, + "aggregations": { + "responsetime": { + "sum": { + "field": "responsetime" + } + } + } + } + } + } + } + }, + "job_config": { + "analysis_config" : { + "bucket_span": "1h", + "summary_count_field_name": "doc_count", + "detectors" :[{"function":"sum","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + "time_field":"time" + } + } + } + - length: { $body: 3 } + - match: { 0.time: 1487377800000 } + - match: { 0.airline: foo } + - match: { 0.responsetime: 2.0 } + - match: { 0.doc_count: 2 } + - match: { 1.time: 1487379660000 } + - match: { 1.airline: bar } + - match: { 1.responsetime: 42.0 } + - match: { 1.doc_count: 1 } + - match: { 1.time: 1487379660000 } + - match: { 2.airline: foo } + - match: { 2.responsetime: 42.0 } + - match: { 2.doc_count: 1 } --- "Test preview single metric aggregation datafeed with different summary count field": @@ -327,6 +440,43 @@ setup: datafeed_id: missing-feed --- +"Test preview with datafeed_id and job config": + + - do: + catch: bad_request + ml.preview_datafeed: + datafeed_id: some_datafeed_id + body: > + { + "job_config": { + "analysis_config" : { + "bucket_span": "1h", + "detectors" :[{"function":"sum","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + "time_field":"time" + } + } + } +--- +"Test preview with datafeed id and config": + + - do: + catch: bad_request + ml.preview_datafeed: + body: > + { + "job_config": { + "analysis_config" : { + "bucket_span": "1h", + "detectors" :[{"function":"sum","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + "time_field":"time" + } + } + } +--- "Test preview datafeed with unavailable index": - do: