From e530bd4ab8d84b83745337afc139406b9b0f76bc Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Wed, 6 Jun 2018 23:16:01 +0200 Subject: [PATCH 1/6] Added simulate pipeline API --- .../elasticsearch/client/IngestClient.java | 28 +++++ .../client/RequestConverters.java | 15 +++ .../client/ESRestHighLevelClientTestCase.java | 10 +- .../elasticsearch/client/IngestClientIT.java | 76 ++++++++++++ .../client/RequestConvertersTests.java | 24 ++++ .../IngestClientDocumentationIT.java | 111 +++++++++++++++++ .../ingest/simulate_pipeline.asciidoc | 90 ++++++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../ingest/SimulateDocumentBaseResult.java | 36 ++++++ .../ingest/SimulateDocumentVerboseResult.java | 23 +++- .../ingest/SimulatePipelineRequest.java | 14 ++- .../ingest/SimulatePipelineResponse.java | 75 ++++++++++++ .../ingest/SimulateProcessorResult.java | 66 +++++++++- .../ingest/WriteableIngestDocument.java | 83 ++++++++++++- .../elasticsearch/ingest/IngestDocument.java | 13 +- ...a => SimulateDocumentBaseResultTests.java} | 63 ++++++++-- .../SimulateDocumentVerboseResultTests.java | 72 +++++++++++ .../ingest/SimulatePipelineResponseTests.java | 115 +++++++++++++----- .../ingest/SimulateProcessorResultTests.java | 79 +++++++++--- .../ingest/WriteableIngestDocumentTests.java | 75 +++++++++++- .../ingest/RandomDocumentPicks.java | 26 ++-- 21 files changed, 1015 insertions(+), 81 deletions(-) create mode 100644 docs/java-rest/high-level/ingest/simulate_pipeline.asciidoc rename server/src/test/java/org/elasticsearch/action/ingest/{SimulateDocumentSimpleResultTests.java => SimulateDocumentBaseResultTests.java} (59%) create mode 100644 server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java index 5c5a82b52f438..f4bba28386a12 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.ingest.GetPipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; import java.io.IOException; @@ -125,4 +127,30 @@ public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions op restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options, WritePipelineResponse::fromXContent, listener, emptySet()); } + + /** + * Simulate a pipeline on a set of documents provided in the request + *

+ * See + * + * Simulate Pipeline API on elastic.co + */ + public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request, Header... headers) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::simulatePipeline, + SimulatePipelineResponse::fromXContent, emptySet(), headers); + } + + /** + * Asynchronously simulate a pipeline on a set of documents provided in the request + *

+ * See + * + * Simulate Pipeline API on elastic.co + */ + public void simulatePipelineAsync(SimulatePipelineRequest request, + ActionListener listener, + Header... headers) { + restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::simulatePipeline, + SimulatePipelineResponse::fromXContent, listener, emptySet(), headers); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index ab85af9f1fd7e..40b5c3c7df43c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -68,6 +68,7 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.GetPipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; @@ -871,6 +872,20 @@ static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws I return request; } + static Request simulatePipeline(SimulatePipelineRequest simulatePipelineRequest) throws IOException { + EndpointBuilder builder = new EndpointBuilder().addPathPartAsIs("_ingest/pipeline"); + if (simulatePipelineRequest.getId() != null && !simulatePipelineRequest.getId().isEmpty()) { + builder.addPathPart(simulatePipelineRequest.getId()); + } + builder.addPathPart("_simulate"); + String endpoint = builder.build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + Params params = new Params(request); + params.putParam("verbose", Boolean.toString(simulatePipelineRequest.isVerbose())); + request.setEntity(createEntity(simulatePipelineRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request getAlias(GetAliasesRequest getAliasesRequest) { String[] indices = getAliasesRequest.indices() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.indices(); String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java index 4ad39f547584b..69fbab30c336c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java @@ -85,9 +85,7 @@ private HighLevelClient(RestClient restClient) { } } - protected static XContentBuilder buildRandomXContentPipeline() throws IOException { - XContentType xContentType = randomFrom(XContentType.values()); - XContentBuilder pipelineBuilder = XContentBuilder.builder(xContentType.xContent()); + protected static XContentBuilder buildRandomXContentPipeline(XContentBuilder pipelineBuilder) throws IOException { pipelineBuilder.startObject(); { pipelineBuilder.field(Pipeline.DESCRIPTION_KEY, "some random set of processors"); @@ -114,6 +112,12 @@ protected static XContentBuilder buildRandomXContentPipeline() throws IOExceptio return pipelineBuilder; } + protected static XContentBuilder buildRandomXContentPipeline() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder pipelineBuilder = XContentBuilder.builder(xContentType.xContent()); + return buildRandomXContentPipeline(pipelineBuilder); + } + protected static void createPipeline(String pipelineId) throws IOException { XContentBuilder builder = buildRandomXContentPipeline(); createPipeline(new PutPipelineRequest(pipelineId, BytesReference.bytes(builder), builder.contentType())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index ecc0d0052d415..909db80f3417e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -23,12 +23,21 @@ import org.elasticsearch.action.ingest.GetPipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; +import org.elasticsearch.action.ingest.SimulateDocumentResult; +import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineResponse; import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.ingest.RandomDocumentPicks; import java.io.IOException; +import java.util.Map; public class IngestClientIT extends ESRestHighLevelClientTestCase { @@ -80,4 +89,71 @@ public void testDeletePipeline() throws IOException { execute(request, highLevelClient().ingest()::deletePipeline, highLevelClient().ingest()::deletePipelineAsync); assertTrue(response.isAcknowledged()); } + + public void testSimulatePipeline() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + int numDocs = randomIntBetween(1, 10); + boolean isVerbose = randomBoolean(); + builder.startObject(); + { + builder.field("pipeline"); + buildRandomXContentPipeline(builder); + builder.startArray("docs"); + { + for (int i = 0; i < numDocs; i++) { + builder.startObject(); + IngestDocument document = RandomDocumentPicks.randomIngestDocument(random()); + Map metadataMap = document.extractMetadata(); + for (Map.Entry metadata : metadataMap.entrySet()) { + if (metadata.getValue() != null) { + if (metadata.getKey().equals(IngestDocument.MetaData.VERSION)) { + builder.field(metadata.getKey().getFieldName(), (long)metadata.getValue()); + } else { + builder.field(metadata.getKey().getFieldName(), metadata.getValue().toString()); + } + } + } + document.setFieldValue("rank", Integer.toString(randomInt())); + builder.field("_source", document.getSourceAndMetadata()); + builder.endObject(); + } + } + builder.endArray(); + } + builder.endObject(); + + SimulatePipelineRequest request = new SimulatePipelineRequest( + BytesReference.bytes(builder), + builder.contentType() + ); + request.setVerbose(isVerbose); + + SimulatePipelineResponse simulatePipelineResponse = + execute(request, highLevelClient().ingest()::simulatePipeline, highLevelClient().ingest()::simulatePipelineAsync); + + for (SimulateDocumentResult result: simulatePipelineResponse.getResults()) { + if (isVerbose) { + assertTrue(result instanceof SimulateDocumentVerboseResult); + SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult)result; + assertTrue(verboseResult.getProcessorResults().size() > 0); + assertEquals( + verboseResult.getProcessorResults().get(0).getIngestDocument() + .getFieldValue("foo", String.class), + "bar" + ); + } else { + assertTrue(result instanceof SimulateDocumentBaseResult); + SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result; + assertNotNull(baseResult.getIngestDocument()); + assertEquals( + baseResult.getIngestDocument().getFieldValue("foo", String.class), + "bar" + ); + assertNotNull( + baseResult.getIngestDocument().getFieldValue("rank", Integer.class) + ); + } + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 60f427b490462..590ed2e6ea8ad 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -71,6 +71,7 @@ import org.elasticsearch.action.ingest.DeletePipelineRequest; import org.elasticsearch.action.ingest.GetPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.SearchRequest; @@ -1531,6 +1532,29 @@ public void testDeletePipeline() { assertEquals(expectedParams, expectedRequest.getParameters()); } + public void testSimulatePipeline() throws IOException { + String pipelineId = randomBoolean() ? "some_pipeline_id" : null; + boolean verbose = randomBoolean(); + SimulatePipelineRequest request = new SimulatePipelineRequest( + new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), + XContentType.JSON + ); + request.setId(pipelineId); + request.setVerbose(verbose); + Map expectedParams = new HashMap<>(); + expectedParams.put("verbose", Boolean.toString(verbose)); + + Request expectedRequest = RequestConverters.simulatePipeline(request); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + endpoint.add("_ingest/pipeline"); + if (pipelineId != null && !pipelineId.isEmpty()) + endpoint.add(pipelineId); + endpoint.add("_simulate"); + assertEquals(endpoint.toString(), expectedRequest.getEndpoint()); + assertEquals(HttpPost.METHOD_NAME, expectedRequest.getMethod()); + assertEquals(expectedParams, expectedRequest.getParameters()); + } + public void testClusterHealth() { ClusterHealthRequest healthRequest = new ClusterHealthRequest(); Map expectedParams = new HashMap<>(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index f5bdc9f2f3ee5..7636f77d66837 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -25,6 +25,12 @@ import org.elasticsearch.action.ingest.GetPipelineRequest; import org.elasticsearch.action.ingest.GetPipelineResponse; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.ingest.SimulateDocumentBaseResult; +import org.elasticsearch.action.ingest.SimulateDocumentResult; +import org.elasticsearch.action.ingest.SimulateDocumentVerboseResult; +import org.elasticsearch.action.ingest.SimulatePipelineRequest; +import org.elasticsearch.action.ingest.SimulatePipelineResponse; +import org.elasticsearch.action.ingest.SimulateProcessorResult; import org.elasticsearch.action.ingest.WritePipelineResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.RequestOptions; @@ -277,4 +283,109 @@ public void onFailure(Exception e) { } } + public void testSimulatePipeline() throws IOException { + RestHighLevelClient client = highLevelClient(); + + { + // tag::simulate-pipeline-request + String source = + "{\"" + + "pipeline\":{" + + "\"description\":\"_description\"," + + "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + + "}," + + "\"docs\":[" + + "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "]" + + "}"; + SimulatePipelineRequest request = new SimulatePipelineRequest( + new BytesArray(source.getBytes(StandardCharsets.UTF_8)), // <1> + XContentType.JSON // <2> + ); + // end::simulate-pipeline-request + + // tag::simulate-pipeline-request-pipeline-id + request.setId("my-pipeline-id"); // <1> + // end::simulate-pipeline-request-pipeline-id + + // For testing we set this back to null + request.setId(null); + + // tag::simulate-pipeline-request-verbose + request.setVerbose(true); // <1> + // end::simulate-pipeline-request-verbose + + // tag::simulate-pipeline-execute + SimulatePipelineResponse response = client.ingest().simulatePipeline(request); // <1> + // end::simulate-pipeline-execute + + // tag::simulate-pipeline-response + for (SimulateDocumentResult result: response.getResults()) { // <1> + if (request.isVerbose()) { + assert result instanceof SimulateDocumentVerboseResult; + SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult)result; // <2> + for (SimulateProcessorResult processorResult: verboseResult.getProcessorResults()) { // <3> + processorResult.getIngestDocument(); // <4> + processorResult.getFailure(); // <5> + } + } else { + assert result instanceof SimulateDocumentBaseResult; + SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result; // <6> + baseResult.getIngestDocument(); // <7> + baseResult.getFailure(); // <8> + } + } + // end::simulate-pipeline-response + assert(response.getResults().size() > 0); + } + } + + public void testSimulatePipelineAsync() throws Exception { + RestHighLevelClient client = highLevelClient(); + + { + String source = + "{\"" + + "pipeline\":{" + + "\"description\":\"_description\"," + + "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]" + + "}," + + "\"docs\":[" + + "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"bar\"}}," + + "{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}" + + "]" + + "}"; + SimulatePipelineRequest request = new SimulatePipelineRequest( + new BytesArray(source.getBytes(StandardCharsets.UTF_8)), + XContentType.JSON + ); + + // tag::simulate-pipeline-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(SimulatePipelineResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::simulate-pipeline-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::simulate-pipeline-execute-async + client.ingest().simulatePipelineAsync(request, listener); // <1> + // end::simulate-pipeline-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + } diff --git a/docs/java-rest/high-level/ingest/simulate_pipeline.asciidoc b/docs/java-rest/high-level/ingest/simulate_pipeline.asciidoc new file mode 100644 index 0000000000000..9d1bbd06ceb26 --- /dev/null +++ b/docs/java-rest/high-level/ingest/simulate_pipeline.asciidoc @@ -0,0 +1,90 @@ +[[java-rest-high-ingest-simulate-pipeline]] +=== Simulate Pipeline API + +[[java-rest-high-ingest-simulate-pipeline-request]] +==== Simulate Pipeline Request + +A `SimulatePipelineRequest` requires a source and a `XContentType`. The source consists +of the request body. See the https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html[docs] +for more details on the request body. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-request] +-------------------------------------------------- +<1> The request body as a `ByteArray`. +<2> The XContentType for the request body supplied above. + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-request-pipeline-id] +-------------------------------------------------- +<1> You can either specify an existing pipeline to execute against the provided documents, or supply a +pipeline definition in the body of the request. This option sets the id for an existing pipeline. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-request-verbose] +-------------------------------------------------- +<1> To see the intermediate results of each processor in the simulate request, you can add the verbose parameter +to the request. + +[[java-rest-high-ingest-simulate-pipeline-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-execute] +-------------------------------------------------- +<1> Execute the request and get back the response in a `SimulatePipelineResponse` object. + +[[java-rest-high-ingest-simulate-pipeline-async]] +==== Asynchronous Execution + +The asynchronous execution of a simulate pipeline request requires both the `SimulatePipelineRequest` +instance and an `ActionListener` instance to be passed to the asynchronous +method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-execute-async] +-------------------------------------------------- +<1> The `SimulatePipelineRequest` to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `SimulatePipelineResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-ingest-simulate-pipeline-response]] +==== Simulate Pipeline Response + +The returned `SimulatePipelineResponse` allows to retrieve information about the executed + operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/IngestClientDocumentationIT.java[simulate-pipeline-response] +-------------------------------------------------- +<1> Get results for each of the documents provided as instance of `List`. +<2> If the request was in verbose mode cast the response to `SimulateDocumentVerboseResult`. +<3> Check the result after each processor is applied. +<4> Get the ingest document for the result obtained in 3. +<5> Or get the failure for the result obtained in 3. +<6> Get the result as `SimulateDocumentBaseResult` if the result was not verbose. +<7> Get the ingest document for the result obtained in 6. +<8> Or get the failure for the result obtained in 6. diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 4cd87a521d104..4314ad05adc4a 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -123,10 +123,12 @@ The Java High Level REST Client supports the following Ingest APIs: * <> * <> * <> +* <> include::ingest/put_pipeline.asciidoc[] include::ingest/get_pipeline.asciidoc[] include::ingest/delete_pipeline.asciidoc[] +include::ingest/simulate_pipeline.asciidoc[] == Snapshot APIs diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java index c6252feea276c..f7f76a2bbca7d 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java @@ -19,13 +19,18 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.ingest.IngestDocument; import java.io.IOException; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + /** * Holds the end result of what a pipeline did to sample document provided via the simulate api. */ @@ -33,6 +38,33 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult private final WriteableIngestDocument ingestDocument; private final Exception failure; + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "simulate_document_base_result", + true, + a -> { + if (a[1] == null) { + assert a[0] != null; + return new SimulateDocumentBaseResult(((WriteableIngestDocument)a[0]).getIngestDocument()); + } else { + assert a[0] == null; + return new SimulateDocumentBaseResult((ElasticsearchException)a[1]); + } + } + ); + static { + PARSER.declareObject( + optionalConstructorArg(), + WriteableIngestDocument.INGEST_DOC_PARSER, + new ParseField(WriteableIngestDocument.DOC_FIELD) + ); + PARSER.declareObject( + optionalConstructorArg(), + (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error") + ); + } + public SimulateDocumentBaseResult(IngestDocument ingestDocument) { this.ingestDocument = new WriteableIngestDocument(ingestDocument); failure = null; @@ -89,4 +121,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + public static SimulateDocumentBaseResult fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java index 21e802981850c..099e238f2d25e 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResult.java @@ -18,21 +18,38 @@ */ package org.elasticsearch.action.ingest; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + /** * Holds the result of what a pipeline did to a sample document via the simulate api, but instead of {@link SimulateDocumentBaseResult} * this result class holds the intermediate result each processor did to the sample document. */ public final class SimulateDocumentVerboseResult implements SimulateDocumentResult { + public static final String PROCESSOR_RESULT_FIELD = "processor_results"; private final List processorResults; + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "simulate_document_verbose_result", + true, + a -> new SimulateDocumentVerboseResult((List)a[0]) + ); + static { + PARSER.declareObjectArray(constructorArg(), SimulateProcessorResult.PARSER, new ParseField(PROCESSOR_RESULT_FIELD)); + } + public SimulateDocumentVerboseResult(List processorResults) { this.processorResults = processorResults; } @@ -63,7 +80,7 @@ public List getProcessorResults() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.startArray("processor_results"); + builder.startArray(PROCESSOR_RESULT_FIELD); for (SimulateProcessorResult processorResult : processorResults) { processorResult.toXContent(builder, params); } @@ -71,4 +88,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + public static SimulateDocumentVerboseResult fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 3aa697b8e997c..aa753ba0ad4b7 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -25,6 +25,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; @@ -42,7 +44,7 @@ import static org.elasticsearch.ingest.IngestDocument.MetaData; -public class SimulatePipelineRequest extends ActionRequest { +public class SimulatePipelineRequest extends ActionRequest implements ToXContentObject { private String id; private boolean verbose; @@ -126,6 +128,16 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (source != null) { + builder.rawValue(source.streamInput(), xContentType); + } else { + builder.startObject().endObject(); + } + return builder; + } + public static final class Fields { static final String PIPELINE = "pipeline"; static final String DOCS = "docs"; diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index e9ea1a7750738..8537b4daba12d 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -19,22 +19,93 @@ package org.elasticsearch.action.ingest; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + public class SimulatePipelineResponse extends ActionResponse implements ToXContentObject { private String pipelineId; private boolean verbose; private List results; + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "simulate_pipeline_response", + true, + a -> { + List results = (List)a[0]; + boolean verbose = false; + if (results.size() > 0) { + if (results.get(0) instanceof SimulateDocumentVerboseResult) { + verbose = true; + } + } + return new SimulatePipelineResponse(null, verbose, results); + } + ); + static { + PARSER.declareObjectArray( + constructorArg(), + (p, c) -> { + ensureExpectedToken(Token.START_OBJECT, p.currentToken(), p::getTokenLocation); + boolean isVerbose = false; + boolean isFirst = true; + SimulateDocumentResult result = null; + while (p.nextToken().equals(Token.FIELD_NAME)) { + switch (p.currentName()) { + case SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD: + assert isFirst || isVerbose; + isVerbose = true; + ensureExpectedToken(Token.START_ARRAY, p.nextToken(), p::getTokenLocation); + List results = new ArrayList<>(); + while (p.nextToken().equals(Token.START_OBJECT)) { + results.add(SimulateProcessorResult.fromXContent(p)); + } + ensureExpectedToken(Token.END_ARRAY, p.currentToken(), p::getTokenLocation); + result = new SimulateDocumentVerboseResult(results); + break; + case WriteableIngestDocument.DOC_FIELD: + case "error": + assert !isVerbose; + if (p.currentName().equals("error")) { + p.nextToken(); + result = new SimulateDocumentBaseResult(ElasticsearchException.fromXContent(p)); + } else { + result = new SimulateDocumentBaseResult( + WriteableIngestDocument.INGEST_DOC_PARSER.apply(p, null).getIngestDocument() + ); + } + ensureExpectedToken(Token.END_OBJECT, p.currentToken(), p::getTokenLocation); + break; + default: + p.skipChildren(); + break; + } + isFirst = false; + } + ensureExpectedToken(Token.END_OBJECT, p.currentToken(), p::getTokenLocation); + assert result != null; + return result; + }, + new ParseField(Fields.DOCUMENTS)); + } + public SimulatePipelineResponse() { } @@ -98,6 +169,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public static SimulatePipelineResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + static final class Fields { static final String DOCUMENTS = "docs"; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index 386a00b391f3c..79ed61630234e 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -19,33 +19,83 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import java.io.IOException; -class SimulateProcessorResult implements Writeable, ToXContentObject { +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +public class SimulateProcessorResult implements Writeable, ToXContentObject { + + private static final String IGNORED_ERROR_FIELD = "ignored_error"; private final String processorTag; private final WriteableIngestDocument ingestDocument; private final Exception failure; - SimulateProcessorResult(String processorTag, IngestDocument ingestDocument, Exception failure) { + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "simulate_processor_result", + true, + a -> { + String processorTag = a[0] == null ? null : (String)a[0]; + IngestDocument document = a[1] == null ? null : ((WriteableIngestDocument)a[1]).getIngestDocument(); + Exception failure = null; + if (a[2] != null) { + failure = (ElasticsearchException)a[2]; + } else if (a[3] != null) { + failure = (ElasticsearchException)a[3]; + } + return new SimulateProcessorResult(processorTag, document, failure); + } + ); + static { + PARSER.declareString(optionalConstructorArg(), new ParseField(ConfigurationUtils.TAG_KEY)); + PARSER.declareObject( + optionalConstructorArg(), + WriteableIngestDocument.INGEST_DOC_PARSER, + new ParseField(WriteableIngestDocument.DOC_FIELD) + ); + PARSER.declareObject( + optionalConstructorArg(), + (p, c) -> { + ensureExpectedToken(XContentParser.Token.START_OBJECT, p.currentToken(), p::getTokenLocation); + p.nextToken(); + ElasticsearchException e = ElasticsearchException.failureFromXContent(p); + ensureExpectedToken(XContentParser.Token.END_OBJECT, p.currentToken(), p::getTokenLocation); + p.nextToken(); + return e; + }, + new ParseField(IGNORED_ERROR_FIELD) + ); + PARSER.declareObject( + optionalConstructorArg(), + (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error") + ); + } + + public SimulateProcessorResult(String processorTag, IngestDocument ingestDocument, Exception failure) { this.processorTag = processorTag; this.ingestDocument = (ingestDocument == null) ? null : new WriteableIngestDocument(ingestDocument); this.failure = failure; } - SimulateProcessorResult(String processorTag, IngestDocument ingestDocument) { + public SimulateProcessorResult(String processorTag, IngestDocument ingestDocument) { this(processorTag, ingestDocument, null); } - SimulateProcessorResult(String processorTag, Exception failure) { + public SimulateProcessorResult(String processorTag, Exception failure) { this(processorTag, null, failure); } @@ -98,7 +148,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (failure != null && ingestDocument != null) { - builder.startObject("ignored_error"); + builder.startObject(IGNORED_ERROR_FIELD); ElasticsearchException.generateFailureXContent(builder, params, failure, true); builder.endObject(); } else if (failure != null) { @@ -112,4 +162,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + public static SimulateProcessorResult fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java index 87168cb7a9bba..2430868bb5909 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java @@ -20,24 +20,91 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.IngestDocument.MetaData; import java.io.IOException; import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; +import java.util.HashMap; import java.util.Map; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + final class WriteableIngestDocument implements Writeable, ToXContentFragment { + static final String SOURCE_FIELD = "_source"; + static final String INGEST_FIELD = "_ingest"; + static final String DOC_FIELD = "doc"; private final IngestDocument ingestDocument; + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser INGEST_DOC_PARSER = + new ConstructingObjectParser<>( + "ingest_document", + true, + a -> { + HashMap sourceAndMetadata = new HashMap<>(); + sourceAndMetadata.put(MetaData.INDEX.getFieldName(), a[0]); + sourceAndMetadata.put(MetaData.TYPE.getFieldName(), a[1]); + sourceAndMetadata.put(MetaData.ID.getFieldName(), a[2]); + if (a[3] != null) { + sourceAndMetadata.put(MetaData.ROUTING.getFieldName(), a[3]); + } + if (a[4] != null) { + sourceAndMetadata.put(MetaData.VERSION.getFieldName(), a[4]); + } + if (a[5] != null) { + sourceAndMetadata.put(MetaData.VERSION_TYPE.getFieldName(), a[5]); + } + sourceAndMetadata.putAll((Map)a[6]); + return new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, (Map)a[7])); + } + ); + static { + INGEST_DOC_PARSER.declareString(constructorArg(), new ParseField(MetaData.INDEX.getFieldName())); + INGEST_DOC_PARSER.declareString(constructorArg(), new ParseField(MetaData.TYPE.getFieldName())); + INGEST_DOC_PARSER.declareString(constructorArg(), new ParseField(MetaData.ID.getFieldName())); + INGEST_DOC_PARSER.declareString(optionalConstructorArg(), new ParseField(MetaData.ROUTING.getFieldName())); + INGEST_DOC_PARSER.declareLong(optionalConstructorArg(), new ParseField(MetaData.VERSION.getFieldName())); + INGEST_DOC_PARSER.declareString(optionalConstructorArg(), new ParseField(MetaData.VERSION_TYPE.getFieldName())); + INGEST_DOC_PARSER.declareObject(constructorArg(), (p, c) -> p.map(), new ParseField(SOURCE_FIELD)); + INGEST_DOC_PARSER.declareObject( + constructorArg(), + (p, c) -> { + Map ingestMap = p.map(); + ingestMap.computeIfPresent( + "timestamp", + (k, o) -> ZonedDateTime.parse((String)o) + ); + return ingestMap; + }, + new ParseField(INGEST_FIELD) + ); + } + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "writeable_ingest_document", + true, + a -> (WriteableIngestDocument)a[0] + ); + static { + PARSER.declareObject(constructorArg(), INGEST_DOC_PARSER, new ParseField(DOC_FIELD)); + } + WriteableIngestDocument(IngestDocument ingestDocument) { assert ingestDocument != null; this.ingestDocument = ingestDocument; @@ -67,19 +134,25 @@ IngestDocument getIngestDocument() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("doc"); - Map metadataMap = ingestDocument.extractMetadata(); + builder.startObject(DOC_FIELD); + Map metadataMap = ingestDocument.getMetadata(); for (Map.Entry metadata : metadataMap.entrySet()) { if (metadata.getValue() != null) { builder.field(metadata.getKey().getFieldName(), metadata.getValue().toString()); } } - builder.field("_source", ingestDocument.getSourceAndMetadata()); - builder.field("_ingest", ingestDocument.getIngestMetadata()); + Map source = IngestDocument.deepCopyMap(ingestDocument.getSourceAndMetadata()); + metadataMap.keySet().forEach(mD -> source.remove(mD.getFieldName())); + builder.field(SOURCE_FIELD, source); + builder.field(INGEST_FIELD, ingestDocument.getIngestMetadata()); builder.endObject(); return builder; } + public static WriteableIngestDocument fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index e31a97dc2c6ce..2bd842e72b107 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -570,6 +570,17 @@ public Map extractMetadata() { return metadataMap; } + /** + * Does the same thing as {@link #extractMetadata} but does not mutate the map. + */ + public Map getMetadata() { + Map metadataMap = new EnumMap<>(MetaData.class); + for (MetaData metaData : MetaData.values()) { + metadataMap.put(metaData, sourceAndMetadata.get(metaData.getFieldName())); + } + return metadataMap; + } + /** * Returns the available ingest metadata fields, by default only timestamp, but it is possible to set additional ones. * Use only for reading values, modify them instead using {@link #setFieldValue(String, Object)} and {@link #removeField(String)} @@ -588,7 +599,7 @@ public Map getSourceAndMetadata() { } @SuppressWarnings("unchecked") - private static Map deepCopyMap(Map source) { + public static Map deepCopyMap(Map source) { return (Map) deepCopy(source); } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java similarity index 59% rename from server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java rename to server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java index 83aad26f6a07b..c422f7b2ee1c9 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java @@ -21,27 +21,23 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; -public class SimulateDocumentSimpleResultTests extends ESTestCase { +public class SimulateDocumentBaseResultTests extends AbstractXContentTestCase { public void testSerialization() throws IOException { boolean isFailure = randomBoolean(); - SimulateDocumentBaseResult simulateDocumentBaseResult; - if (isFailure) { - simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); - } else { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); - simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); - } + SimulateDocumentBaseResult simulateDocumentBaseResult = createTestInstance(isFailure, true); BytesStreamOutput out = new BytesStreamOutput(); simulateDocumentBaseResult.writeTo(out); @@ -57,4 +53,53 @@ public void testSerialization() throws IOException { assertIngestDocument(otherSimulateDocumentBaseResult.getIngestDocument(), simulateDocumentBaseResult.getIngestDocument()); } } + + protected SimulateDocumentBaseResult createTestInstance(boolean isFailure, boolean withByteArray) { + SimulateDocumentBaseResult simulateDocumentBaseResult; + if (isFailure) { + simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); + } else { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArray); + simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); + } + return simulateDocumentBaseResult; + } + + @Override + protected SimulateDocumentBaseResult createTestInstance() { + return createTestInstance(randomBoolean(), false); + } + + @Override + protected SimulateDocumentBaseResult doParseInstance(XContentParser parser) { + return SimulateDocumentBaseResult.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + public static void assertEqualDocs(SimulateDocumentBaseResult response, SimulateDocumentBaseResult parsedResponse) { + assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); + if (response.getFailure() != null) { + assertNotNull(parsedResponse.getFailure()); + assertThat( + parsedResponse.getFailure().getMessage(), + containsString(response.getFailure().getMessage()) + ); + } else { + assertNull(parsedResponse.getFailure()); + } + } + + @Override + public void assertEqualInstances(SimulateDocumentBaseResult response, SimulateDocumentBaseResult parsedResponse) { + assertEqualDocs(response, parsedResponse); + } + + @Override + protected boolean assertToXContentEquivalence() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java new file mode 100644 index 0000000000000..18920630c5e86 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.action.ingest; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class SimulateDocumentVerboseResultTests extends AbstractXContentTestCase { + + @Override + protected SimulateDocumentVerboseResult createTestInstance() { + int numDocs = randomIntBetween(0, 10); + List results = new ArrayList<>(); + for (int i = 0; i { public void testSerialization() throws IOException { boolean isVerbose = randomBoolean(); String id = randomBoolean() ? randomAlphaOfLengthBetween(1, 10) : null; - int numResults = randomIntBetween(1, 10); - List results = new ArrayList<>(numResults); - for (int i = 0; i < numResults; i++) { - boolean isFailure = randomBoolean(); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); - if (isVerbose) { - int numProcessors = randomIntBetween(1, 10); - List processorResults = new ArrayList<>(numProcessors); - for (int j = 0; j < numProcessors; j++) { - String processorTag = randomAlphaOfLengthBetween(1, 10); - SimulateProcessorResult processorResult; - if (isFailure) { - processorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test")); - } else { - processorResult = new SimulateProcessorResult(processorTag, ingestDocument); - } - processorResults.add(processorResult); - } - results.add(new SimulateDocumentVerboseResult(processorResults)); - } else { - results.add(new SimulateDocumentBaseResult(ingestDocument)); - SimulateDocumentBaseResult simulateDocumentBaseResult; - if (isFailure) { - simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); - } else { - simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); - } - results.add(simulateDocumentBaseResult); - } - } - SimulatePipelineResponse response = new SimulatePipelineResponse(id, isVerbose, results); + SimulatePipelineResponse response = createInstance(id, isVerbose, true); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); StreamInput streamInput = out.bytes().streamInput(); @@ -120,4 +92,83 @@ public void testSerialization() throws IOException { } } } + + public static SimulatePipelineResponse createInstance(String pipelineId, boolean isVerbose, boolean withByteArraySource) { + int numResults = randomIntBetween(1, 10); + List results = new ArrayList<>(numResults); + for (int i = 0; i < numResults; i++) { + boolean isFailure = randomBoolean(); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArraySource); + if (isVerbose) { + int numProcessors = randomIntBetween(1, 10); + List processorResults = new ArrayList<>(numProcessors); + for (int j = 0; j < numProcessors; j++) { + String processorTag = randomAlphaOfLengthBetween(1, 10); + SimulateProcessorResult processorResult; + if (isFailure) { + processorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test")); + } else { + processorResult = new SimulateProcessorResult(processorTag, ingestDocument); + } + processorResults.add(processorResult); + } + results.add(new SimulateDocumentVerboseResult(processorResults)); + } else { + results.add(new SimulateDocumentBaseResult(ingestDocument)); + SimulateDocumentBaseResult simulateDocumentBaseResult; + if (isFailure) { + simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); + } else { + simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); + } + results.add(simulateDocumentBaseResult); + } + } + return new SimulatePipelineResponse(pipelineId, isVerbose, results); + } + + @Override + protected SimulatePipelineResponse createTestInstance() { + boolean isVerbose = randomBoolean(); + // since the pipeline id is not serialized with XContent we set it to null for equality tests. + return createInstance(null, isVerbose, false); + } + + @Override + protected SimulatePipelineResponse doParseInstance(XContentParser parser) { + return SimulatePipelineResponse.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + @Override + protected void assertEqualInstances(SimulatePipelineResponse response, + SimulatePipelineResponse parsedResponse) { + assertEquals(response.getPipelineId(), parsedResponse.getPipelineId()); + assertEquals(response.isVerbose(), parsedResponse.isVerbose()); + assertEquals(response.getResults().size(), parsedResponse.getResults().size()); + for (int i=0; i < response.getResults().size(); i++) { + if (response.isVerbose()) { + assert response.getResults().get(i) instanceof SimulateDocumentVerboseResult; + assert parsedResponse.getResults().get(i) instanceof SimulateDocumentVerboseResult; + SimulateDocumentVerboseResult responseResult = (SimulateDocumentVerboseResult)response.getResults().get(i); + SimulateDocumentVerboseResult parsedResult = (SimulateDocumentVerboseResult)parsedResponse.getResults().get(i); + SimulateDocumentVerboseResultTests.assertEqualDocuments(responseResult, parsedResult); + } else { + assert response.getResults().get(i) instanceof SimulateDocumentBaseResult; + assert parsedResponse.getResults().get(i) instanceof SimulateDocumentBaseResult; + SimulateDocumentBaseResult responseResult = (SimulateDocumentBaseResult)response.getResults().get(i); + SimulateDocumentBaseResult parsedResult = (SimulateDocumentBaseResult)parsedResponse.getResults().get(i); + SimulateDocumentBaseResultTests.assertEqualDocs(responseResult, parsedResult); + } + } + } + + @Override + protected boolean assertToXContentEquivalence() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java index 3014a1a4ae61d..7d042ef12f0e0 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java @@ -21,35 +21,26 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class SimulateProcessorResultTests extends ESTestCase { +public class SimulateProcessorResultTests extends AbstractXContentTestCase { public void testSerialization() throws IOException { - String processorTag = randomAlphaOfLengthBetween(1, 10); boolean isSuccessful = randomBoolean(); boolean isIgnoredException = randomBoolean(); - SimulateProcessorResult simulateProcessorResult; - if (isSuccessful) { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); - if (isIgnoredException) { - simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test")); - } else { - simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument); - } - } else { - simulateProcessorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test")); - } + SimulateProcessorResult simulateProcessorResult = createTestInstance(isSuccessful, isIgnoredException, true); BytesStreamOutput out = new BytesStreamOutput(); simulateProcessorResult.writeTo(out); @@ -72,4 +63,64 @@ public void testSerialization() throws IOException { assertThat(e.getMessage(), equalTo("test")); } } + + protected static SimulateProcessorResult createTestInstance(boolean isSuccessful, + boolean isIgnoredException, + boolean withByteArray) { + String processorTag = randomAlphaOfLengthBetween(1, 10); + SimulateProcessorResult simulateProcessorResult; + if (isSuccessful) { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArray); + if (isIgnoredException) { + simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test")); + } else { + simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument); + } + } else { + simulateProcessorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test")); + } + return simulateProcessorResult; + } + + @Override + protected SimulateProcessorResult createTestInstance() { + boolean isSuccessful = randomBoolean(); + boolean isIgnoredException = randomBoolean(); + return createTestInstance(isSuccessful, isIgnoredException, false); + } + + @Override + protected SimulateProcessorResult doParseInstance(XContentParser parser) { + return SimulateProcessorResult.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } + + protected static void assertEqualProcessorResults(SimulateProcessorResult response, + SimulateProcessorResult parsedResponse) { + assertEquals(response.getProcessorTag(), parsedResponse.getProcessorTag()); + assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); + if (response.getFailure() != null ) { + assertNotNull(parsedResponse.getFailure()); + assertThat( + parsedResponse.getFailure().getMessage(), + containsString(response.getFailure().getMessage()) + ); + } else { + assertNull(parsedResponse.getFailure()); + } + } + + @Override + protected void assertEqualInstances(SimulateProcessorResult response, SimulateProcessorResult parsedResponse) { + assertEqualProcessorResults(response, parsedResponse); + } + + @Override + protected boolean assertToXContentEquivalence() { + return false; + } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index 4d8e0f544c458..1fc9627e67fda 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -22,14 +22,18 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -40,7 +44,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; -public class WriteableIngestDocumentTests extends ESTestCase { +public class WriteableIngestDocumentTests extends AbstractXContentTestCase { public void testEqualsAndHashcode() throws Exception { Map sourceAndMetadata = RandomDocumentPicks.randomSource(random()); @@ -147,4 +151,71 @@ public void testToXContent() throws IOException { IngestDocument serializedIngestDocument = new IngestDocument(toXContentSource, toXContentIngestMetadata); assertThat(serializedIngestDocument, equalTo(serializedIngestDocument)); } + + public void testSourceFromXContentWithByteArray() throws IOException { + WriteableIngestDocument testInstance = new WriteableIngestDocument(RandomDocumentPicks.randomIngestDocument(random(), true)); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference shuffled = toShuffledXContent(testInstance, xContentType, ToXContent.EMPTY_PARAMS, false); + XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); + WriteableIngestDocument parsed = doParseInstance(parser); + assertTrue( + testSourceMapEquality( + testInstance.getIngestDocument().getSourceAndMetadata(), + parsed.getIngestDocument().getSourceAndMetadata() + ) + ); + } + + private boolean testSourceMapEquality(Map first, Map second) { + if (first == null) { + return second == null; + } else if (second == null) { + return false; + } else if (first.size() != second.size()) { + return false; + } else { + for (Map.Entry entry: first.entrySet()) { + Object value = entry.getValue(); + Object value2; + if ((value2 = second.get(entry.getKey())) != null) { + if (value2.getClass() == value.getClass()) { + if (value instanceof byte[]) { + if (!Arrays.equals((byte[])value, (byte[])value2)) { + return false; + } + } else if (value instanceof Map) { + //noinspection unchecked + if (!testSourceMapEquality((Map)value, (Map)value2)) { + return false; + } + } else if (!value.equals(value2)) { + return false; + } + } + } else { + return false; + } + } + return true; + } + } + + @Override + protected WriteableIngestDocument createTestInstance() { + // We want to create a test instance without byte arrays. For testing byte arrays see testSourceFromXContentWithByteArray. + // This is needed because for comparing byte[] Object.equal fails + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), false); + return new WriteableIngestDocument(new IngestDocument(ingestDocument)); + } + + @Override + protected WriteableIngestDocument doParseInstance(XContentParser parser) { + return WriteableIngestDocument.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + // Cannot support unknown fields because equality changes if new keys are added to _source + return false; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java index 58eb1df129291..fb354209a97b2 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java @@ -131,6 +131,10 @@ public static IngestDocument randomIngestDocument(Random random) { return randomIngestDocument(random, randomSource(random)); } + public static IngestDocument randomIngestDocument(Random random, boolean withByteArray) { + return randomIngestDocument(random, randomSource(random, withByteArray)); + } + /** * Generates a document that holds random metadata and the document provided as a map argument */ @@ -149,8 +153,12 @@ public static IngestDocument randomIngestDocument(Random random, Map randomSource(Random random) { + return randomSource(random, true); + } + + public static Map randomSource(Random random, boolean withByteArray) { Map document = new HashMap<>(); - addRandomFields(random, document, 0); + addRandomFields(random, document, 0, withByteArray); return document; } @@ -158,11 +166,15 @@ public static Map randomSource(Random random) { * Generates a random field value, can be a string, a number, a list of an object itself. */ public static Object randomFieldValue(Random random) { - return randomFieldValue(random, 0); + return randomFieldValue(random, 0, true); + } + + public static Object randomFieldValue(Random random, boolean withByteArray) { + return randomFieldValue(random, 0, withByteArray); } - private static Object randomFieldValue(Random random, int currentDepth) { - switch(RandomNumbers.randomIntBetween(random, 0, 9)) { + private static Object randomFieldValue(Random random, int currentDepth, boolean withByteArray) { + switch(RandomNumbers.randomIntBetween(random, 0, withByteArray ? 9 : 8)) { case 0: return randomString(random); case 1: @@ -201,7 +213,7 @@ private static Object randomFieldValue(Random random, int currentDepth) { return doubleList; case 8: Map newNode = new HashMap<>(); - addRandomFields(random, newNode, ++currentDepth); + addRandomFields(random, newNode, ++currentDepth, withByteArray); return newNode; case 9: byte[] byteArray = new byte[RandomNumbers.randomIntBetween(random, 1, 1024)]; @@ -224,14 +236,14 @@ private static Long randomNonNegtiveLong(Random random) { return randomLong == Long.MIN_VALUE ? 0 : Math.abs(randomLong); } - private static void addRandomFields(Random random, Map parentNode, int currentDepth) { + private static void addRandomFields(Random random, Map parentNode, int currentDepth, boolean withByteArray) { if (currentDepth > 5) { return; } int numFields = RandomNumbers.randomIntBetween(random, 1, 10); for (int i = 0; i < numFields; i++) { String fieldName = randomLeafFieldName(random); - Object fieldValue = randomFieldValue(random, currentDepth); + Object fieldValue = randomFieldValue(random, currentDepth, withByteArray); parentNode.put(fieldName, fieldValue); } } From 3497e0c82d74ab769c1fa070b96400e8f705764d Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Thu, 14 Jun 2018 02:39:17 +0200 Subject: [PATCH 2/6] Fixes after review -- removed changed to RandomDocumentPicks --- .../elasticsearch/client/IngestClient.java | 21 +++-- .../client/RequestConverters.java | 2 +- .../elasticsearch/client/IngestClientIT.java | 88 ++++++++++--------- .../client/RequestConvertersTests.java | 7 +- .../IngestClientDocumentationIT.java | 4 +- .../ingest/SimulatePipelineRequest.java | 6 +- .../ingest/SimulatePipelineResponse.java | 1 + .../ingest/SimulateProcessorResult.java | 26 ++++-- .../SimulateDocumentBaseResultTests.java | 11 +-- .../SimulateDocumentVerboseResultTests.java | 8 +- .../ingest/SimulatePipelineResponseTests.java | 38 ++++++-- .../ingest/SimulateProcessorResultTests.java | 51 ++++++++--- .../ingest/WriteableIngestDocumentTests.java | 69 ++++----------- .../ingest/RandomDocumentPicks.java | 26 ++---- .../hamcrest/ElasticsearchAssertions.java | 23 +++-- 15 files changed, 208 insertions(+), 173 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java index f4bba28386a12..340e14653971b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java @@ -134,10 +134,14 @@ public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions op * See * * Simulate Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response */ - public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::simulatePipeline, - SimulatePipelineResponse::fromXContent, emptySet(), headers); + public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::simulatePipeline, options, + SimulatePipelineResponse::fromXContent, emptySet()); } /** @@ -146,11 +150,14 @@ public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request * See * * Simulate Pipeline API on elastic.co + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion */ public void simulatePipelineAsync(SimulatePipelineRequest request, - ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::simulatePipeline, - SimulatePipelineResponse::fromXContent, listener, emptySet(), headers); + RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::simulatePipeline, options, + SimulatePipelineResponse::fromXContent, listener, emptySet()); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 40b5c3c7df43c..cc5111526fe84 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -877,7 +877,7 @@ static Request simulatePipeline(SimulatePipelineRequest simulatePipelineRequest) if (simulatePipelineRequest.getId() != null && !simulatePipelineRequest.getId().isEmpty()) { builder.addPathPart(simulatePipelineRequest.getId()); } - builder.addPathPart("_simulate"); + builder.addPathPartAsIs("_simulate"); String endpoint = builder.build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); Params params = new Params(request); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 909db80f3417e..8a8cb4019ed17 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -32,12 +32,9 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.PipelineConfiguration; -import org.elasticsearch.ingest.RandomDocumentPicks; import java.io.IOException; -import java.util.Map; public class IngestClientIT extends ESRestHighLevelClientTestCase { @@ -93,7 +90,6 @@ public void testDeletePipeline() throws IOException { public void testSimulatePipeline() throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); - int numDocs = randomIntBetween(1, 10); boolean isVerbose = randomBoolean(); builder.startObject(); { @@ -101,23 +97,18 @@ public void testSimulatePipeline() throws IOException { buildRandomXContentPipeline(builder); builder.startArray("docs"); { - for (int i = 0; i < numDocs; i++) { - builder.startObject(); - IngestDocument document = RandomDocumentPicks.randomIngestDocument(random()); - Map metadataMap = document.extractMetadata(); - for (Map.Entry metadata : metadataMap.entrySet()) { - if (metadata.getValue() != null) { - if (metadata.getKey().equals(IngestDocument.MetaData.VERSION)) { - builder.field(metadata.getKey().getFieldName(), (long)metadata.getValue()); - } else { - builder.field(metadata.getKey().getFieldName(), metadata.getValue().toString()); - } - } - } - document.setFieldValue("rank", Integer.toString(randomInt())); - builder.field("_source", document.getSourceAndMetadata()); - builder.endObject(); - } + builder.startObject() + .field("_index", "index") + .field("_type", "doc") + .field("_id", "doc_" + 1) + .startObject("_source").field("foo", "rab_" + 1).field("rank", "1234").endObject() + .endObject(); + builder.startObject() + .field("_index", "index") + .field("_type", "doc") + .field("_id", "doc_" + 2) + .startObject("_source").field("foo", "rab_" + 1).field("rank", "non-int").endObject() + .endObject(); } builder.endArray(); } @@ -132,28 +123,39 @@ public void testSimulatePipeline() throws IOException { SimulatePipelineResponse simulatePipelineResponse = execute(request, highLevelClient().ingest()::simulatePipeline, highLevelClient().ingest()::simulatePipelineAsync); - for (SimulateDocumentResult result: simulatePipelineResponse.getResults()) { - if (isVerbose) { - assertTrue(result instanceof SimulateDocumentVerboseResult); - SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult)result; - assertTrue(verboseResult.getProcessorResults().size() > 0); - assertEquals( - verboseResult.getProcessorResults().get(0).getIngestDocument() - .getFieldValue("foo", String.class), - "bar" - ); - } else { - assertTrue(result instanceof SimulateDocumentBaseResult); - SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result; - assertNotNull(baseResult.getIngestDocument()); - assertEquals( - baseResult.getIngestDocument().getFieldValue("foo", String.class), - "bar" - ); - assertNotNull( - baseResult.getIngestDocument().getFieldValue("rank", Integer.class) - ); - } + SimulateDocumentResult result0 = simulatePipelineResponse.getResults().get(0); + SimulateDocumentResult result1 = simulatePipelineResponse.getResults().get(1); + if (isVerbose) { + assertTrue(result0 instanceof SimulateDocumentVerboseResult); + SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult)result0; + SimulateDocumentVerboseResult failedVerboseResult = (SimulateDocumentVerboseResult)result1; + assertTrue(verboseResult.getProcessorResults().size() > 0); + assertEquals( + verboseResult.getProcessorResults().get(0).getIngestDocument() + .getFieldValue("foo", String.class), + "bar" + ); + assertEquals( + Integer.valueOf(1234), + verboseResult.getProcessorResults().get(1).getIngestDocument() + .getFieldValue("rank", Integer.class) + ); + assertNotNull(failedVerboseResult.getProcessorResults().get(1).getFailure()); + } else { + assertTrue(result0 instanceof SimulateDocumentBaseResult); + SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result0; + SimulateDocumentBaseResult failedBaseResult = (SimulateDocumentBaseResult)result0; + assertNotNull(baseResult.getIngestDocument()); + assertEquals( + baseResult.getIngestDocument().getFieldValue("foo", String.class), + "bar" + ); + assertEquals( + Integer.valueOf(1234), + baseResult.getIngestDocument() + .getFieldValue("rank", Integer.class) + ); + assertNotNull(failedBaseResult.getFailure()); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 590ed2e6ea8ad..ecca83bfd01ae 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1535,8 +1535,12 @@ public void testDeletePipeline() { public void testSimulatePipeline() throws IOException { String pipelineId = randomBoolean() ? "some_pipeline_id" : null; boolean verbose = randomBoolean(); + String json = "{\"pipeline\":{" + + "\"description\":\"_description\"," + + "\"processors\":[{\"set\":{\"field\":\"field2\",\"value\":\"_value\"}}]}," + + "\"docs\":[{\"_index\":\"index\",\"_type\":\"_doc\",\"_id\":\"id\",\"_source\":{\"foo\":\"rab\"}}]}"; SimulatePipelineRequest request = new SimulatePipelineRequest( - new BytesArray("{}".getBytes(StandardCharsets.UTF_8)), + new BytesArray(json.getBytes(StandardCharsets.UTF_8)), XContentType.JSON ); request.setId(pipelineId); @@ -1553,6 +1557,7 @@ public void testSimulatePipeline() throws IOException { assertEquals(endpoint.toString(), expectedRequest.getEndpoint()); assertEquals(HttpPost.METHOD_NAME, expectedRequest.getMethod()); assertEquals(expectedParams, expectedRequest.getParameters()); + assertToXContentBody(request, expectedRequest.getEntity()); } public void testClusterHealth() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java index 7636f77d66837..c53ec2b5d7cc7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IngestClientDocumentationIT.java @@ -317,7 +317,7 @@ public void testSimulatePipeline() throws IOException { // end::simulate-pipeline-request-verbose // tag::simulate-pipeline-execute - SimulatePipelineResponse response = client.ingest().simulatePipeline(request); // <1> + SimulatePipelineResponse response = client.ingest().simulatePipeline(request, RequestOptions.DEFAULT); // <1> // end::simulate-pipeline-execute // tag::simulate-pipeline-response @@ -381,7 +381,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::simulate-pipeline-execute-async - client.ingest().simulatePipelineAsync(request, listener); // <1> + client.ingest().simulatePipelineAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::simulate-pipeline-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index aa753ba0ad4b7..9a7d6bb7feea9 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -130,11 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (source != null) { - builder.rawValue(source.streamInput(), xContentType); - } else { - builder.startObject().endObject(); - } + builder.rawValue(source.streamInput(), xContentType); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index 8537b4daba12d..42473355c80f6 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -94,6 +94,7 @@ public class SimulatePipelineResponse extends ActionResponse implements ToXConte ensureExpectedToken(Token.END_OBJECT, p.currentToken(), p::getTokenLocation); break; default: + p.nextToken(); p.skipChildren(); break; } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java index 79ed61630234e..101ce7ec260e1 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java @@ -33,7 +33,7 @@ import java.io.IOException; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; -import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; public class SimulateProcessorResult implements Writeable, ToXContentObject { @@ -42,6 +42,21 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject { private final WriteableIngestDocument ingestDocument; private final Exception failure; + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser IGNORED_ERROR_PARSER = + new ConstructingObjectParser<>( + "ignored_error_parser", + true, + a -> (ElasticsearchException)a[0] + ); + static { + IGNORED_ERROR_PARSER.declareObject( + constructorArg(), + (p, c) -> ElasticsearchException.fromXContent(p), + new ParseField("error") + ); + } + @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -68,14 +83,7 @@ public class SimulateProcessorResult implements Writeable, ToXContentObject { ); PARSER.declareObject( optionalConstructorArg(), - (p, c) -> { - ensureExpectedToken(XContentParser.Token.START_OBJECT, p.currentToken(), p::getTokenLocation); - p.nextToken(); - ElasticsearchException e = ElasticsearchException.failureFromXContent(p); - ensureExpectedToken(XContentParser.Token.END_OBJECT, p.currentToken(), p::getTokenLocation); - p.nextToken(); - return e; - }, + IGNORED_ERROR_PARSER, new ParseField(IGNORED_ERROR_FIELD) ); PARSER.declareObject( diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java index c422f7b2ee1c9..af7b3245c94b6 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java @@ -32,12 +32,13 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.assertEqualIngestDocs; public class SimulateDocumentBaseResultTests extends AbstractXContentTestCase { public void testSerialization() throws IOException { boolean isFailure = randomBoolean(); - SimulateDocumentBaseResult simulateDocumentBaseResult = createTestInstance(isFailure, true); + SimulateDocumentBaseResult simulateDocumentBaseResult = createTestInstance(isFailure); BytesStreamOutput out = new BytesStreamOutput(); simulateDocumentBaseResult.writeTo(out); @@ -54,12 +55,12 @@ public void testSerialization() throws IOException { } } - protected SimulateDocumentBaseResult createTestInstance(boolean isFailure, boolean withByteArray) { + protected SimulateDocumentBaseResult createTestInstance(boolean isFailure) { SimulateDocumentBaseResult simulateDocumentBaseResult; if (isFailure) { simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); } else { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArray); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); } return simulateDocumentBaseResult; @@ -67,7 +68,7 @@ protected SimulateDocumentBaseResult createTestInstance(boolean isFailure, boole @Override protected SimulateDocumentBaseResult createTestInstance() { - return createTestInstance(randomBoolean(), false); + return createTestInstance(randomBoolean()); } @Override @@ -81,7 +82,7 @@ protected boolean supportsUnknownFields() { } public static void assertEqualDocs(SimulateDocumentBaseResult response, SimulateDocumentBaseResult parsedResponse) { - assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); + assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); if (response.getFailure() != null) { assertNotNull(parsedResponse.getFailure()); assertThat( diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java index 18920630c5e86..363ce36ce115a 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java @@ -32,7 +32,7 @@ protected SimulateDocumentVerboseResult createTestInstance() { List results = new ArrayList<>(); for (int i = 0; i results = new ArrayList<>(numResults); for (int i = 0; i < numResults; i++) { - boolean isFailure = randomBoolean(); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArraySource); + boolean isFailure = withFailure && randomBoolean(); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); if (isVerbose) { int numProcessors = randomIntBetween(1, 10); List processorResults = new ArrayList<>(numProcessors); @@ -127,10 +128,16 @@ public static SimulatePipelineResponse createInstance(String pipelineId, boolean return new SimulatePipelineResponse(pipelineId, isVerbose, results); } + private static SimulatePipelineResponse createTestInstanceWithFailures() { + boolean isVerbose = randomBoolean(); + return createInstance(null, isVerbose, false); + } + @Override protected SimulatePipelineResponse createTestInstance() { boolean isVerbose = randomBoolean(); // since the pipeline id is not serialized with XContent we set it to null for equality tests. + // we test failures separately since comparing XContent is not possible with failures return createInstance(null, isVerbose, false); } @@ -141,7 +148,7 @@ protected SimulatePipelineResponse doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } @Override @@ -156,7 +163,7 @@ protected void assertEqualInstances(SimulatePipelineResponse response, assert parsedResponse.getResults().get(i) instanceof SimulateDocumentVerboseResult; SimulateDocumentVerboseResult responseResult = (SimulateDocumentVerboseResult)response.getResults().get(i); SimulateDocumentVerboseResult parsedResult = (SimulateDocumentVerboseResult)parsedResponse.getResults().get(i); - SimulateDocumentVerboseResultTests.assertEqualDocuments(responseResult, parsedResult); + SimulateDocumentVerboseResultTests.assertEqualDocs(responseResult, parsedResult); } else { assert response.getResults().get(i) instanceof SimulateDocumentBaseResult; assert parsedResponse.getResults().get(i) instanceof SimulateDocumentBaseResult; @@ -168,7 +175,22 @@ protected void assertEqualInstances(SimulatePipelineResponse response, } @Override - protected boolean assertToXContentEquivalence() { - return false; + protected Predicate getRandomFieldsExcludeFilter() { + // We cannot have random fields in the _source field + return field -> field.contains("doc._source") || field.contains("doc._ingest"); + } + + /** + * Test parsing {@link SimulatePipelineResponse} with inner failures as they don't support asserting on xcontent equivalence, given that + * exceptions are not parsed back as the same original class. We run the usual {@link AbstractXContentTestCase#testFromXContent()} + * without failures, and this other test with failures where we disable asserting on xcontent equivalence at the end. + */ + public void testFromXContentWithFailures() throws IOException { + Supplier instanceSupplier = SimulatePipelineResponseTests::createTestInstanceWithFailures; + //exceptions are not of the same type whenever parsed back + boolean assertToXContentEquivalence = false; + AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields(), getShuffleFieldsExceptions(), + getRandomFieldsExcludeFilter(), this::createParser, this::doParseInstance, + this::assertEqualInstances, assertToXContentEquivalence, getToXContentParams()); } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java index 7d042ef12f0e0..81db521993b7f 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.util.function.Predicate; +import java.util.function.Supplier; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.hamcrest.CoreMatchers.containsString; @@ -34,13 +36,14 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.assertEqualIngestDocs; public class SimulateProcessorResultTests extends AbstractXContentTestCase { public void testSerialization() throws IOException { boolean isSuccessful = randomBoolean(); boolean isIgnoredException = randomBoolean(); - SimulateProcessorResult simulateProcessorResult = createTestInstance(isSuccessful, isIgnoredException, true); + SimulateProcessorResult simulateProcessorResult = createTestInstance(isSuccessful, isIgnoredException); BytesStreamOutput out = new BytesStreamOutput(); simulateProcessorResult.writeTo(out); @@ -65,12 +68,11 @@ public void testSerialization() throws IOException { } protected static SimulateProcessorResult createTestInstance(boolean isSuccessful, - boolean isIgnoredException, - boolean withByteArray) { + boolean isIgnoredException) { String processorTag = randomAlphaOfLengthBetween(1, 10); SimulateProcessorResult simulateProcessorResult; if (isSuccessful) { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), withByteArray); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); if (isIgnoredException) { simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test")); } else { @@ -82,11 +84,16 @@ protected static SimulateProcessorResult createTestInstance(boolean isSuccessful return simulateProcessorResult; } - @Override - protected SimulateProcessorResult createTestInstance() { + private static SimulateProcessorResult createTestInstanceWithFailures() { boolean isSuccessful = randomBoolean(); boolean isIgnoredException = randomBoolean(); - return createTestInstance(isSuccessful, isIgnoredException, false); + return createTestInstance(isSuccessful, isIgnoredException); + } + + @Override + protected SimulateProcessorResult createTestInstance() { + // we test failures separately since comparing XContent is not possible with failures + return createTestInstance(true, false); } @Override @@ -96,13 +103,19 @@ protected SimulateProcessorResult doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; } - protected static void assertEqualProcessorResults(SimulateProcessorResult response, + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // We cannot have random fields in the _source field and _ingest field + return field -> field.contains("doc._source") || field.contains("doc._ingest"); + } + + static void assertEqualProcessorResults(SimulateProcessorResult response, SimulateProcessorResult parsedResponse) { assertEquals(response.getProcessorTag(), parsedResponse.getProcessorTag()); - assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); + assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); if (response.getFailure() != null ) { assertNotNull(parsedResponse.getFailure()); assertThat( @@ -119,8 +132,20 @@ protected void assertEqualInstances(SimulateProcessorResult response, SimulatePr assertEqualProcessorResults(response, parsedResponse); } - @Override - protected boolean assertToXContentEquivalence() { - return false; + /** + * Test parsing {@link SimulateProcessorResult} with inner failures as they don't support asserting on xcontent equivalence, given that + * exceptions are not parsed back as the same original class. We run the usual {@link AbstractXContentTestCase#testFromXContent()} + * without failures, and this other test with failures where we disable asserting on xcontent equivalence at the end. + */ + public void testFromXContentWithFailures() throws IOException { + Supplier instanceSupplier = SimulateProcessorResultTests::createTestInstanceWithFailures; + //with random fields insertion in the inner exceptions, some random stuff may be parsed back as metadata, + //but that does not bother our assertions, as we only want to test that we don't break. + boolean supportsUnknownFields = true; + //exceptions are not of the same type whenever parsed back + boolean assertToXContentEquivalence = false; + AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields, + getShuffleFieldsExceptions(), getRandomFieldsExcludeFilter(), this::createParser, this::doParseInstance, + this::assertEqualInstances, assertToXContentEquivalence, getToXContentParams()); } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index 1fc9627e67fda..1f12ba82e53a8 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -22,18 +22,16 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import java.io.IOException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -152,59 +150,30 @@ public void testToXContent() throws IOException { assertThat(serializedIngestDocument, equalTo(serializedIngestDocument)); } - public void testSourceFromXContentWithByteArray() throws IOException { - WriteableIngestDocument testInstance = new WriteableIngestDocument(RandomDocumentPicks.randomIngestDocument(random(), true)); - XContentType xContentType = randomFrom(XContentType.values()); - BytesReference shuffled = toShuffledXContent(testInstance, xContentType, ToXContent.EMPTY_PARAMS, false); - XContentParser parser = createParser(XContentFactory.xContent(xContentType), shuffled); - WriteableIngestDocument parsed = doParseInstance(parser); - assertTrue( - testSourceMapEquality( - testInstance.getIngestDocument().getSourceAndMetadata(), - parsed.getIngestDocument().getSourceAndMetadata() - ) - ); - } - - private boolean testSourceMapEquality(Map first, Map second) { - if (first == null) { - return second == null; - } else if (second == null) { - return false; - } else if (first.size() != second.size()) { - return false; + protected static void assertEqualIngestDocs(IngestDocument response, IngestDocument parsedResponse) { + if (response != null && parsedResponse != null) { + ElasticsearchAssertions.assertMapEquals( + response.getSourceAndMetadata(), + parsedResponse.getSourceAndMetadata(), + true + ); + assertEquals(response.getIngestMetadata(), parsedResponse.getIngestMetadata()); + } else if (response == null) { + assertNull(parsedResponse); } else { - for (Map.Entry entry: first.entrySet()) { - Object value = entry.getValue(); - Object value2; - if ((value2 = second.get(entry.getKey())) != null) { - if (value2.getClass() == value.getClass()) { - if (value instanceof byte[]) { - if (!Arrays.equals((byte[])value, (byte[])value2)) { - return false; - } - } else if (value instanceof Map) { - //noinspection unchecked - if (!testSourceMapEquality((Map)value, (Map)value2)) { - return false; - } - } else if (!value.equals(value2)) { - return false; - } - } - } else { - return false; - } - } - return true; + fail("parsed response was null but the expected was non null."); } + + } + + @Override + protected void assertEqualInstances(WriteableIngestDocument response, WriteableIngestDocument parsedResponse) { + assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); } @Override protected WriteableIngestDocument createTestInstance() { - // We want to create a test instance without byte arrays. For testing byte arrays see testSourceFromXContentWithByteArray. - // This is needed because for comparing byte[] Object.equal fails - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), false); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); return new WriteableIngestDocument(new IngestDocument(ingestDocument)); } diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java index fb354209a97b2..58eb1df129291 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java @@ -131,10 +131,6 @@ public static IngestDocument randomIngestDocument(Random random) { return randomIngestDocument(random, randomSource(random)); } - public static IngestDocument randomIngestDocument(Random random, boolean withByteArray) { - return randomIngestDocument(random, randomSource(random, withByteArray)); - } - /** * Generates a document that holds random metadata and the document provided as a map argument */ @@ -153,12 +149,8 @@ public static IngestDocument randomIngestDocument(Random random, Map randomSource(Random random) { - return randomSource(random, true); - } - - public static Map randomSource(Random random, boolean withByteArray) { Map document = new HashMap<>(); - addRandomFields(random, document, 0, withByteArray); + addRandomFields(random, document, 0); return document; } @@ -166,15 +158,11 @@ public static Map randomSource(Random random, boolean withByteAr * Generates a random field value, can be a string, a number, a list of an object itself. */ public static Object randomFieldValue(Random random) { - return randomFieldValue(random, 0, true); - } - - public static Object randomFieldValue(Random random, boolean withByteArray) { - return randomFieldValue(random, 0, withByteArray); + return randomFieldValue(random, 0); } - private static Object randomFieldValue(Random random, int currentDepth, boolean withByteArray) { - switch(RandomNumbers.randomIntBetween(random, 0, withByteArray ? 9 : 8)) { + private static Object randomFieldValue(Random random, int currentDepth) { + switch(RandomNumbers.randomIntBetween(random, 0, 9)) { case 0: return randomString(random); case 1: @@ -213,7 +201,7 @@ private static Object randomFieldValue(Random random, int currentDepth, boolean return doubleList; case 8: Map newNode = new HashMap<>(); - addRandomFields(random, newNode, ++currentDepth, withByteArray); + addRandomFields(random, newNode, ++currentDepth); return newNode; case 9: byte[] byteArray = new byte[RandomNumbers.randomIntBetween(random, 1, 1024)]; @@ -236,14 +224,14 @@ private static Long randomNonNegtiveLong(Random random) { return randomLong == Long.MIN_VALUE ? 0 : Math.abs(randomLong); } - private static void addRandomFields(Random random, Map parentNode, int currentDepth, boolean withByteArray) { + private static void addRandomFields(Random random, Map parentNode, int currentDepth) { if (currentDepth > 5) { return; } int numFields = RandomNumbers.randomIntBetween(random, 1, 10); for (int i = 0; i < numFields; i++) { String fieldName = randomLeafFieldName(random); - Object fieldValue = randomFieldValue(random, currentDepth, withByteArray); + Object fieldValue = randomFieldValue(random, currentDepth); parentNode.put(fieldName, fieldValue); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index cf3cc39d34d88..a969461554648 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -64,6 +64,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -669,6 +670,10 @@ public static void assertToXContentEquivalent(BytesReference expected, BytesRefe * Compares two maps recursively, using arrays comparisons for byte[] through Arrays.equals(byte[], byte[]) */ private static void assertMapEquals(Map expected, Map actual) { + assertMapEquals(expected, actual, false); + } + + public static void assertMapEquals(Map expected, Map actual, boolean convertBase64Strings) { assertEquals(expected.size(), actual.size()); for (Map.Entry expectedEntry : expected.entrySet()) { String expectedKey = expectedEntry.getKey(); @@ -677,7 +682,7 @@ private static void assertMapEquals(Map expected, Map expected, Map expected, List actual) { + private static void assertListEquals(List expected, List actual, boolean convertBase64Strings) { assertEquals(expected.size(), actual.size()); Iterator actualIterator = actual.iterator(); for (Object expectedValue : expected) { Object actualValue = actualIterator.next(); - assertObjectEquals(expectedValue, actualValue); + assertObjectEquals(expectedValue, actualValue, convertBase64Strings); } } @@ -700,15 +705,21 @@ private static void assertListEquals(List expected, List actual) * for byte[] through Arrays.equals(byte[], byte[]) */ @SuppressWarnings("unchecked") - private static void assertObjectEquals(Object expected, Object actual) { + private static void assertObjectEquals(Object expected, Object actual, boolean convertBase64Strings) { if (expected instanceof Map) { assertThat(actual, instanceOf(Map.class)); - assertMapEquals((Map) expected, (Map) actual); + assertMapEquals((Map) expected, (Map) actual, convertBase64Strings); } else if (expected instanceof List) { - assertListEquals((List) expected, (List) actual); + assertListEquals((List) expected, (List) actual, convertBase64Strings); } else if (expected instanceof byte[]) { //byte[] is really a special case for binary values when comparing SMILE and CBOR, arrays of other types //don't need to be handled. Ordinary arrays get parsed as lists. + if (convertBase64Strings && actual instanceof String) { + // for objects deserialized from XContent the content might be a Base64 encoded string + actual = Base64.getDecoder().decode((String)actual); + } else { + assertTrue(actual instanceof byte[]); + } assertArrayEquals((byte[]) expected, (byte[]) actual); } else { assertEquals(expected, actual); From 1de794e3c62ca4ef49f1946768fad10f3da3ed78 Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Thu, 14 Jun 2018 06:11:35 +0200 Subject: [PATCH 3/6] fixed integ test --- .../src/test/java/org/elasticsearch/client/IngestClientIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 8a8cb4019ed17..39beef9e74861 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -144,7 +144,7 @@ public void testSimulatePipeline() throws IOException { } else { assertTrue(result0 instanceof SimulateDocumentBaseResult); SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result0; - SimulateDocumentBaseResult failedBaseResult = (SimulateDocumentBaseResult)result0; + SimulateDocumentBaseResult failedBaseResult = (SimulateDocumentBaseResult)result1; assertNotNull(baseResult.getIngestDocument()); assertEquals( baseResult.getIngestDocument().getFieldValue("foo", String.class), From d08dcb50321392a02df518cb16909550950760de Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Mon, 18 Jun 2018 13:08:09 +0200 Subject: [PATCH 4/6] Fixes after review (2) -- Started using RandomObjects instead of RandomDocumentPicks -- Removed changes to ElasticsearchAssertions --- .../ingest/SimulatePipelineResponse.java | 39 +++++-------- .../SimulateDocumentBaseResultTests.java | 53 ++++++++++++++--- .../SimulateDocumentVerboseResultTests.java | 58 +++++++++++++++++-- .../ingest/SimulatePipelineResponseTests.java | 56 ++++++++---------- .../ingest/SimulateProcessorResultTests.java | 22 +++++-- .../ingest/WriteableIngestDocumentTests.java | 47 ++++++++------- .../hamcrest/ElasticsearchAssertions.java | 23 ++------ 7 files changed, 184 insertions(+), 114 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index 42473355c80f6..93a6d02a13f55 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -62,45 +62,38 @@ public class SimulatePipelineResponse extends ActionResponse implements ToXConte static { PARSER.declareObjectArray( constructorArg(), - (p, c) -> { - ensureExpectedToken(Token.START_OBJECT, p.currentToken(), p::getTokenLocation); - boolean isVerbose = false; - boolean isFirst = true; + (parser, context) -> { + ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); SimulateDocumentResult result = null; - while (p.nextToken().equals(Token.FIELD_NAME)) { - switch (p.currentName()) { + while (parser.nextToken() != Token.END_OBJECT) { + ensureExpectedToken(parser.currentToken(), Token.FIELD_NAME, parser::getTokenLocation); + String fieldName = parser.currentName(); + parser.nextToken(); + switch(fieldName) { case SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD: - assert isFirst || isVerbose; - isVerbose = true; - ensureExpectedToken(Token.START_ARRAY, p.nextToken(), p::getTokenLocation); + ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); List results = new ArrayList<>(); - while (p.nextToken().equals(Token.START_OBJECT)) { - results.add(SimulateProcessorResult.fromXContent(p)); + while (parser.nextToken().equals(Token.START_OBJECT)) { + results.add(SimulateProcessorResult.fromXContent(parser)); } - ensureExpectedToken(Token.END_ARRAY, p.currentToken(), p::getTokenLocation); + ensureExpectedToken(Token.END_ARRAY, parser.currentToken(), parser::getTokenLocation); result = new SimulateDocumentVerboseResult(results); break; case WriteableIngestDocument.DOC_FIELD: case "error": - assert !isVerbose; - if (p.currentName().equals("error")) { - p.nextToken(); - result = new SimulateDocumentBaseResult(ElasticsearchException.fromXContent(p)); + if (fieldName.equals("error")) { + result = new SimulateDocumentBaseResult(ElasticsearchException.fromXContent(parser)); } else { result = new SimulateDocumentBaseResult( - WriteableIngestDocument.INGEST_DOC_PARSER.apply(p, null).getIngestDocument() + WriteableIngestDocument.INGEST_DOC_PARSER.apply(parser, null).getIngestDocument() ); } - ensureExpectedToken(Token.END_OBJECT, p.currentToken(), p::getTokenLocation); + ensureExpectedToken(Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); break; default: - p.nextToken(); - p.skipChildren(); - break; + parser.skipChildren(); } - isFirst = false; } - ensureExpectedToken(Token.END_OBJECT, p.currentToken(), p::getTokenLocation); assert result != null; return result; }, diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java index af7b3245c94b6..c31c7716eaae2 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java @@ -22,17 +22,19 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.util.StringJoiner; +import java.util.function.Predicate; +import java.util.function.Supplier; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.assertEqualIngestDocs; +import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.createRandomIngestDoc; public class SimulateDocumentBaseResultTests extends AbstractXContentTestCase { @@ -55,20 +57,24 @@ public void testSerialization() throws IOException { } } - protected SimulateDocumentBaseResult createTestInstance(boolean isFailure) { + static SimulateDocumentBaseResult createTestInstance(boolean isFailure) { SimulateDocumentBaseResult simulateDocumentBaseResult; if (isFailure) { simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); } else { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + IngestDocument ingestDocument = createRandomIngestDoc(); simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); } return simulateDocumentBaseResult; } + private static SimulateDocumentBaseResult createTestInstanceWithFailures() { + return createTestInstance(randomBoolean()); + } + @Override protected SimulateDocumentBaseResult createTestInstance() { - return createTestInstance(randomBoolean()); + return createTestInstance(false); } @Override @@ -78,11 +84,27 @@ protected SimulateDocumentBaseResult doParseInstance(XContentParser parser) { @Override protected boolean supportsUnknownFields() { - return false; + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + // We cannot have random fields in the _source field and _ingest field + return field -> + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.SOURCE_FIELD).toString() + ) || + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.INGEST_FIELD).toString() + ); } public static void assertEqualDocs(SimulateDocumentBaseResult response, SimulateDocumentBaseResult parsedResponse) { - assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); + assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); if (response.getFailure() != null) { assertNotNull(parsedResponse.getFailure()); assertThat( @@ -101,6 +123,21 @@ public void assertEqualInstances(SimulateDocumentBaseResult response, SimulateDo @Override protected boolean assertToXContentEquivalence() { - return false; + return true; + } + + /** + * Test parsing {@link SimulateDocumentBaseResult} with inner failures as they don't support asserting on xcontent + * equivalence, given that exceptions are not parsed back as the same original class. We run the usual + * {@link AbstractXContentTestCase#testFromXContent()} without failures, and this other test with failures where + * we disable asserting on xcontent equivalence at the end. + */ + public void testFromXContentWithFailures() throws IOException { + Supplier instanceSupplier = SimulateDocumentBaseResultTests::createTestInstanceWithFailures; + //exceptions are not of the same type whenever parsed back + boolean assertToXContentEquivalence = false; + AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields(), + getShuffleFieldsExceptions(), getRandomFieldsExcludeFilter(), this::createParser, this::doParseInstance, + this::assertEqualInstances, assertToXContentEquivalence, getToXContentParams()); } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java index 363ce36ce115a..3cdd0f9f33678 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java @@ -21,23 +21,38 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.StringJoiner; +import java.util.function.Predicate; +import java.util.function.Supplier; public class SimulateDocumentVerboseResultTests extends AbstractXContentTestCase { - @Override - protected SimulateDocumentVerboseResult createTestInstance() { + static SimulateDocumentVerboseResult createTestInstance(boolean withFailures) { int numDocs = randomIntBetween(0, 10); List results = new ArrayList<>(); for (int i = 0; i getRandomFieldsExcludeFilter() { + // We cannot have random fields in the _source field and _ingest field + return field -> + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.SOURCE_FIELD).toString() + ) || + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.INGEST_FIELD).toString() + ); + } + + /** + * Test parsing {@link SimulateDocumentVerboseResult} with inner failures as they don't support asserting on xcontent + * equivalence, given that exceptions are not parsed back as the same original class. We run the usual + * {@link AbstractXContentTestCase#testFromXContent()} without failures, and this other test with failures where we + * disable asserting on xcontent equivalence at the end. + */ + public void testFromXContentWithFailures() throws IOException { + Supplier instanceSupplier = SimulateDocumentVerboseResultTests::createTestInstanceWithFailures; + //exceptions are not of the same type whenever parsed back + boolean assertToXContentEquivalence = false; + AbstractXContentTestCase.testFromXContent(NUMBER_OF_TEST_RUNS, instanceSupplier, supportsUnknownFields(), + getShuffleFieldsExceptions(), getRandomFieldsExcludeFilter(), this::createParser, this::doParseInstance, + this::assertEqualInstances, assertToXContentEquivalence, getToXContentParams()); } } diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java index e333766f401a2..60bad4aad460f 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java @@ -22,14 +22,13 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.ingest.IngestDocument; -import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.StringJoiner; import java.util.function.Predicate; import java.util.function.Supplier; @@ -94,35 +93,18 @@ public void testSerialization() throws IOException { } } - public static SimulatePipelineResponse createInstance(String pipelineId, boolean isVerbose, boolean withFailure) { + static SimulatePipelineResponse createInstance(String pipelineId, boolean isVerbose, boolean withFailure) { int numResults = randomIntBetween(1, 10); List results = new ArrayList<>(numResults); for (int i = 0; i < numResults; i++) { - boolean isFailure = withFailure && randomBoolean(); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); if (isVerbose) { - int numProcessors = randomIntBetween(1, 10); - List processorResults = new ArrayList<>(numProcessors); - for (int j = 0; j < numProcessors; j++) { - String processorTag = randomAlphaOfLengthBetween(1, 10); - SimulateProcessorResult processorResult; - if (isFailure) { - processorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test")); - } else { - processorResult = new SimulateProcessorResult(processorTag, ingestDocument); - } - processorResults.add(processorResult); - } - results.add(new SimulateDocumentVerboseResult(processorResults)); + results.add( + SimulateDocumentVerboseResultTests.createTestInstance(withFailure) + ); } else { - results.add(new SimulateDocumentBaseResult(ingestDocument)); - SimulateDocumentBaseResult simulateDocumentBaseResult; - if (isFailure) { - simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test")); - } else { - simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument); - } - results.add(simulateDocumentBaseResult); + results.add( + SimulateDocumentBaseResultTests.createTestInstance(withFailure && randomBoolean()) + ); } } return new SimulatePipelineResponse(pipelineId, isVerbose, results); @@ -159,14 +141,14 @@ protected void assertEqualInstances(SimulatePipelineResponse response, assertEquals(response.getResults().size(), parsedResponse.getResults().size()); for (int i=0; i < response.getResults().size(); i++) { if (response.isVerbose()) { - assert response.getResults().get(i) instanceof SimulateDocumentVerboseResult; - assert parsedResponse.getResults().get(i) instanceof SimulateDocumentVerboseResult; + assertThat(response.getResults().get(i), instanceOf(SimulateDocumentVerboseResult.class)); + assertThat(parsedResponse.getResults().get(i), instanceOf(SimulateDocumentVerboseResult.class)); SimulateDocumentVerboseResult responseResult = (SimulateDocumentVerboseResult)response.getResults().get(i); SimulateDocumentVerboseResult parsedResult = (SimulateDocumentVerboseResult)parsedResponse.getResults().get(i); SimulateDocumentVerboseResultTests.assertEqualDocs(responseResult, parsedResult); } else { - assert response.getResults().get(i) instanceof SimulateDocumentBaseResult; - assert parsedResponse.getResults().get(i) instanceof SimulateDocumentBaseResult; + assertThat(response.getResults().get(i), instanceOf(SimulateDocumentBaseResult.class)); + assertThat(parsedResponse.getResults().get(i), instanceOf(SimulateDocumentBaseResult.class)); SimulateDocumentBaseResult responseResult = (SimulateDocumentBaseResult)response.getResults().get(i); SimulateDocumentBaseResult parsedResult = (SimulateDocumentBaseResult)parsedResponse.getResults().get(i); SimulateDocumentBaseResultTests.assertEqualDocs(responseResult, parsedResult); @@ -176,8 +158,18 @@ protected void assertEqualInstances(SimulatePipelineResponse response, @Override protected Predicate getRandomFieldsExcludeFilter() { - // We cannot have random fields in the _source field - return field -> field.contains("doc._source") || field.contains("doc._ingest"); + // We cannot have random fields in the _source field and _ingest field + return field -> + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.SOURCE_FIELD).toString() + ) || + field.contains( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.INGEST_FIELD).toString() + ); } /** diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java index 81db521993b7f..2e0d6a75749bb 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java @@ -22,21 +22,21 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.AbstractXContentTestCase; import java.io.IOException; +import java.util.StringJoiner; import java.util.function.Predicate; import java.util.function.Supplier; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; +import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.createRandomIngestDoc; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import static org.elasticsearch.action.ingest.WriteableIngestDocumentTests.assertEqualIngestDocs; public class SimulateProcessorResultTests extends AbstractXContentTestCase { @@ -67,12 +67,12 @@ public void testSerialization() throws IOException { } } - protected static SimulateProcessorResult createTestInstance(boolean isSuccessful, + static SimulateProcessorResult createTestInstance(boolean isSuccessful, boolean isIgnoredException) { String processorTag = randomAlphaOfLengthBetween(1, 10); SimulateProcessorResult simulateProcessorResult; if (isSuccessful) { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); + IngestDocument ingestDocument = createRandomIngestDoc(); if (isIgnoredException) { simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test")); } else { @@ -109,13 +109,23 @@ protected boolean supportsUnknownFields() { @Override protected Predicate getRandomFieldsExcludeFilter() { // We cannot have random fields in the _source field and _ingest field - return field -> field.contains("doc._source") || field.contains("doc._ingest"); + return field -> + field.startsWith( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.SOURCE_FIELD).toString() + ) || + field.startsWith( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.INGEST_FIELD).toString() + ); } static void assertEqualProcessorResults(SimulateProcessorResult response, SimulateProcessorResult parsedResponse) { assertEquals(response.getProcessorTag(), parsedResponse.getProcessorTag()); - assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); + assertEquals(response.getIngestDocument(), parsedResponse.getIngestDocument()); if (response.getFailure() != null ) { assertNotNull(parsedResponse.getFailure()); assertThat( diff --git a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index 1f12ba82e53a8..bc4589ff5d36c 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -26,15 +26,18 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.test.AbstractXContentTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.test.RandomObjects; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.StringJoiner; +import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; @@ -150,31 +153,21 @@ public void testToXContent() throws IOException { assertThat(serializedIngestDocument, equalTo(serializedIngestDocument)); } - protected static void assertEqualIngestDocs(IngestDocument response, IngestDocument parsedResponse) { - if (response != null && parsedResponse != null) { - ElasticsearchAssertions.assertMapEquals( - response.getSourceAndMetadata(), - parsedResponse.getSourceAndMetadata(), - true - ); - assertEquals(response.getIngestMetadata(), parsedResponse.getIngestMetadata()); - } else if (response == null) { - assertNull(parsedResponse); - } else { - fail("parsed response was null but the expected was non null."); - } - + static IngestDocument createRandomIngestDoc() { + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference sourceBytes = RandomObjects.randomSource(random(), xContentType); + Map randomSource = XContentHelper.convertToMap(sourceBytes, false, xContentType).v2(); + return RandomDocumentPicks.randomIngestDocument(random(), randomSource); } @Override - protected void assertEqualInstances(WriteableIngestDocument response, WriteableIngestDocument parsedResponse) { - assertEqualIngestDocs(response.getIngestDocument(), parsedResponse.getIngestDocument()); + protected boolean supportsUnknownFields() { + return true; } @Override protected WriteableIngestDocument createTestInstance() { - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); - return new WriteableIngestDocument(new IngestDocument(ingestDocument)); + return new WriteableIngestDocument(createRandomIngestDoc()); } @Override @@ -183,8 +176,18 @@ protected WriteableIngestDocument doParseInstance(XContentParser parser) { } @Override - protected boolean supportsUnknownFields() { - // Cannot support unknown fields because equality changes if new keys are added to _source - return false; + protected Predicate getRandomFieldsExcludeFilter() { + // We cannot have random fields in the _source field and _ingest field + return field -> + field.startsWith( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.SOURCE_FIELD).toString() + ) || + field.startsWith( + new StringJoiner(".") + .add(WriteableIngestDocument.DOC_FIELD) + .add(WriteableIngestDocument.INGEST_FIELD).toString() + ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index a969461554648..cf3cc39d34d88 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -64,7 +64,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -670,10 +669,6 @@ public static void assertToXContentEquivalent(BytesReference expected, BytesRefe * Compares two maps recursively, using arrays comparisons for byte[] through Arrays.equals(byte[], byte[]) */ private static void assertMapEquals(Map expected, Map actual) { - assertMapEquals(expected, actual, false); - } - - public static void assertMapEquals(Map expected, Map actual, boolean convertBase64Strings) { assertEquals(expected.size(), actual.size()); for (Map.Entry expectedEntry : expected.entrySet()) { String expectedKey = expectedEntry.getKey(); @@ -682,7 +677,7 @@ public static void assertMapEquals(Map expected, Map expected, Map expected, List actual, boolean convertBase64Strings) { + private static void assertListEquals(List expected, List actual) { assertEquals(expected.size(), actual.size()); Iterator actualIterator = actual.iterator(); for (Object expectedValue : expected) { Object actualValue = actualIterator.next(); - assertObjectEquals(expectedValue, actualValue, convertBase64Strings); + assertObjectEquals(expectedValue, actualValue); } } @@ -705,21 +700,15 @@ private static void assertListEquals(List expected, List actual, * for byte[] through Arrays.equals(byte[], byte[]) */ @SuppressWarnings("unchecked") - private static void assertObjectEquals(Object expected, Object actual, boolean convertBase64Strings) { + private static void assertObjectEquals(Object expected, Object actual) { if (expected instanceof Map) { assertThat(actual, instanceOf(Map.class)); - assertMapEquals((Map) expected, (Map) actual, convertBase64Strings); + assertMapEquals((Map) expected, (Map) actual); } else if (expected instanceof List) { - assertListEquals((List) expected, (List) actual, convertBase64Strings); + assertListEquals((List) expected, (List) actual); } else if (expected instanceof byte[]) { //byte[] is really a special case for binary values when comparing SMILE and CBOR, arrays of other types //don't need to be handled. Ordinary arrays get parsed as lists. - if (convertBase64Strings && actual instanceof String) { - // for objects deserialized from XContent the content might be a Base64 encoded string - actual = Base64.getDecoder().decode((String)actual); - } else { - assertTrue(actual instanceof byte[]); - } assertArrayEquals((byte[]) expected, (byte[]) actual); } else { assertEquals(expected, actual); From 755de507ed09a5e31eda981acf98b106b0d5235e Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Tue, 19 Jun 2018 16:07:01 +0200 Subject: [PATCH 5/6] Fixes after review (3) -- Added separate tests for failure and verbose responses instead of randomization -- Changed parsing for SimulatePipelineResponse --- .../elasticsearch/client/IngestClientIT.java | 105 +++++++++++------- .../ingest/SimulatePipelineResponse.java | 56 +++++----- .../SimulateDocumentBaseResultTests.java | 5 - .../SimulateDocumentVerboseResultTests.java | 5 - 4 files changed, 94 insertions(+), 77 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java index 39beef9e74861..6fd6f95059577 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IngestClientIT.java @@ -35,6 +35,10 @@ import org.elasticsearch.ingest.PipelineConfiguration; import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.IsInstanceOf.instanceOf; public class IngestClientIT extends ESRestHighLevelClientTestCase { @@ -88,9 +92,26 @@ public void testDeletePipeline() throws IOException { } public void testSimulatePipeline() throws IOException { + testSimulatePipeline(false, false); + } + + public void testSimulatePipelineWithFailure() throws IOException { + testSimulatePipeline(false, true); + } + + public void testSimulatePipelineVerbose() throws IOException { + testSimulatePipeline(true, false); + } + + public void testSimulatePipelineVerboseWithFailure() throws IOException { + testSimulatePipeline(true, true); + } + + private void testSimulatePipeline(boolean isVerbose, + boolean isFailure) throws IOException { XContentType xContentType = randomFrom(XContentType.values()); XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); - boolean isVerbose = randomBoolean(); + String rankValue = isFailure ? "non-int" : Integer.toString(1234); builder.startObject(); { builder.field("pipeline"); @@ -101,13 +122,7 @@ public void testSimulatePipeline() throws IOException { .field("_index", "index") .field("_type", "doc") .field("_id", "doc_" + 1) - .startObject("_source").field("foo", "rab_" + 1).field("rank", "1234").endObject() - .endObject(); - builder.startObject() - .field("_index", "index") - .field("_type", "doc") - .field("_id", "doc_" + 2) - .startObject("_source").field("foo", "rab_" + 1).field("rank", "non-int").endObject() + .startObject("_source").field("foo", "rab_" + 1).field("rank", rankValue).endObject() .endObject(); } builder.endArray(); @@ -119,43 +134,49 @@ public void testSimulatePipeline() throws IOException { builder.contentType() ); request.setVerbose(isVerbose); - - SimulatePipelineResponse simulatePipelineResponse = + SimulatePipelineResponse response = execute(request, highLevelClient().ingest()::simulatePipeline, highLevelClient().ingest()::simulatePipelineAsync); - - SimulateDocumentResult result0 = simulatePipelineResponse.getResults().get(0); - SimulateDocumentResult result1 = simulatePipelineResponse.getResults().get(1); + List results = response.getResults(); + assertEquals(1, results.size()); if (isVerbose) { - assertTrue(result0 instanceof SimulateDocumentVerboseResult); - SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult)result0; - SimulateDocumentVerboseResult failedVerboseResult = (SimulateDocumentVerboseResult)result1; - assertTrue(verboseResult.getProcessorResults().size() > 0); - assertEquals( - verboseResult.getProcessorResults().get(0).getIngestDocument() - .getFieldValue("foo", String.class), - "bar" - ); - assertEquals( - Integer.valueOf(1234), - verboseResult.getProcessorResults().get(1).getIngestDocument() - .getFieldValue("rank", Integer.class) - ); - assertNotNull(failedVerboseResult.getProcessorResults().get(1).getFailure()); + assertThat(results.get(0), instanceOf(SimulateDocumentVerboseResult.class)); + SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult) results.get(0); + assertEquals(2, verboseResult.getProcessorResults().size()); + if (isFailure) { + assertNotNull(verboseResult.getProcessorResults().get(1).getFailure()); + assertThat(verboseResult.getProcessorResults().get(1).getFailure().getMessage(), + containsString("unable to convert [non-int] to integer")); + } else { + assertEquals( + verboseResult.getProcessorResults().get(0).getIngestDocument() + .getFieldValue("foo", String.class), + "bar" + ); + assertEquals( + Integer.valueOf(1234), + verboseResult.getProcessorResults().get(1).getIngestDocument() + .getFieldValue("rank", Integer.class) + ); + } } else { - assertTrue(result0 instanceof SimulateDocumentBaseResult); - SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)result0; - SimulateDocumentBaseResult failedBaseResult = (SimulateDocumentBaseResult)result1; - assertNotNull(baseResult.getIngestDocument()); - assertEquals( - baseResult.getIngestDocument().getFieldValue("foo", String.class), - "bar" - ); - assertEquals( - Integer.valueOf(1234), - baseResult.getIngestDocument() - .getFieldValue("rank", Integer.class) - ); - assertNotNull(failedBaseResult.getFailure()); + assertThat(results.get(0), instanceOf(SimulateDocumentBaseResult.class)); + SimulateDocumentBaseResult baseResult = (SimulateDocumentBaseResult)results.get(0); + if (isFailure) { + assertNotNull(baseResult.getFailure()); + assertThat(baseResult.getFailure().getMessage(), + containsString("unable to convert [non-int] to integer")); + } else { + assertNotNull(baseResult.getIngestDocument()); + assertEquals( + baseResult.getIngestDocument().getFieldValue("foo", String.class), + "bar" + ); + assertEquals( + Integer.valueOf(1234), + baseResult.getIngestDocument() + .getFieldValue("rank", Integer.class) + ); + } } } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index 93a6d02a13f55..02fd6c8f0907b 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -63,36 +63,42 @@ public class SimulatePipelineResponse extends ActionResponse implements ToXConte PARSER.declareObjectArray( constructorArg(), (parser, context) -> { - ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + Token token = parser.currentToken(); + ensureExpectedToken(Token.START_OBJECT, token, parser::getTokenLocation); SimulateDocumentResult result = null; - while (parser.nextToken() != Token.END_OBJECT) { - ensureExpectedToken(parser.currentToken(), Token.FIELD_NAME, parser::getTokenLocation); + while ((token = parser.nextToken()) != Token.END_OBJECT) { + ensureExpectedToken(token, Token.FIELD_NAME, parser::getTokenLocation); String fieldName = parser.currentName(); - parser.nextToken(); - switch(fieldName) { - case SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD: - ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); - List results = new ArrayList<>(); - while (parser.nextToken().equals(Token.START_OBJECT)) { - results.add(SimulateProcessorResult.fromXContent(parser)); - } - ensureExpectedToken(Token.END_ARRAY, parser.currentToken(), parser::getTokenLocation); - result = new SimulateDocumentVerboseResult(results); - break; - case WriteableIngestDocument.DOC_FIELD: - case "error": - if (fieldName.equals("error")) { - result = new SimulateDocumentBaseResult(ElasticsearchException.fromXContent(parser)); - } else { + token = parser.nextToken(); + if (token == Token.START_ARRAY) { + switch (fieldName) { + case SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD: + List results = new ArrayList<>(); + while ((token = parser.nextToken()).equals(Token.START_OBJECT)) { + results.add(SimulateProcessorResult.fromXContent(parser)); + } + ensureExpectedToken(Token.END_ARRAY, token, parser::getTokenLocation); + result = new SimulateDocumentVerboseResult(results); + break; + default: + parser.skipChildren(); + break; + } + } else if (token.equals(Token.START_OBJECT)) { + switch (fieldName) { + case WriteableIngestDocument.DOC_FIELD: result = new SimulateDocumentBaseResult( WriteableIngestDocument.INGEST_DOC_PARSER.apply(parser, null).getIngestDocument() ); - } - ensureExpectedToken(Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); - break; - default: - parser.skipChildren(); - } + break; + case "error": + result = new SimulateDocumentBaseResult(ElasticsearchException.fromXContent(parser)); + break; + default: + parser.skipChildren(); + break; + } + } // else it is a value skip it } assert result != null; return result; diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java index c31c7716eaae2..bfa6c1eb9b8c3 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResultTests.java @@ -121,11 +121,6 @@ public void assertEqualInstances(SimulateDocumentBaseResult response, SimulateDo assertEqualDocs(response, parsedResponse); } - @Override - protected boolean assertToXContentEquivalence() { - return true; - } - /** * Test parsing {@link SimulateDocumentBaseResult} with inner failures as they don't support asserting on xcontent * equivalence, given that exceptions are not parsed back as the same original class. We run the usual diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java index 3cdd0f9f33678..5701bcc27800f 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentVerboseResultTests.java @@ -80,11 +80,6 @@ protected void assertEqualInstances(SimulateDocumentVerboseResult response, assertEqualDocs(response, parsedResponse); } - @Override - protected boolean assertToXContentEquivalence() { - return true; - } - @Override protected Predicate getRandomFieldsExcludeFilter() { // We cannot have random fields in the _source field and _ingest field From 54e7f6f219d1dc205c41f87a66ae8603231cd053 Mon Sep 17 00:00:00 2001 From: Sohaib Iftikhar Date: Thu, 21 Jun 2018 14:18:10 +0200 Subject: [PATCH 6/6] changed switch to if --- .../ingest/SimulatePipelineResponse.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java index 02fd6c8f0907b..991e81a14553b 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineResponse.java @@ -71,18 +71,15 @@ public class SimulatePipelineResponse extends ActionResponse implements ToXConte String fieldName = parser.currentName(); token = parser.nextToken(); if (token == Token.START_ARRAY) { - switch (fieldName) { - case SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD: - List results = new ArrayList<>(); - while ((token = parser.nextToken()).equals(Token.START_OBJECT)) { - results.add(SimulateProcessorResult.fromXContent(parser)); - } - ensureExpectedToken(Token.END_ARRAY, token, parser::getTokenLocation); - result = new SimulateDocumentVerboseResult(results); - break; - default: - parser.skipChildren(); - break; + if (fieldName.equals(SimulateDocumentVerboseResult.PROCESSOR_RESULT_FIELD)) { + List results = new ArrayList<>(); + while ((token = parser.nextToken()) == Token.START_OBJECT) { + results.add(SimulateProcessorResult.fromXContent(parser)); + } + ensureExpectedToken(Token.END_ARRAY, token, parser::getTokenLocation); + result = new SimulateDocumentVerboseResult(results); + } else { + parser.skipChildren(); } } else if (token.equals(Token.START_OBJECT)) { switch (fieldName) {