From ed154bd208681da208c2ffe5e55c62245adc5929 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 11 Mar 2020 16:41:17 -0400 Subject: [PATCH 1/3] [ML] renaming inference processor field field_mappings to new name field_map --- .../client/MachineLearningIT.java | 2 +- .../ingest/processors/inference.asciidoc | 4 +- .../ml/qa/ml-with-security/build.gradle | 1 + .../plugin/ml/qa/ml-with-security/roles.yml | 2 + .../ml/integration/InferenceIngestIT.java | 14 +-- .../inference/ingest/InferenceProcessor.java | 22 +++- .../MachineLearningLicensingTests.java | 2 +- ...sportGetTrainedModelsStatsActionTests.java | 2 +- .../InferenceProcessorFactoryTests.java | 18 +-- .../rest-api-spec/test/ml/inference_crud.yml | 2 +- .../test/ml/inference_processor.yml | 112 ++++++++++++++++++ .../test/ml/inference_stats_crud.yml | 4 +- 12 files changed, 155 insertions(+), 30 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 2f923abf04983..e13464aeddabd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -2265,7 +2265,7 @@ public void testGetTrainedModelsStats() throws Exception { " \"target_field\": \"regression_value\",\n" + " \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" + " \"inference_config\": {\"regression\": {}},\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index 009fbd619aace..b1b110e2b3889 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -14,7 +14,7 @@ ingested in the pipeline. | Name | Required | Default | Description | `model_id` | yes | - | (String) The ID of the model to load and infer against. | `target_field` | no | `ml.inference.` | (String) Field added to incoming documents to contain results objects. -| `field_mappings` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration. +| `field_map` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration. | `inference_config` | yes | - | (Object) Contains the inference type and its options. There are two types: <> and <>. include::common-options.asciidoc[] |====== @@ -26,7 +26,7 @@ include::common-options.asciidoc[] "inference": { "model_id": "flight_delay_regression-1571767128603", "target_field": "FlightDelayMin_prediction_infer", - "field_mappings": {}, + "field_map": {}, "inference_config": { "regression": {} } } } diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index ed90561faf1e9..c19d82a5d3b40 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -139,6 +139,7 @@ integTest.runner { 'ml/inference_crud/Test put ensemble with empty models', 'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index', 'ml/inference_crud/Test put model with empty input.field_names', + 'ml/inference_processor/Test create processor with missing mandatory fields', 'ml/inference_stats_crud/Test get stats given missing trained model', 'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false', 'ml/jobs_crud/Test cannot create job with existing categorizer state document', diff --git a/x-pack/plugin/ml/qa/ml-with-security/roles.yml b/x-pack/plugin/ml/qa/ml-with-security/roles.yml index 48c4abb9f4262..8ca2d5b062e08 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/roles.yml +++ b/x-pack/plugin/ml/qa/ml-with-security/roles.yml @@ -3,6 +3,8 @@ minimal: # This is always required because the REST client uses it to find the version of # Elasticsearch it's talking to - cluster:monitor/main + # certain tests need to create pipelines + - cluster:admin/ingest/pipeline/* indices: # Give all users involved in these tests access to the indices where the data to # be analyzed is stored, because the ML roles alone do not provide access to diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java index 8bae148fd55c1..d6c88fbfdb994 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java @@ -120,7 +120,7 @@ public void testSimulate() throws IOException { " \"num_top_feature_importance_values\": 2" + " }},\n" + " \"model_id\": \"test_classification\",\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + @@ -133,7 +133,7 @@ public void testSimulate() throws IOException { " \"target_field\": \"ml.regression\",\n" + " \"model_id\": \"test_regression\",\n" + " \"inference_config\": {\"regression\":{}},\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + @@ -166,7 +166,7 @@ public void testSimulate() throws IOException { " \"inference\": {\n" + " \"model_id\": \"test_classification_missing\",\n" + " \"inference_config\": {\"classification\":{}},\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + @@ -204,7 +204,7 @@ public void testSimulateWithDefaultMappedField() throws IOException { " \"num_top_feature_importance_values\": 2" + " }},\n" + " \"model_id\": \"test_classification\",\n" + - " \"field_mappings\": {}\n" + + " \"field_map\": {}\n" + " }\n" + " }\n"+ " ]\n" + @@ -233,7 +233,7 @@ public void testSimulateLangIdent() throws IOException { " \"inference\": {\n" + " \"inference_config\": {\"classification\":{}},\n" + " \"model_id\": \"lang_ident_model_1\",\n" + - " \"field_mappings\": {}\n" + + " \"field_map\": {}\n" + " }\n" + " }\n" + " ]\n" + @@ -570,7 +570,7 @@ protected NamedXContentRegistry xContentRegistry() { " \"model_id\": \"test_classification\",\n" + " \"tag\": \"classification\",\n" + " \"inference_config\": {\"classification\": {}},\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + @@ -586,7 +586,7 @@ protected NamedXContentRegistry xContentRegistry() { " \"model_id\": \"test_regression\",\n" + " \"tag\": \"regression\",\n" + " \"inference_config\": {\"regression\": {}},\n" + - " \"field_mappings\": {\n" + + " \"field_map\": {\n" + " \"col1\": \"col1\",\n" + " \"col2\": \"col2\",\n" + " \"col3\": \"col3\",\n" + diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index d2a2c7896f86b..072f285d09591 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -16,8 +16,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -63,6 +65,7 @@ public class InferenceProcessor extends AbstractProcessor { public static final String INFERENCE_CONFIG = "inference_config"; public static final String TARGET_FIELD = "target_field"; public static final String FIELD_MAPPINGS = "field_mappings"; + public static final String FIELD_MAP = "field_map"; private static final String DEFAULT_TARGET_FIELD = "ml.inference"; private final Client client; @@ -70,7 +73,7 @@ public class InferenceProcessor extends AbstractProcessor { private final String targetField; private final InferenceConfig inferenceConfig; - private final Map fieldMapping; + private final Map fieldMap; private final InferenceAuditor auditor; private volatile boolean previouslyLicensed; private final AtomicBoolean shouldAudit = new AtomicBoolean(true); @@ -81,14 +84,14 @@ public InferenceProcessor(Client client, String targetField, String modelId, InferenceConfig inferenceConfig, - Map fieldMapping) { + Map fieldMap) { super(tag); this.client = ExceptionsHelper.requireNonNull(client, "client"); this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD); this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor"); this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID); this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG); - this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS); + this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP); } public String getModelId() { @@ -126,7 +129,7 @@ void handleResponse(InternalInferModelAction.Response response, InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) { Map fields = new HashMap<>(ingestDocument.getSourceAndMetadata()); - Model.mapFieldsIfNecessary(fields, fieldMapping); + Model.mapFieldsIfNecessary(fields, fieldMap); return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed); } @@ -235,7 +238,14 @@ public InferenceProcessor create(Map processorFactori // If multiple inference processors are in the same pipeline, it is wise to tag them // The tag will keep default value entries from stepping on each other String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField); - Map fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS); + Map fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP); + if (fieldMap == null) { + fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS); + //TODO Remove in 8.x + if (fieldMap != null) { + LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP); + } + } InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG)); return new InferenceProcessor(client, @@ -244,7 +254,7 @@ public InferenceProcessor create(Map processorFactori targetField, modelId, inferenceConfig, - fieldMapping); + fieldMap); } // Package private for testing diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java index 0925d252e2019..dfbc4be8e8265 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java @@ -484,7 +484,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() throws Excep " \"target_field\": \"regression_value\",\n" + " \"model_id\": \"modelprocessorlicensetest\",\n" + " \"inference_config\": {\"regression\": {}},\n" + - " \"field_mappings\": {}\n" + + " \"field_map\": {}\n" + " }\n" + " }]}\n"; // Creating a pipeline should work diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java index 03a011c717292..d776ffd9818d9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java @@ -267,7 +267,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri new HashMap() {{ put(InferenceProcessor.MODEL_ID, modelId); put("inference_config", Collections.singletonMap("regression", Collections.emptyMap())); - put("field_mappings", Collections.emptyMap()); + put("field_map", Collections.emptyMap()); put("target_field", randomAlphaOfLength(10)); }}))))) { return new PipelineConfiguration("pipeline_with_model_" + modelId + num, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java index 9ed05e56c2c30..cb0519d845b19 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java @@ -131,7 +131,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { ingestService); Map config = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap())); @@ -143,7 +143,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]")); Map config2 = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom")); @@ -154,7 +154,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() { equalTo("inference_config must be an object with one inference type mapped to an object.")); Map config3 = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap()); @@ -173,7 +173,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException { processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1")); Map regression = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, @@ -191,7 +191,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException { } Map classification = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(), @@ -216,7 +216,7 @@ public void testCreateProcessor() { ingestService); Map regression = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, @@ -230,7 +230,7 @@ public void testCreateProcessor() { } Map classification = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "result"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(), @@ -251,7 +251,7 @@ public void testCreateProcessorWithDuplicateFields() { ingestService); Map regression = new HashMap<>() {{ - put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap()); + put(InferenceProcessor.FIELD_MAP, Collections.emptyMap()); put(InferenceProcessor.MODEL_ID, "my_model"); put(InferenceProcessor.TARGET_FIELD, "ml"); put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), @@ -305,7 +305,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap())); put(InferenceProcessor.TARGET_FIELD, "new_field"); - put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest")); + put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest")); }}))))) { return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml index cb5ccd90e1311..8c1f1e581de5b 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml @@ -313,7 +313,7 @@ setup: "model_id" : "a-regression-model-0", "inference_config": {"regression": {}}, "target_field": "regression_field", - "field_mappings": {} + "field_map": {} } } ] diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml new file mode 100644 index 0000000000000..b09d64d8bdc21 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml @@ -0,0 +1,112 @@ +setup: + - skip: + features: headers + - do: + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + ml.put_trained_model: + model_id: a-perfect-regression-model + body: > + { + "description": "empty model for tests", + "tags": ["regression", "tag1"], + "input": {"field_names": ["field1", "field2"]}, + "definition": { + "preprocessors": [], + "trained_model": { + "tree": { + "feature_names": ["field1", "field2"], + "tree_structure": [ + {"node_index": 0, "leaf_value": 42} + ], + "target_type": "regression" + } + } + } + } + +--- +"Test create and delete pipeline with inference processor": + - do: + ingest.put_pipeline: + id: "regression-model-pipeline" + body: > + { + "processors": [ + { + "inference" : { + "model_id" : "a-perfect-regression-model", + "inference_config": {"regression": {}}, + "target_field": "regression_field", + "field_map": {} + } + } + ] + } + - match: { acknowledged: true } + - do: + ingest.delete_pipeline: + id: "regression-model-pipeline" +--- +"Test create processor with missing mandatory fields": + - do: + catch: /\[model_id\] required property is missing/ + ingest.put_pipeline: + id: "regression-model-pipeline" + body: > + { + "processors": [ + { + "inference" : { + "inference_config": {"regression": {}}, + "target_field": "regression_field", + "field_map": {} + } + } + ] + } + - do: + catch: /\[inference_config\] required property is missing/ + ingest.put_pipeline: + id: "regression-model-pipeline" + body: > + { + "processors": [ + { + "inference" : { + "model_id" : "a-perfect-regression-model", + "target_field": "regression_field", + "field_map": {} + } + } + ] + } +--- +"Test create processor with deprecated fields": + - skip: + features: + - "warnings" + - "allowed_warnings" + - do: + warnings: + - 'Deprecated field [field_mappings] used, expected [field_map] instead' + ingest.put_pipeline: + id: "regression-model-pipeline" + body: > + { + "processors": [ + { + "inference" : { + "model_id" : "a-perfect-regression-model", + "inference_config": {"regression": {}}, + "field_mappings": {} + } + } + ] + } + + - do: + allowed_warnings: + - 'Deprecated field [field_mappings] used, expected [field_map] instead' + ingest.delete_pipeline: + id: "regression-model-pipeline" diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml index 5143a690b018a..40d0f6ba01ab7 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_stats_crud.yml @@ -66,7 +66,7 @@ setup: "model_id" : "a-used-regression-model", "inference_config": {"regression": {}}, "target_field": "regression_field", - "field_mappings": {} + "field_map": {} } } ] @@ -84,7 +84,7 @@ setup: "model_id" : "a-used-regression-model", "inference_config": {"regression": {}}, "target_field": "regression_field", - "field_mappings": {} + "field_map": {} } } ] From 791bbe280b05f4ea8f98ba353ee3c630ce6a9af2 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 12 Mar 2020 08:28:12 -0400 Subject: [PATCH 2/3] removing unused import --- .../xpack/ml/inference/ingest/InferenceProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java index 072f285d09591..2a4861c5247f7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; From 897f2760a8e25637d549dbf5304b23e6809d8871 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 12 Mar 2020 09:28:50 -0400 Subject: [PATCH 3/3] fixing ml with security testing --- x-pack/plugin/ml/qa/ml-with-security/build.gradle | 2 ++ x-pack/plugin/ml/qa/ml-with-security/roles.yml | 2 -- .../resources/rest-api-spec/test/ml/inference_processor.yml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ml/qa/ml-with-security/build.gradle b/x-pack/plugin/ml/qa/ml-with-security/build.gradle index c19d82a5d3b40..516e5c626fca2 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/build.gradle +++ b/x-pack/plugin/ml/qa/ml-with-security/build.gradle @@ -140,6 +140,8 @@ integTest.runner { 'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index', 'ml/inference_crud/Test put model with empty input.field_names', 'ml/inference_processor/Test create processor with missing mandatory fields', + 'ml/inference_processor/Test create and delete pipeline with inference processor', + 'ml/inference_processor/Test create processor with deprecated fields', 'ml/inference_stats_crud/Test get stats given missing trained model', 'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false', 'ml/jobs_crud/Test cannot create job with existing categorizer state document', diff --git a/x-pack/plugin/ml/qa/ml-with-security/roles.yml b/x-pack/plugin/ml/qa/ml-with-security/roles.yml index 8ca2d5b062e08..48c4abb9f4262 100644 --- a/x-pack/plugin/ml/qa/ml-with-security/roles.yml +++ b/x-pack/plugin/ml/qa/ml-with-security/roles.yml @@ -3,8 +3,6 @@ minimal: # This is always required because the REST client uses it to find the version of # Elasticsearch it's talking to - cluster:monitor/main - # certain tests need to create pipelines - - cluster:admin/ingest/pipeline/* indices: # Give all users involved in these tests access to the indices where the data to # be analyzed is stored, because the ML roles alone do not provide access to diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml index b09d64d8bdc21..9c043eee3cd2c 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_processor.yml @@ -1,3 +1,5 @@ +# These tests are not part of the ml-with-security tests +# These are more like ingest processor tests than actual ML focused tests. setup: - skip: features: headers