Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] renaming inference processor field field_mappings to new name field_map #53433

Merged
merged 5 commits into from
Mar 12, 2020
Merged

[ML] renaming inference processor field field_mappings to new name field_map #53433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ed154bd
[ML] renaming inference processor field field_mappings to new name fi…
benwtrent Mar 11, 2020
c882395
Merge branch 'master' into feature/ml-inference-rename-field_mappings…
elasticmachine Mar 12, 2020
791bbe2
removing unused import
benwtrent Mar 12, 2020
974b540
Merge branch 'master' into feature/ml-inference-rename-field_mappings…
benwtrent Mar 12, 2020
897f276
fixing ml with security testing
benwtrent Mar 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2265,7 +2265,7 @@ public void testGetTrainedModelsStats() throws Exception {
" \"target_field\": \"regression_value\",\n" +
" \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand Down
4 changes: 2 additions & 2 deletions docs/reference/ingest/processors/inference.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ ingested in the pipeline.
| Name | Required | Default | Description
| `model_id` | yes | - | (String) The ID of the model to load and infer against.
| `target_field` | no | `ml.inference.<processor_tag>` | (String) Field added to incoming documents to contain results objects.
| `field_mappings` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
| `field_map` | yes | - | (Object) Maps the document field names to the known field names of the model. This mapping takes precedence over any default mappings provided in the model configuration.
| `inference_config` | yes | - | (Object) Contains the inference type and its options. There are two types: <<inference-processor-regression-opt,`regression`>> and <<inference-processor-classification-opt,`classification`>>.
include::common-options.asciidoc[]
|======
Expand All @@ -26,7 +26,7 @@ include::common-options.asciidoc[]
"inference": {
"model_id": "flight_delay_regression-1571767128603",
"target_field": "FlightDelayMin_prediction_infer",
"field_mappings": {},
"field_map": {},
"inference_config": { "regression": {} }
}
}
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugin/ml/qa/ml-with-security/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,9 @@ integTest.runner {
'ml/inference_crud/Test put ensemble with empty models',
'ml/inference_crud/Test put ensemble with tree where tree has out of bounds feature_names index',
'ml/inference_crud/Test put model with empty input.field_names',
'ml/inference_processor/Test create processor with missing mandatory fields',
'ml/inference_processor/Test create and delete pipeline with inference processor',
'ml/inference_processor/Test create processor with deprecated fields',
'ml/inference_stats_crud/Test get stats given missing trained model',
'ml/inference_stats_crud/Test get stats given expression without matches and allow_no_match is false',
'ml/jobs_crud/Test cannot create job with existing categorizer state document',
Expand Down
14 changes: 7 additions & 7 deletions ...ti-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InferenceIngestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public void testSimulate() throws IOException {
" \"num_top_feature_importance_values\": 2" +
" }},\n" +
" \"model_id\": \"test_classification\",\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand All @@ -133,7 +133,7 @@ public void testSimulate() throws IOException {
" \"target_field\": \"ml.regression\",\n" +
" \"model_id\": \"test_regression\",\n" +
" \"inference_config\": {\"regression\":{}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand Down Expand Up @@ -166,7 +166,7 @@ public void testSimulate() throws IOException {
" \"inference\": {\n" +
" \"model_id\": \"test_classification_missing\",\n" +
" \"inference_config\": {\"classification\":{}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand Down Expand Up @@ -204,7 +204,7 @@ public void testSimulateWithDefaultMappedField() throws IOException {
" \"num_top_feature_importance_values\": 2" +
" }},\n" +
" \"model_id\": \"test_classification\",\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }\n"+
" ]\n" +
Expand Down Expand Up @@ -233,7 +233,7 @@ public void testSimulateLangIdent() throws IOException {
" \"inference\": {\n" +
" \"inference_config\": {\"classification\":{}},\n" +
" \"model_id\": \"lang_ident_model_1\",\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }\n" +
" ]\n" +
Expand Down Expand Up @@ -570,7 +570,7 @@ protected NamedXContentRegistry xContentRegistry() {
" \"model_id\": \"test_classification\",\n" +
" \"tag\": \"classification\",\n" +
" \"inference_config\": {\"classification\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand All @@ -586,7 +586,7 @@ protected NamedXContentRegistry xContentRegistry() {
" \"model_id\": \"test_regression\",\n" +
" \"tag\": \"regression\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {\n" +
" \"field_map\": {\n" +
" \"col1\": \"col1\",\n" +
" \"col2\": \"col2\",\n" +
" \"col3\": \"col3\",\n" +
Expand Down
21 changes: 15 additions & 6 deletions ...ugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
Expand Down Expand Up @@ -63,14 +64,15 @@ public class InferenceProcessor extends AbstractProcessor {
public static final String INFERENCE_CONFIG = "inference_config";
public static final String TARGET_FIELD = "target_field";
public static final String FIELD_MAPPINGS = "field_mappings";
public static final String FIELD_MAP = "field_map";
private static final String DEFAULT_TARGET_FIELD = "ml.inference";

private final Client client;
private final String modelId;

private final String targetField;
private final InferenceConfig inferenceConfig;
private final Map<String, String> fieldMapping;
private final Map<String, String> fieldMap;
private final InferenceAuditor auditor;
private volatile boolean previouslyLicensed;
private final AtomicBoolean shouldAudit = new AtomicBoolean(true);
Expand All @@ -81,14 +83,14 @@ public InferenceProcessor(Client client,
String targetField,
String modelId,
InferenceConfig inferenceConfig,
Map<String, String> fieldMapping) {
Map<String, String> fieldMap) {
super(tag);
this.client = ExceptionsHelper.requireNonNull(client, "client");
this.targetField = ExceptionsHelper.requireNonNull(targetField, TARGET_FIELD);
this.auditor = ExceptionsHelper.requireNonNull(auditor, "auditor");
this.modelId = ExceptionsHelper.requireNonNull(modelId, MODEL_ID);
this.inferenceConfig = ExceptionsHelper.requireNonNull(inferenceConfig, INFERENCE_CONFIG);
this.fieldMapping = ExceptionsHelper.requireNonNull(fieldMapping, FIELD_MAPPINGS);
this.fieldMap = ExceptionsHelper.requireNonNull(fieldMap, FIELD_MAP);
}

public String getModelId() {
Expand Down Expand Up @@ -126,7 +128,7 @@ void handleResponse(InternalInferModelAction.Response response,

InternalInferModelAction.Request buildRequest(IngestDocument ingestDocument) {
Map<String, Object> fields = new HashMap<>(ingestDocument.getSourceAndMetadata());
Model.mapFieldsIfNecessary(fields, fieldMapping);
Model.mapFieldsIfNecessary(fields, fieldMap);
return new InternalInferModelAction.Request(modelId, fields, inferenceConfig, previouslyLicensed);
}

Expand Down Expand Up @@ -235,7 +237,14 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
// If multiple inference processors are in the same pipeline, it is wise to tag them
// The tag will keep default value entries from stepping on each other
String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, TARGET_FIELD, defaultTargetField);
Map<String, String> fieldMapping = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
Map<String, String> fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAP);
if (fieldMap == null) {
fieldMap = ConfigurationUtils.readOptionalMap(TYPE, tag, config, FIELD_MAPPINGS);
//TODO Remove in 8.x
if (fieldMap != null) {
LoggingDeprecationHandler.INSTANCE.usedDeprecatedName(FIELD_MAPPINGS, FIELD_MAP);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

}
}
InferenceConfig inferenceConfig = inferenceConfigFromMap(ConfigurationUtils.readMap(TYPE, tag, config, INFERENCE_CONFIG));

return new InferenceProcessor(client,
Expand All @@ -244,7 +253,7 @@ public InferenceProcessor create(Map<String, Processor.Factory> processorFactori
targetField,
modelId,
inferenceConfig,
fieldMapping);
fieldMap);
}

// Package private for testing
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/ml/src/test/java/org/elasticsearch/license/MachineLearningLicensingTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,7 +484,7 @@ public void testMachineLearningCreateInferenceProcessorRestricted() throws Excep
" \"target_field\": \"regression_value\",\n" +
" \"model_id\": \"modelprocessorlicensetest\",\n" +
" \"inference_config\": {\"regression\": {}},\n" +
" \"field_mappings\": {}\n" +
" \"field_map\": {}\n" +
" }\n" +
" }]}\n";
// Creating a pipeline should work
Expand Down
2 changes: 1 addition & 1 deletion ...est/java/org/elasticsearch/xpack/ml/action/TransportGetTrainedModelsStatsActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
new HashMap<String, Object>() {{
put(InferenceProcessor.MODEL_ID, modelId);
put("inference_config", Collections.singletonMap("regression", Collections.emptyMap()));
put("field_mappings", Collections.emptyMap());
put("field_map", Collections.emptyMap());
put("target_field", randomAlphaOfLength(10));
}}))))) {
return new PipelineConfiguration("pipeline_with_model_" + modelId + num,
Expand Down
18 changes: 9 additions & 9 deletions ...test/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessorFactoryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
ingestService);

Map<String, Object> config = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("unknown_type", Collections.emptyMap()));
Expand All @@ -143,7 +143,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
equalTo("unrecognized inference configuration type [unknown_type]. Supported types [classification, regression]"));

Map<String, Object> config2 = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap("regression", "boom"));
Expand All @@ -154,7 +154,7 @@ public void testCreateProcessorWithInvalidInferenceConfig() {
equalTo("inference_config must be an object with one inference type mapped to an object."));

Map<String, Object> config3 = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.emptyMap());
Expand All @@ -173,7 +173,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
processorFactory.accept(builderClusterStateWithModelReferences(Version.V_7_5_0, "model1"));

Map<String, Object> regression = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG,
Expand All @@ -191,7 +191,7 @@ public void testCreateProcessorWithTooOldMinNodeVersion() throws IOException {
}

Map<String, Object> classification = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
Expand All @@ -216,7 +216,7 @@ public void testCreateProcessor() {
ingestService);

Map<String, Object> regression = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG,
Expand All @@ -230,7 +230,7 @@ public void testCreateProcessor() {
}

Map<String, Object> classification = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "result");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(ClassificationConfig.NAME.getPreferredName(),
Expand All @@ -251,7 +251,7 @@ public void testCreateProcessorWithDuplicateFields() {
ingestService);

Map<String, Object> regression = new HashMap<>() {{
put(InferenceProcessor.FIELD_MAPPINGS, Collections.emptyMap());
put(InferenceProcessor.FIELD_MAP, Collections.emptyMap());
put(InferenceProcessor.MODEL_ID, "my_model");
put(InferenceProcessor.TARGET_FIELD, "ml");
put(InferenceProcessor.INFERENCE_CONFIG, Collections.singletonMap(RegressionConfig.NAME.getPreferredName(),
Expand Down Expand Up @@ -305,7 +305,7 @@ private static PipelineConfiguration newConfigurationWithInferenceProcessor(Stri
put(InferenceProcessor.INFERENCE_CONFIG,
Collections.singletonMap(RegressionConfig.NAME.getPreferredName(), Collections.emptyMap()));
put(InferenceProcessor.TARGET_FIELD, "new_field");
put(InferenceProcessor.FIELD_MAPPINGS, Collections.singletonMap("source", "dest"));
put(InferenceProcessor.FIELD_MAP, Collections.singletonMap("source", "dest"));
}}))))) {
return new PipelineConfiguration("pipeline_with_model_" + modelId, BytesReference.bytes(xContentBuilder), XContentType.JSON);
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/src/test/resources/rest-api-spec/test/ml/inference_crud.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ setup:
"model_id" : "a-regression-model-0",
"inference_config": {"regression": {}},
"target_field": "regression_field",
"field_mappings": {}
"field_map": {}
}
}
]
Expand Down
Loading