Skip to content

Commit

Permalink
Add migration tool checks for _field_names disabling
Browse files Browse the repository at this point in the history 
This change adds a check to the migration tool that warns about the deprecated
"enabled" setting for the "_field_names" field on 7.x indices and issues a
critical error for templates containing this setting, since it has been removed
with 8.0.

Relates to elastic#42854, elastic#46681
  • Loading branch information
Christoph Büscher committed Sep 23, 2019
1 parent ee4e6b1 commit b3e5098
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 4 deletions.
53 changes: 53 additions & 0 deletions ...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,20 @@
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.ingest.PipelineConfiguration;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -91,6 +96,54 @@ static DeprecationIssue checkTemplatesWithTooManyFields(ClusterState state) {
return null;
}

/**
* Check templates that use `_field_names` explicitly, which was deprecated in https://github.com/elastic/elasticsearch/pull/42854
* and will throw an error on new indices in 8.0
*/
static DeprecationIssue checkTemplatesWithFieldNamesDisabled(ClusterState state) {
Set<String> templatesContainingFieldNamed = new HashSet<>();
state.getMetaData().getTemplates().forEach((templateCursor) -> {
String templateName = templateCursor.key;
templateCursor.value.getMappings().forEach((mappingCursor) -> {
Map<String, Object> map = XContentHelper.convertToMap(mappingCursor.value.compressedReference(), false, XContentType.JSON)
.v2();
if (mapContainsFieldNamesDisabled(map)) {
templatesContainingFieldNamed.add(templateName);
}
});
});

if (templatesContainingFieldNamed.isEmpty() == false) {
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, "Index templates contain _field_names settings.",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling",
"Index templates " + templatesContainingFieldNamed + " use the deprecated `enable` setting for the `"
+ FieldNamesFieldMapper.NAME + "` field. Using this setting in new index mappings will throw an error "
+ "in the next major version and needs to be removed from existing mappings and templates.");
}
return null;
}

/**
* check for "_field_names" entries in the map that contain another property "enabled" in the sub-map
*/
static boolean mapContainsFieldNamesDisabled(Map<?, ?> map) {
if (map != null) {
Object fieldNamesMapping = map.get(FieldNamesFieldMapper.NAME);
if (fieldNamesMapping != null && fieldNamesMapping instanceof Map) {
if (((Map<?, ?>) fieldNamesMapping).keySet().contains("enabled")) {
return true;
}
} else {
for (Object value : map.values()) {
if (value instanceof Map) {
return mapContainsFieldNamesDisabled((Map<?, ?>) value);
}
}
}
}
return false;
}

static DeprecationIssue checkPollIntervalTooLow(ClusterState state) {
String pollIntervalString = state.metaData().settings().get(LIFECYCLE_POLL_INTERVAL_SETTING.getKey());
if (Strings.isNullOrEmpty(pollIntervalString)) {
Expand Down
6 changes: 4 additions & 2 deletions ...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ private DeprecationChecks() {
Collections.unmodifiableList(Arrays.asList(
ClusterDeprecationChecks::checkUserAgentPipelines,
ClusterDeprecationChecks::checkTemplatesWithTooManyFields,
ClusterDeprecationChecks::checkPollIntervalTooLow
ClusterDeprecationChecks::checkPollIntervalTooLow,
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled
));


Expand All @@ -51,7 +52,8 @@ private DeprecationChecks() {
IndexDeprecationChecks::tooManyFieldsCheck,
IndexDeprecationChecks::chainedMultiFieldsCheck,
IndexDeprecationChecks::deprecatedDateTimeFormat,
IndexDeprecationChecks::translogRetentionSettingCheck
IndexDeprecationChecks::translogRetentionSettingCheck,
IndexDeprecationChecks::fieldNamesEnabling
));

static List<BiFunction<DatafeedConfig, NamedXContentRegistry, DeprecationIssue>> ML_SETTINGS_CHECKS =
Expand Down
16 changes: 16 additions & 0 deletions ...deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@


import com.carrotsearch.hppc.cursors.ObjectCursor;

import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
Expand Down Expand Up @@ -185,6 +186,21 @@ private static boolean containsChainedMultiFields(Map<?, ?> property) {
return false;
}

/**
* warn about existing explicit "_field_names" enabled settings in existing mappings
*/
static DeprecationIssue fieldNamesEnabling(IndexMetaData indexMetaData) {
MappingMetaData mapping = indexMetaData.mapping();
if ((mapping != null) && ClusterDeprecationChecks.mapContainsFieldNamesDisabled(mapping.getSourceAsMap())) {
return new DeprecationIssue(DeprecationIssue.Level.WARNING,
"Index mapping contain explicit _field_names enabling settings.",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
"#fieldnames-enabling",
"The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.");
}
return null;
}

private static final Set<String> TYPES_THAT_DONT_COUNT;
static {
HashSet<String> typesThatDontCount = new HashSet<>();
Expand Down
75 changes: 75 additions & 0 deletions ...tion/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
Expand Down Expand Up @@ -158,6 +159,80 @@ public void testTemplateWithTooManyFields() throws IOException {
assertEquals(singletonList(expected), issues);
}

public void testTemplatesWithFieldNamesDisabled() throws IOException {
XContentBuilder goodMappingBuilder = jsonBuilder();
goodMappingBuilder.startObject();
{
goodMappingBuilder.startObject("_doc");
{
goodMappingBuilder.startObject("properties");
{
addRandomFields(10, goodMappingBuilder);
}
goodMappingBuilder.endObject();
}
goodMappingBuilder.endObject();
}
goodMappingBuilder.endObject();
assertFieldNamesEnabledTemplate(goodMappingBuilder, false);

XContentBuilder badMappingWithTypeBuilder = jsonBuilder();
badMappingWithTypeBuilder.startObject();
{
badMappingWithTypeBuilder.startObject("_doc");
{
badMappingWithTypeBuilder.startObject(FieldNamesFieldMapper.NAME);
{
badMappingWithTypeBuilder.field("enabled", randomBoolean());
}
badMappingWithTypeBuilder.endObject();
}
badMappingWithTypeBuilder.endObject();
}
badMappingWithTypeBuilder.endObject();
assertFieldNamesEnabledTemplate(badMappingWithTypeBuilder, true);

XContentBuilder badMappingWithoutTypeBuilder = jsonBuilder();
badMappingWithoutTypeBuilder.startObject();
{
badMappingWithoutTypeBuilder.startObject(FieldNamesFieldMapper.NAME);
{
badMappingWithoutTypeBuilder.field("enabled", randomBoolean());
}
badMappingWithoutTypeBuilder.endObject();
}
badMappingWithoutTypeBuilder.endObject();
assertFieldNamesEnabledTemplate(badMappingWithoutTypeBuilder, true);
}

private void assertFieldNamesEnabledTemplate(XContentBuilder templateBuilder, boolean expectIssue) throws IOException {
String badTemplateName = randomAlphaOfLength(5);
final ClusterState state = ClusterState.builder(new ClusterName(randomAlphaOfLength(5)))
.metaData(MetaData.builder()
.put(IndexTemplateMetaData.builder(badTemplateName)
.patterns(Collections.singletonList(randomAlphaOfLength(5)))
.putMapping("_doc", Strings.toString(templateBuilder))
.build())
.build())
.build();

List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(state));
if (expectIssue) {
assertEquals(1, issues.size());
DeprecationIssue issue = issues.get(0);
assertEquals(DeprecationIssue.Level.CRITICAL, issue.getLevel());
assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
, issue.getUrl());
assertEquals("Index templates contain _field_names settings.", issue.getMessage());
assertEquals("Index templates [" + badTemplateName + "] "
+ "use the deprecated `enable` setting for the `" + FieldNamesFieldMapper.NAME +
"` field. Using this setting in new index mappings will throw an error in the next major version and " +
"needs to be removed from existing mappings and templates.", issue.getDetails());
} else {
assertTrue(issues.isEmpty());
}
}

public void testPollIntervalTooLow() {
{
final String tooLowInterval = randomTimeValue(1, 999, "ms", "micros", "nanos");
Expand Down
32 changes: 30 additions & 2 deletions ...cation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecksTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.joda.JodaDeprecationPatterns;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
Expand All @@ -27,8 +28,8 @@
import static java.util.Collections.singletonList;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.deprecation.DeprecationChecks.INDEX_SETTINGS_CHECKS;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.collection.IsIterableContainingInOrder.contains;

public class IndexDeprecationChecksTests extends ESTestCase {
Expand Down Expand Up @@ -161,6 +162,7 @@ public void testChainedMultiFields() throws IOException {
"The names of fields that contain chained multi-fields: [[type: _doc, field: invalid-field]]");
assertEquals(singletonList(expected), issues);
}

public void testDefinedPatternsDoNotWarn() throws IOException {
String simpleMapping = "{\n" +
"\"properties\" : {\n" +
Expand Down Expand Up @@ -412,4 +414,30 @@ public void testDefaultTranslogRetentionSettings() {
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(indexMetaData));
assertThat(issues, empty());
}

public void testFieldNamesEnabling() throws IOException {
XContentBuilder xContent = XContentFactory.jsonBuilder().startObject()
.startObject(FieldNamesFieldMapper.NAME)
.field("enabled", randomBoolean())
.endObject()
.endObject();
String mapping = BytesReference.bytes(xContent).utf8ToString();

IndexMetaData simpleIndex = IndexMetaData.builder(randomAlphaOfLengthBetween(5, 10))
.settings(settings(
VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT)))
.numberOfShards(1)
.numberOfReplicas(0)
.putMapping("_doc", mapping).build();
List<DeprecationIssue> issues = DeprecationChecks.filterChecks(INDEX_SETTINGS_CHECKS, c -> c.apply(simpleIndex));
assertEquals(1, issues.size());

DeprecationIssue issue = issues.get(0);
assertEquals(DeprecationIssue.Level.WARNING, issue.getLevel());
assertEquals("https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html#fieldnames-enabling"
, issue.getUrl());
assertEquals("Index mapping contain explicit _field_names enabling settings.", issue.getMessage());
assertEquals("The index mapping contains a deprecated `enabled` setting for `_field_names` that should be removed moving foward.",
issue.getDetails());
}
}

0 comments on commit b3e5098

Please sign in to comment.