From 8ea0aaac42b0b98e4c95a3a440a381f7aa206e73 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 16 Sep 2024 13:27:31 -0700 Subject: [PATCH 1/7] Add option to skip using _ignored_source field for synthetic source --- .../common/settings/IndexScopedSettings.java | 3 + .../elasticsearch/index/IndexSettings.java | 26 ++++ .../index/mapper/DocumentParserContext.java | 4 +- .../mapper/IgnoredSourceFieldMapper.java | 71 ++++++++++- .../index/mapper/SourceLoader.java | 1 - ...edSourceFieldMapperConfigurationTests.java | 120 ++++++++++++++++++ 6 files changed, 218 insertions(+), 7 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index e4588e648318c..778136cbf5d31 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; @@ -183,6 +184,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 140133378ec06..41523c6dc2c7e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; @@ -778,6 +779,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; + private volatile boolean skipIgnoredSourceWrite; + private volatile boolean skipIgnoredSourceRead; /** * The maximum number of refresh listeners allows on this shard. @@ -936,6 +939,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); + skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); + skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1018,6 +1023,11 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); + scopedSettings.addSettingsUpdateConsumer( + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + this::setSkipIgnoredSourceWrite + ); + scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1594,6 +1604,22 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } + public boolean getSkipIgnoredSourceWrite() { + return skipIgnoredSourceWrite; + } + + private void setSkipIgnoredSourceWrite(boolean value) { + this.skipIgnoredSourceWrite = value; + } + + public boolean getSkipIgnoredSourceRead() { + return skipIgnoredSourceRead; + } + + private void setSkipIgnoredSourceRead(boolean value) { + this.skipIgnoredSourceRead = value; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 26db211345eae..5b21a3f9c945b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -333,7 +333,7 @@ final boolean getClonedSource() { } public final boolean canAddIgnoredField() { - return mappingLookup.isSourceSynthetic() && clonedSource == false; + return mappingLookup.isSourceSynthetic() && clonedSource == false && indexSettings().getSkipIgnoredSourceWrite() == false; } Mapper.SourceKeepMode sourceKeepModeFromIndexSettings() { @@ -367,7 +367,7 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic()) { + if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite()) { /* Mark this field as containing copied data meaning it should not be present in synthetic _source (to be consistent with stored _source). diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 35bbec6355762..6585fd3de655c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -10,12 +10,15 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -27,6 +30,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.stream.Stream; /** @@ -40,6 +44,7 @@ * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable. */ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { + private final IndexSettings indexSettings; // This factor is used to combine two offsets within the same integer: // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET) @@ -49,12 +54,26 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { public static final String NAME = "_ignored_source"; - public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper(); - - public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); + public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings())); static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); + public static final Setting SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_write", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope, + Setting.Property.ServerlessPublic + ); + + public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_read", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope, + Setting.Property.ServerlessPublic + ); + /* * Container for the ignored field data: * - the full name @@ -108,8 +127,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } } - private IgnoredSourceFieldMapper() { + private IgnoredSourceFieldMapper(IndexSettings indexSettings) { super(IgnoredValuesFieldMapperType.INSTANCE); + this.indexSettings = indexSettings; } @Override @@ -129,6 +149,49 @@ public void postParse(DocumentParserContext context) { } } + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + // Loader implementation that only controls if this fields values are loaded. + // Handling of loaded values is in SourceLoader and ObjectMapper#SyntheticSourceFieldLoader. + // The reason for this implementation is that it is easier to work with index settings here + // than in SourceLoader. + return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() { + @Override + public Stream> storedFieldLoaders() { + if (indexSettings.getSkipIgnoredSourceRead()) { + return Stream.empty(); + } + return Stream.of(Map.entry(NAME, (v) -> {})); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + + } + + @Override + public String fieldName() { + // Doesn't really matter + return NAME; + } + + @Override + public void reset() { + + } + }); + } + static byte[] encode(NameValue values) { assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 43bf6f2bd83dd..baff3835d104b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -118,7 +118,6 @@ public Synthetic(Supplier fieldLoaderSupplier, SourceField .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); this.metrics = metrics; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java new file mode 100644 index 0000000000000..958bc45598cb6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase { + public void testDisableIgnoredSourceRead() throws IOException { + var mapper = documentMapperWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapper.parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapper, inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + } + + public void testDisableIgnoredSourceWrite() throws IOException { + var mapper = documentMapperWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapper.parse(source(inputDocument)); + // Field is not written. + assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapper, inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + } + + private DocumentMapper documentMapperWithCustomSettings( + Map customSettings, + CheckedConsumer mapping + ) throws IOException { + var settings = Settings.builder(); + for (var entry : customSettings.entrySet()) { + settings.put(entry.getKey(), entry.getValue()); + } + + return createMapperService(settings.build(), syntheticSourceMapping(mapping)).documentMapper(); + } + + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) + throws IOException { + // Disabling this field via index settings leads to some values not being present in source and assertReaderEquals validation to + // fail as a result. + // This is expected, these settings are introduced only as a safety net when related logic blocks ingestion or search + // and we would rather lose some part of source but unblock the workflow. + } +} From cdffb599a36bcba2e32c7cacd160909505f925a3 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 16 Sep 2024 14:02:31 -0700 Subject: [PATCH 2/7] iter --- .../index/mapper/IgnoredSourceFieldMapper.java | 6 ++---- .../xpack/ccr/action/TransportResumeFollowActionTests.java | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 6585fd3de655c..7b4c95f464d9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -62,16 +62,14 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { "index.mapping.synthetic_source.skip_ignored_source_write", false, Setting.Property.Dynamic, - Setting.Property.IndexScope, - Setting.Property.ServerlessPublic + Setting.Property.IndexScope ); public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( "index.mapping.synthetic_source.skip_ignored_source_read", false, Setting.Property.Dynamic, - Setting.Property.IndexScope, - Setting.Property.ServerlessPublic + Setting.Property.IndexScope ); /* diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 7de0d775ba150..b4be0b33a464e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; @@ -331,6 +332,8 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC From 1ec39f37629cfdc95eb4c0a636a3fd9137aa7925 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 16 Sep 2024 16:05:39 -0700 Subject: [PATCH 3/7] Change everything --- .../index/get/ShardGetService.java | 6 +- .../index/mapper/DocumentParserContext.java | 2 +- .../mapper/IgnoredSourceFieldMapper.java | 45 ---- .../index/mapper/NestedObjectMapper.java | 35 ++- .../index/mapper/SourceFieldMapper.java | 89 ++++--- .../index/mapper/SourceLoader.java | 6 +- .../index/query/SearchExecutionContext.java | 8 +- .../search/lookup/SourceProvider.java | 5 +- .../mapper/DocCountFieldMapperTests.java | 4 +- .../mapper/DynamicFieldsBuilderTests.java | 10 +- ...edSourceFieldMapperConfigurationTests.java | 16 +- .../mapper/IgnoredSourceFieldMapperTests.java | 234 +++++++++--------- .../index/mapper/KeywordFieldMapperTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 54 ++-- .../index/mapper/RangeFieldMapperTests.java | 10 +- .../index/mapper/SourceFieldMetricsTests.java | 7 +- .../index/mapper/SourceLoaderTests.java | 29 ++- .../query/SearchExecutionContextTests.java | 8 +- .../index/mapper/MapperServiceTestCase.java | 30 ++- .../index/mapper/MapperTestCase.java | 20 +- .../mapper/HistogramFieldMapperTests.java | 5 +- .../accesscontrol/FieldSubsetReaderTests.java | 18 +- ...AggregateDoubleMetricFieldMapperTests.java | 4 +- .../ConstantKeywordFieldMapperTests.java | 4 +- 24 files changed, 345 insertions(+), 306 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 41879c64c3338..528762883600a 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -307,7 +307,11 @@ private GetResult innerGetFetch( Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()) + ? new SourceLoader.Synthetic( + mappingLookup.getMapping()::syntheticFieldLoader, + indexSettings, + mapperMetrics.sourceFieldMetrics() + ) : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 5b21a3f9c945b..38fa8d02db84e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -367,7 +367,7 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite()) { + if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) { /* Mark this field as containing copied data meaning it should not be present in synthetic _source (to be consistent with stored _source). diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 7b4c95f464d9d..eae644db1a34c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Setting; @@ -30,7 +29,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.stream.Stream; /** @@ -147,49 +145,6 @@ public void postParse(DocumentParserContext context) { } } - @Override - protected SyntheticSourceSupport syntheticSourceSupport() { - // Loader implementation that only controls if this fields values are loaded. - // Handling of loaded values is in SourceLoader and ObjectMapper#SyntheticSourceFieldLoader. - // The reason for this implementation is that it is easier to work with index settings here - // than in SourceLoader. - return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() { - @Override - public Stream> storedFieldLoaders() { - if (indexSettings.getSkipIgnoredSourceRead()) { - return Stream.empty(); - } - return Stream.of(Map.entry(NAME, (v) -> {})); - } - - @Override - public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { - return null; - } - - @Override - public boolean hasValue() { - return false; - } - - @Override - public void write(XContentBuilder b) throws IOException { - - } - - @Override - public String fieldName() { - // Doesn't really matter - return NAME; - } - - @Override - public void reset() { - - } - }); - } - static byte[] encode(NameValue values) { assert values.parentOffset < PARENT_OFFSET_IN_NAME_OFFSET; assert values.parentOffset * (long) PARENT_OFFSET_IN_NAME_OFFSET < Integer.MAX_VALUE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index adf1b329d9e83..b23e6fbf0e369 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -48,11 +49,18 @@ public static class Builder extends ObjectMapper.Builder { private Explicit includeInParent = Explicit.IMPLICIT_FALSE; private final IndexVersion indexCreatedVersion; private final Function bitSetProducer; + private final IndexSettings indexSettings; - public Builder(String name, IndexVersion indexCreatedVersion, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name, Optional.empty()); this.indexCreatedVersion = indexCreatedVersion; this.bitSetProducer = bitSetProducer; + this.indexSettings = indexSettings; } Builder includeInRoot(boolean includeInRoot) { @@ -113,7 +121,8 @@ public NestedObjectMapper build(MapperBuilderContext context) { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitSetProducer + bitSetProducer, + indexSettings ); } } @@ -128,7 +137,8 @@ public Mapper.Builder parse(String name, Map node, MappingParser NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder( name, parserContext.indexVersionCreated(), - parserContext::bitSetProducer + parserContext::bitSetProducer, + parserContext.getIndexSettings() ); parseNested(name, node, builder); parseObjectFields(node, parserContext, builder); @@ -195,6 +205,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { private final Query nestedTypeFilter; // Function to create a bitset for identifying parent documents private final Function bitsetProducer; + private final IndexSettings indexSettings; NestedObjectMapper( String name, @@ -208,7 +219,8 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Query parentTypeFilter, String nestedTypePath, Query nestedTypeFilter, - Function bitsetProducer + Function bitsetProducer, + IndexSettings indexSettings ) { super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; @@ -217,6 +229,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { this.includeInParent = includeInParent; this.includeInRoot = includeInRoot; this.bitsetProducer = bitsetProducer; + this.indexSettings = indexSettings; } public Query parentTypeFilter() { @@ -254,7 +267,7 @@ public Map getChildren() { @Override public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer, indexSettings); builder.enabled = enabled; builder.dynamic = dynamic; builder.includeInRoot = includeInRoot; @@ -276,7 +289,8 @@ NestedObjectMapper withoutMappers() { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -351,7 +365,8 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -383,7 +398,11 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return SourceLoader.SyntheticFieldLoader.NOTHING; } - SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); + SourceLoader sourceLoader = new SourceLoader.Synthetic( + () -> super.syntheticFieldLoader(mappers.values().stream(), true), + indexSettings, + NOOP + ); var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()); return new NestedSyntheticFieldLoader( storedFieldLoader, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3318595ed7129..583a0be70a903 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -36,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.function.Function; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING; @@ -63,81 +65,90 @@ private enum Mode { SYNTHETIC } - private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( + private static final Function DEFAULT = (settings) -> new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, - true + true, + settings ); - private static final SourceFieldMapper DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( + private static final Function DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, - false + false, + settings ); - private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( + private static final Function TSDB_DEFAULT = (settings) -> new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - true + true, + settings ); - private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( + private static final Function TSDB_DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - false + false, + settings ); - private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( + private static final Function LOGSDB_DEFAULT = (settings) -> new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.LOGSDB, - true + true, + settings ); - private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( + private static final Function LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.LOGSDB, - false + false, + settings ); /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( + private static final Function TSDB_LEGACY_DEFAULT = (settings) -> new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - true + true, + settings ); - private static final SourceFieldMapper TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - false - ); + private static final Function TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE = ( + settings) -> new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + false, + settings + ); public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -197,17 +208,21 @@ public static class Builder extends MetadataFieldMapper.Builder { private final boolean enableRecoverySource; + private final IndexSettings indexSettings; + public Builder( IndexMode indexMode, final Settings settings, boolean supportsCheckForNonDefaultParams, - boolean enableRecoverySource + boolean enableRecoverySource, + IndexSettings indexSettings ) { super(Defaults.NAME); this.indexMode = indexMode; this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); this.enableRecoverySource = enableRecoverySource; + this.indexSettings = indexSettings; } public Builder setSynthetic() { @@ -240,11 +255,12 @@ public SourceFieldMapper build() { } } if (isDefault()) { - return switch (indexMode) { + var factory = switch (indexMode) { case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; }; + return factory.apply(indexSettings); } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -274,7 +290,8 @@ public SourceFieldMapper build() { includes.getValue().toArray(Strings.EMPTY_ARRAY), excludes.getValue().toArray(Strings.EMPTY_ARRAY), indexMode, - enableRecoverySource + enableRecoverySource, + indexSettings ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -290,21 +307,24 @@ public SourceFieldMapper build() { if (indexMode.isSyntheticSourceEnabled()) { if (indexMode == IndexMode.TIME_SERIES) { if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) { - return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; + return (enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); } else { - return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; + return (enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE).apply( + c.getIndexSettings() + ); } } else if (indexMode == IndexMode.LOGSDB) { - return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; + return (enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); } } - return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; + return (enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); }, c -> new Builder( c.getIndexSettings().getMode(), c.getSettings(), c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK), - INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()) + INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()), + c.getIndexSettings() ) ); @@ -358,6 +378,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final IndexMode indexMode; private final boolean enableRecoverySource; + private final IndexSettings indexSettings; private SourceFieldMapper( Mode mode, @@ -365,7 +386,8 @@ private SourceFieldMapper( String[] includes, String[] excludes, IndexMode indexMode, - boolean enableRecoverySource + boolean enableRecoverySource, + IndexSettings indexSettings ) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; @@ -380,6 +402,7 @@ private SourceFieldMapper( this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; this.enableRecoverySource = enableRecoverySource; + this.indexSettings = indexSettings; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -452,7 +475,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, Settings.EMPTY, false, enableRecoverySource).init(this); + return new Builder(indexMode, Settings.EMPTY, false, enableRecoverySource, indexSettings).init(this); } /** @@ -460,7 +483,7 @@ public FieldMapper.Builder getMergeBuilder() { */ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics); + return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, indexSettings, metrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index baff3835d104b..2c1c3a8a343b3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.LeafReader; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; @@ -112,12 +113,15 @@ class Synthetic implements SourceLoader { * @param fieldLoaderSupplier A supplier to create {@link SyntheticFieldLoader}, one for each leaf. * @param metrics Metrics for profiling. */ - public Synthetic(Supplier fieldLoaderSupplier, SourceFieldMetrics metrics) { + public Synthetic(Supplier fieldLoaderSupplier, IndexSettings indexSettings, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = fieldLoaderSupplier; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); + if (indexSettings.getSkipIgnoredSourceRead() == false) { + this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); + } this.metrics = metrics; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index b07112440d3c2..4a3f16a0579dc 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -439,7 +439,11 @@ public boolean isSourceSynthetic() { */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()); + return new SourceLoader.Synthetic( + mappingLookup.getMapping()::syntheticFieldLoader, + indexSettings, + mapperMetrics.sourceFieldMetrics() + ); } return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); } @@ -494,7 +498,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), indexSettings, mapperMetrics.sourceFieldMetrics()) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index e232aec5d1f6c..8654e95bb3c30 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,6 +10,7 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceFieldMetrics; @@ -48,7 +49,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) { - return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics)); + static SourceProvider fromSyntheticSource(Mapping mapping, IndexSettings indexSettings, SourceFieldMetrics metrics) { + return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, indexSettings, metrics)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index b4bc2f23af087..e57a215a0cf26 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -84,8 +84,8 @@ public void testInvalidDocument_ArrayDocCount() throws Exception { } public void testSyntheticSource() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> {})); - assertThat(syntheticSource(mapper, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}")); + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> {})); + assertThat(syntheticSource(mapperService, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}")); } public void testSyntheticSourceMany() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 358ad07664870..246ad204be3bb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -9,8 +9,10 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -69,7 +71,13 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder( + null, + Settings.EMPTY, + false, + true, + new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()) + ).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 958bc45598cb6..5836c975cf7b5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -19,7 +19,7 @@ public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase { public void testDisableIgnoredSourceRead() throws IOException { - var mapper = documentMapperWithCustomSettings( + var mapperService = mapperServiceWithCustomSettings( Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true), b -> { b.startObject("fallback_field"); @@ -49,17 +49,17 @@ public void testDisableIgnoredSourceRead() throws IOException { b.endObject(); }; - var doc = mapper.parse(source(inputDocument)); + var doc = mapperService.documentMapper().parse(source(inputDocument)); // Field was written. assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); - String syntheticSource = syntheticSource(mapper, inputDocument); + String syntheticSource = syntheticSource(mapperService, inputDocument); // Values are not loaded. assertEquals("{}", syntheticSource); } public void testDisableIgnoredSourceWrite() throws IOException { - var mapper = documentMapperWithCustomSettings( + var mapperService = mapperServiceWithCustomSettings( Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true), b -> { b.startObject("fallback_field"); @@ -89,16 +89,16 @@ public void testDisableIgnoredSourceWrite() throws IOException { b.endObject(); }; - var doc = mapper.parse(source(inputDocument)); + var doc = mapperService.documentMapper().parse(source(inputDocument)); // Field is not written. assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); - String syntheticSource = syntheticSource(mapper, inputDocument); + String syntheticSource = syntheticSource(mapperService, inputDocument); // Values are not loaded. assertEquals("{}", syntheticSource); } - private DocumentMapper documentMapperWithCustomSettings( + private MapperService mapperServiceWithCustomSettings( Map customSettings, CheckedConsumer mapping ) throws IOException { @@ -107,7 +107,7 @@ private DocumentMapper documentMapperWithCustomSettings( settings.put(entry.getKey(), entry.getValue()); } - return createMapperService(settings.build(), syntheticSourceMapping(mapping)).documentMapper(); + return createMapperService(settings.build(), syntheticSourceMapping(mapping)); } protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index efa12d29f0e57..ede1f97c7a9eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -26,7 +26,7 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase { - private DocumentMapper getDocumentMapperWithFieldLimit() throws IOException { + private MapperService getMapperServiceWithFieldLimit() throws IOException { return createMapperService( Settings.builder() .put("index.mapping.total_fields.limit", 2) @@ -36,17 +36,17 @@ private DocumentMapper getDocumentMapperWithFieldLimit() throws IOException { b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "object").endObject(); }) - ).documentMapper(); + ); } private ParsedDocument getParsedDocumentWithFieldLimit(CheckedConsumer build) throws IOException { - DocumentMapper mapper = getDocumentMapperWithFieldLimit(); - return mapper.parse(source(build)); + MapperService mapperService = getMapperServiceWithFieldLimit(); + return mapperService.documentMapper().parse(source(build)); } private String getSyntheticSourceWithFieldLimit(CheckedConsumer build) throws IOException { - DocumentMapper documentMapper = getDocumentMapperWithFieldLimit(); - return syntheticSource(documentMapper, build); + MapperService mapperService = getMapperServiceWithFieldLimit(); + return syntheticSource(mapperService, build); } private MapperService createMapperServiceWithStoredArraySource(XContentBuilder mappings) throws IOException { @@ -236,11 +236,11 @@ public void testIgnoredDynamicArrayNestedInObject() throws IOException { public void testDisabledRootObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(topMapping(b -> { + MapperService mapperService = createMapperService(topMapping(b -> { b.startObject("_source").field("mode", "synthetic").endObject(); b.field("enabled", false); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { b.field("name", name); }); + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.field("name", name); }); assertEquals(String.format(Locale.ROOT, """ {"name":"%s"}""", name), syntheticSource); } @@ -250,11 +250,11 @@ public void testDisabledRootObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(topMapping(b -> { + MapperService mapperService = createMapperService(topMapping(b -> { b.startObject("_source").field("mode", "synthetic").endObject(); b.field("enabled", false); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -292,10 +292,10 @@ public void testDisabledRootObjectManyFields() throws IOException { public void testDisabledObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.field("name", name); @@ -308,10 +308,10 @@ public void testDisabledObjectSingleField() throws IOException { public void testDisabledObjectContainsArray() throws IOException { String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -328,11 +328,11 @@ public void testDisabledObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -372,7 +372,7 @@ public void testDisabledSubObject() throws IOException { boolean booleanValue = randomBoolean(); int intValue = randomInt(); String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -385,8 +385,8 @@ public void testDisabledSubObject() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -404,7 +404,7 @@ public void testDisabledSubObject() throws IOException { } public void testDisabledSubobjectContainsArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -417,11 +417,11 @@ public void testDisabledSubobjectContainsArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -447,7 +447,7 @@ public void testMixedDisabledEnabledObjects() throws IOException { int intValue = randomInt(); String foo = randomAlphaOfLength(20); String bar = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -477,8 +477,8 @@ public void testMixedDisabledEnabledObjects() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -507,11 +507,11 @@ public void testMixedDisabledEnabledObjects() throws IOException { } public void testIndexStoredArraySourceRootValueArray() throws IOException { - DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.array("int_value", new int[] { 30, 20, 10 }); b.field("bool_value", true); }); @@ -520,7 +520,7 @@ public void testIndexStoredArraySourceRootValueArray() throws IOException { } public void testIndexStoredArraySourceRootObjectArray() throws IOException { - DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -532,8 +532,8 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { } b.endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); b.startObject().field("int_value", 10).endObject(); b.startObject().field("int_value", 20).endObject(); @@ -545,7 +545,7 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { } public void testIndexStoredArraySourceNestedValueArray() throws IOException { - DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -557,8 +557,8 @@ public void testIndexStoredArraySourceNestedValueArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.array("int_value", new int[] { 30, 20, 10 }); @@ -571,7 +571,7 @@ public void testIndexStoredArraySourceNestedValueArray() throws IOException { } public void testIndexStoredArraySourceNestedObjectArray() throws IOException { - DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -592,8 +592,8 @@ public void testIndexStoredArraySourceNestedObjectArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.startArray("to"); @@ -609,7 +609,7 @@ public void testIndexStoredArraySourceNestedObjectArray() throws IOException { } public void testRootArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -621,8 +621,8 @@ public void testRootArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); b.startObject().field("int_value", 10).endObject(); b.startObject().field("int_value", 20).endObject(); @@ -633,7 +633,7 @@ public void testRootArray() throws IOException { } public void testNestedArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -670,10 +670,10 @@ public void testNestedArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -707,7 +707,7 @@ public void testNestedArray() throws IOException { } public void testArrayWithinArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object").field("store_array_source", true); @@ -726,10 +726,10 @@ public void testArrayWithinArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject(); @@ -760,7 +760,7 @@ public void testArrayWithinArray() throws IOException { } public void testDisabledObjectWithinHigherLevelArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -779,10 +779,10 @@ public void testDisabledObjectWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject(); @@ -803,7 +803,7 @@ public void testDisabledObjectWithinHigherLevelArray() throws IOException { } public void testStoredArrayWithinHigherLevelArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -822,10 +822,10 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject(); @@ -856,7 +856,7 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { } public void testFallbackFieldWithinHigherLevelArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -867,10 +867,10 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { @@ -886,7 +886,7 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { } public void testFieldOrdering() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("A").field("type", "integer").endObject(); b.startObject("B").field("type", "object").field("store_array_source", true); { @@ -910,9 +910,9 @@ public void testFieldOrdering() throws IOException { } b.endObject(); b.startObject("E").field("type", "integer").endObject(); - })).documentMapper(); + })); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("C", 10); b.startArray("D"); { @@ -934,7 +934,7 @@ public void testFieldOrdering() throws IOException { } public void testNestedObjectWithField() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.field("store_array_source", true); @@ -946,9 +946,9 @@ public void testNestedObjectWithField() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); var syntheticSource = syntheticSource( - documentMapper, + mapperService, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -956,7 +956,7 @@ public void testNestedObjectWithField() throws IOException { } public void testNestedObjectWithArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.field("store_array_source", true); @@ -968,8 +968,8 @@ public void testNestedObjectWithArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -982,7 +982,7 @@ public void testNestedObjectWithArray() throws IOException { } public void testNestedSubobjectWithField() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1005,11 +1005,11 @@ public void testNestedSubobjectWithField() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1023,7 +1023,7 @@ public void testNestedSubobjectWithField() throws IOException { } public void testNestedSubobjectWithArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1046,11 +1046,11 @@ public void testNestedSubobjectWithArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1072,7 +1072,7 @@ public void testNestedSubobjectWithArray() throws IOException { } public void testNestedObjectIncludeInRoot() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested").field("store_array_source", true).field("include_in_root", true); { b.startObject("properties"); @@ -1083,9 +1083,9 @@ public void testNestedObjectIncludeInRoot() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); var syntheticSource = syntheticSource( - documentMapper, + mapperService, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1094,10 +1094,10 @@ public void testNestedObjectIncludeInRoot() throws IOException { public void testNoDynamicObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.field("name", name); @@ -1113,7 +1113,7 @@ public void testNoDynamicObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("dynamic", "false"); { @@ -1124,9 +1124,9 @@ public void testNoDynamicObjectManyFields() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1157,10 +1157,10 @@ public void testNoDynamicObjectManyFields() throws IOException { } public void testNoDynamicObjectSimpleArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("name", "foo").endObject(); @@ -1173,22 +1173,19 @@ public void testNoDynamicObjectSimpleArray() throws IOException { } public void testNoDynamicObjectSimpleValueArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource( - documentMapper, - b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); } - ); + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); }); assertEquals(""" {"path":{"name":["A","B","C","D"]}}""", syntheticSource); } public void testNoDynamicObjectNestedArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().startObject("to").field("foo", "A").field("bar", "B").endObject().endObject(); @@ -1202,10 +1199,10 @@ public void testNoDynamicObjectNestedArray() throws IOException { public void testRuntimeDynamicObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.field("name", name); @@ -1221,7 +1218,7 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("dynamic", "runtime"); { @@ -1232,9 +1229,9 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1265,10 +1262,10 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { } public void testRuntimeDynamicObjectSimpleArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("name", "foo").endObject(); @@ -1281,22 +1278,19 @@ public void testRuntimeDynamicObjectSimpleArray() throws IOException { } public void testRuntimeDynamicObjectSimpleValueArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource( - documentMapper, - b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); } - ); + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); }); assertEquals(""" {"path":{"name":["A","B","C","D"]}}""", syntheticSource); } public void testRuntimeDynamicObjectNestedArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().startObject("to").field("foo", "A").field("bar", "B").endObject().endObject(); @@ -1309,7 +1303,7 @@ public void testRuntimeDynamicObjectNestedArray() throws IOException { } public void testDisabledSubObjectWithNameOverlappingParentName() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); b.startObject("properties"); { @@ -1317,8 +1311,8 @@ public void testDisabledSubObjectWithNameOverlappingParentName() throws IOExcept } b.endObject(); b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.startObject("at").field("foo", "A").endObject(); @@ -1330,7 +1324,7 @@ public void testDisabledSubObjectWithNameOverlappingParentName() throws IOExcept } public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); b.startObject("properties"); { @@ -1338,8 +1332,8 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx } b.endObject(); b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.startObject("at").field("foo", "A").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 2d2fad23b3831..5791e171af215 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -666,7 +666,7 @@ public void testSyntheticSourceForDisabledField() throws Exception { ) ); String value = randomAlphaOfLengthBetween(1, 20); - assertEquals("{\"field\":\"" + value + "\"}", syntheticSource(mapper.documentMapper(), b -> b.field("field", value))); + assertEquals("{\"field\":\"" + value + "\"}", syntheticSource(mapper, b -> b.field("field", value))); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 37971a1908fa2..b484bd1e5cfd1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1584,7 +1584,7 @@ public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOExceptio } public void testSyntheticNestedWithObject() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1595,9 +1595,9 @@ public void testSyntheticNestedWithObject() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); var syntheticSource = syntheticSource( - documentMapper, + mapperService, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1605,7 +1605,7 @@ public void testSyntheticNestedWithObject() throws IOException { } public void testSyntheticNestedWithArray() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1616,8 +1616,8 @@ public void testSyntheticNestedWithArray() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -1630,7 +1630,7 @@ public void testSyntheticNestedWithArray() throws IOException { } public void testSyntheticNestedWithSubObjects() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1652,11 +1652,11 @@ public void testSyntheticNestedWithSubObjects() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1670,7 +1670,7 @@ public void testSyntheticNestedWithSubObjects() throws IOException { } public void testSyntheticNestedWithSubArrays() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1692,11 +1692,11 @@ public void testSyntheticNestedWithSubArrays() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(documentMapper, b -> { + var syntheticSource = syntheticSource(mapperService, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1718,7 +1718,7 @@ public void testSyntheticNestedWithSubArrays() throws IOException { } public void testSyntheticNestedWithIncludeInRoot() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested").field("include_in_root", true); { b.startObject("properties"); @@ -1729,9 +1729,9 @@ public void testSyntheticNestedWithIncludeInRoot() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); + })); var syntheticSource = syntheticSource( - documentMapper, + mapperService, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1739,7 +1739,7 @@ public void testSyntheticNestedWithIncludeInRoot() throws IOException { } public void testSyntheticNestedWithEmptyObject() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1749,14 +1749,14 @@ public void testSyntheticNestedWithEmptyObject() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path").nullField("foo").endObject(); }); + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").nullField("foo").endObject(); }); assertEquals(""" {"path":{}}""", syntheticSource); } public void testSyntheticNestedWithEmptySubObject() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1770,8 +1770,8 @@ public void testSyntheticNestedWithEmptySubObject() throws IOException { b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path"); { b.startObject("to").nullField("foo").endObject(); @@ -1783,7 +1783,7 @@ public void testSyntheticNestedWithEmptySubObject() throws IOException { } public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1793,8 +1793,8 @@ public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOExcepti b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().field("foo", "A").endObject(); @@ -1807,7 +1807,7 @@ public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOExcepti } public void testSyntheticNestedWithArrayContainingOnlyEmptyObject() throws IOException { - DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1817,8 +1817,8 @@ public void testSyntheticNestedWithArrayContainingOnlyEmptyObject() throws IOExc b.endObject(); } b.endObject(); - })).documentMapper(); - var syntheticSource = syntheticSource(documentMapper, b -> { + })); + var syntheticSource = syntheticSource(mapperService, b -> { b.startArray("path"); { b.startObject().nullField("foo").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 5676c5c92e5a8..fcddb329efb0c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -384,7 +384,7 @@ protected TestRange randomRangeForSyntheticSourceTest() { } protected Source getSourceFor(CheckedConsumer mapping, List inputValues) throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(mapping)); + MapperService mapperService = createMapperService(syntheticSourceMapping(mapping)); CheckedConsumer input = b -> { b.field("field"); @@ -401,11 +401,15 @@ protected Source getSourceFor(CheckedConsumer mapp try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = mapper.parse(source(input)).rootDoc(); + LuceneDocument doc = mapperService.documentMapper().parse(source(input)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromSyntheticSource( + mapperService.documentMapper().mapping(), + mapperService.getIndexSettings(), + SourceFieldMetrics.NOOP + ); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index 81532114a7050..ac605c1ae31d3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -37,16 +37,17 @@ public void testFieldHasValueWithEmptyFieldInfos() {} public void testSyntheticSourceLoadLatency() throws IOException { var mapping = syntheticSourceMapping(b -> b.startObject("kwd").field("type", "keyword").endObject()); - var mapper = createDocumentMapper(mapping); + var mapperService = createMapperService(mapping); try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = mapper.parse(source(b -> b.field("kwd", "foo"))).rootDoc(); + LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("kwd", "foo"))).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { SourceProvider provider = SourceProvider.fromSyntheticSource( - mapper.mapping(), + mapperService.documentMapper().mapping(), + mapperService.getIndexSettings(), createTestMapperMetrics().sourceFieldMetrics() ); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index c6a4021d8a542..02bae022492ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -25,25 +25,25 @@ public void testNonSynthetic() throws IOException { } public void testEmptyObject() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); - assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" + assertTrue(mapperService.documentMapper().mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); + assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); } public void testDotsInFieldName() throws IOException { - DocumentMapper mapper = createDocumentMapper( + MapperService mapperService = createMapperService( syntheticSourceMapping(b -> b.startObject("foo.bar.baz").field("type", "keyword").endObject()) ); - assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo":{"bar":{"baz":"aaa"}}}""")); } public void testNoSubobjectsIntermediateObject() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("foo"); { b.field("type", "object").field("subobjects", false); @@ -55,7 +55,7 @@ public void testNoSubobjectsIntermediateObject() throws IOException { } b.endObject(); })); - assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo":{"bar.baz":"aaa"}}""")); } @@ -67,32 +67,35 @@ public void testNoSubobjectsRootObject() throws IOException { b.startObject("foo.bar.baz").field("type", "keyword").endObject(); b.endObject(); }); - DocumentMapper mapper = createDocumentMapper(mappings); - assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + MapperService mapperService = createMapperService(mappings); + assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo.bar.baz":"aaa"}""")); } public void testSorted() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "keyword").endObject(); b.startObject("baz").field("type", "keyword").endObject(); })); assertThat( - syntheticSource(mapper, b -> b.field("foo", "over the lazy dog").field("bar", "the quick").field("baz", "brown fox jumped")), + syntheticSource( + mapperService, + b -> b.field("foo", "over the lazy dog").field("bar", "the quick").field("baz", "brown fox jumped") + ), equalTo(""" {"bar":"the quick","baz":"brown fox jumped","foo":"over the lazy dog"}""") ); } public void testArraysPushedToLeaves() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("o").startObject("properties"); b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "keyword").endObject(); b.endObject().endObject(); })); - assertThat(syntheticSource(mapper, b -> { + assertThat(syntheticSource(mapperService, b -> { b.startArray("o"); b.startObject().field("foo", "a").endObject(); b.startObject().field("bar", "b").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 658176b1d31dd..0b53a73337981 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -384,7 +384,13 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder( + null, + Settings.EMPTY, + false, + true, + new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()) + ).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index a9ee0317ce1ee..0e8a2a68d4bb6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -801,17 +801,18 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t return new RandomIndexWriter(random(), directory, new StandardAnalyzer()); } - protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { + protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) + throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); - ParsedDocument doc = mapper.parse(source(build)); + ParsedDocument doc = mapperService.documentMapper().parse(source(build)); doc.updateSeqID(0, 0); doc.version().setLongValue(0); iw.addDocuments(doc.docs()); iw.close(); try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); - roundTripSyntheticSource(mapper, syntheticSource, indexReader); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + roundTripSyntheticSource(mapperService, syntheticSource, indexReader); return syntheticSource; } } @@ -827,30 +828,35 @@ protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); example.mapping().accept(b); b.endObject(); })); - assertThat(syntheticSource(mapper, example::buildInput), equalTo(example.expected())); + assertThat(syntheticSource(mapperService, example::buildInput), equalTo(example.expected())); } protected boolean supportsEmptyInputArray() { @@ -1218,24 +1218,24 @@ public final void testSyntheticSourceMany() throws IOException { public final void testNoSyntheticSourceForScript() throws IOException { // Fetch the ingest script support to eagerly assumeFalse if the mapper doesn't support ingest scripts ingestScriptSupport(); - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); minimalMapping(b); b.field("script", randomBoolean() ? "empty" : "non-empty"); b.endObject(); })); - assertThat(syntheticSource(mapper, b -> {}), equalTo("{}")); + assertThat(syntheticSource(mapperService, b -> {}), equalTo("{}")); } public final void testSyntheticSourceInObject() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceExample syntheticSourceExample = syntheticSourceSupport(ignoreMalformed).example(5); - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("obj").startObject("properties").startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject().endObject().endObject(); })); - assertThat(syntheticSource(mapper, b -> { + assertThat(syntheticSource(mapperService, b -> { b.startObject("obj"); syntheticSourceExample.buildInput(b); b.endObject(); @@ -1247,14 +1247,14 @@ public final void testSyntheticEmptyList() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceSupport support = syntheticSourceSupport(ignoreMalformed); SyntheticSourceExample syntheticSourceExample = support.example(5); - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject(); })); var expected = support.preservesExactSource() ? "{\"field\":[]}" : "{}"; - assertThat(syntheticSource(mapper, b -> b.startArray("field").endArray()), equalTo(expected)); + assertThat(syntheticSource(mapperService, b -> b.startArray("field").endArray()), equalTo(expected)); } private boolean shouldUseIgnoreMalformed() { @@ -1508,12 +1508,12 @@ public final void testSyntheticSourceInvalid() throws IOException { public final void testSyntheticSourceInNestedObject() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceExample syntheticSourceExample = syntheticSourceSupport(ignoreMalformed).example(5); - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("obj").field("type", "nested").startObject("properties").startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject().endObject().endObject(); })); - assertThat(syntheticSource(mapper, b -> { + assertThat(syntheticSource(mapperService, b -> { b.startObject("obj"); syntheticSourceExample.buildInput(b); b.endObject(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 6fcbf20b8657f..7a859d27dec1b 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -374,7 +375,7 @@ protected IngestScriptSupport ingestScriptSupport() { } public void testArrayValueSyntheticSource() throws Exception { - DocumentMapper mapper = createDocumentMapper( + MapperService mapperService = createMapperService( syntheticSourceFieldMapping(b -> b.field("type", "histogram").field("ignore_malformed", "true")) ); @@ -401,7 +402,7 @@ public void testArrayValueSyntheticSource() throws Exception { } expected.endObject(); - var syntheticSource = syntheticSource(mapper, arrayValue); + var syntheticSource = syntheticSource(mapperService, arrayValue); assertEquals(Strings.toString(expected), syntheticSource); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 560dee9b5843c..344dd412bd5cd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -717,7 +718,7 @@ public void testSourceFilteringIntegration() throws Exception { } public void testIgnoredSourceFilteringIntegration() throws Exception { - DocumentMapper mapper = createMapperService( + MapperService mapperService = createMapperService( Settings.builder() .put("index.mapping.total_fields.limit", 1) .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) @@ -725,7 +726,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { syntheticSourceMapping(b -> { b.startObject("foo").field("type", "keyword").endObject(); }) - ).documentMapper(); + ); + DocumentMapper mapper = mapperService.documentMapper(); try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); @@ -748,7 +750,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals("{\"fieldA\":\"testA\"}", syntheticSource); } } @@ -765,7 +767,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}],"fieldB":"testB","obj":{"fieldC":"testC"}}""", syntheticSource); } @@ -779,7 +781,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals(""" {"obj":{"fieldC":"testC"}}""", syntheticSource); } @@ -797,7 +799,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}],"fieldA":"testA","fieldB":"testB"}""", syntheticSource); } @@ -811,7 +813,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}]}""", syntheticSource); } @@ -829,7 +831,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{}],"fieldA":"testA","fieldB":"testB","obj":{"fieldC":"testC"}}""", syntheticSource); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 5fbc25eb037a7..665339f41c6b7 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -527,7 +527,7 @@ protected IngestScriptSupport ingestScriptSupport() { } public void testArrayValueSyntheticSource() throws Exception { - DocumentMapper mapper = createDocumentMapper( + MapperService mapperService = createMapperService( syntheticSourceFieldMapping( b -> b.field("type", CONTENT_TYPE) .array("metrics", "min", "max") @@ -559,7 +559,7 @@ public void testArrayValueSyntheticSource() throws Exception { } expected.endObject(); - var syntheticSource = syntheticSource(mapper, arrayValue); + var syntheticSource = syntheticSource(mapperService, arrayValue); assertEquals(Strings.toString(expected), syntheticSource); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 92aac7897bcfd..e7bb0c6c9c743 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -325,12 +325,12 @@ protected Function loadBlockExpected() { } public void testNullValueSyntheticSource() throws IOException { - DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); b.field("type", "constant_keyword"); b.endObject(); })); - assertThat(syntheticSource(mapper, b -> {}), equalTo("{}")); + assertThat(syntheticSource(mapperService, b -> {}), equalTo("{}")); } @Override From bb75a5504648fb05d34dfeb37b2d39df6eaf9d8f Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Mon, 16 Sep 2024 16:59:16 -0700 Subject: [PATCH 4/7] fix --- .../FieldAliasMapperValidationTests.java | 5 +- .../index/mapper/NestedLookupTests.java | 5 +- .../index/mapper/NestedObjectMapperTests.java | 10 ++-- .../bucket/nested/NestedAggregatorTests.java | 5 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../mapper/SemanticTextFieldMapper.java | 55 +++++++++++++++---- 6 files changed, 56 insertions(+), 26 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index f303171c7e465..d48c5550631cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -186,9 +186,8 @@ private static ObjectMapper createObjectMapper(String name) { } private static NestedObjectMapper createNestedObjectMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } private static MappingLookup createMappingLookup( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java index 4953a330107b4..d209d08b48469 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java @@ -65,9 +65,8 @@ public void testMultiLevelParents() throws IOException { } private static NestedObjectMapper buildMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } public void testAllParentFilters() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index b484bd1e5cfd1..d8cbc28fae0c7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1505,10 +1505,10 @@ public void testIndexTemplatesMergeIncludes() throws IOException { public void testMergeNested() { NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); MapperException e = expectThrows( MapperException.class, @@ -1855,7 +1855,7 @@ public void testNestedMapperBuilderContextConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); builder.add(new Mapper.Builder("name") { @Override public Mapper build(MapperBuilderContext context) { @@ -1876,7 +1876,7 @@ public void testNestedMapperMergeContextRootConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); { MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(false, false, mergeReason); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.root(isSourceSynthetic, isDataStream, mergeReason, randomLong()); MapperMergeContext childMergeContext = nestedObjectMapper.createChildContext(mapperMergeContext, "name"); @@ -1907,7 +1907,7 @@ public void testNestedMapperMergeContextFromConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.from(mapperBuilderContext, randomLong()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 78943ed6ccdd7..c7e9f02c283ab 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -913,8 +913,7 @@ protected List objectMappers() { ); public static NestedObjectMapper nestedObject(String path) { - return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 0e9ca00702b68..583cdf302ad65 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -197,7 +197,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea }; NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).build(MapperBuilderContext.root(false, false)))); + }, null).build(MapperBuilderContext.root(false, false)))); return new SearchExecutionContext( 0, 0, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1577fbc4a642a..81dfba769136b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -80,13 +81,16 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper { public static final String CONTENT_TYPE = "semantic_text"; + private final IndexSettings indexSettings; + public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer), + (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer, c.getIndexSettings()), List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE)) ); public static class Builder extends FieldMapper.Builder { private final IndexVersion indexVersionCreated; + private final IndexSettings indexSettings; private final Parameter inferenceId = Parameter.stringParam( "inference_id", @@ -113,10 +117,22 @@ public static class Builder extends FieldMapper.Builder { private Function inferenceFieldBuilder; - public Builder(String name, IndexVersion indexVersionCreated, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexVersionCreated, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name); this.indexVersionCreated = indexVersionCreated; - this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer); + this.indexSettings = indexSettings; + this.inferenceFieldBuilder = c -> createInferenceField( + c, + indexVersionCreated, + modelSettings.get(), + bitSetProducer, + indexSettings + ); } public Builder setInferenceId(String id) { @@ -170,13 +186,20 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { indexVersionCreated, meta.getValue() ), - builderParams(this, context) + builderParams(this, context), + indexSettings ); } } - private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams) { + private SemanticTextFieldMapper( + String simpleName, + MappedFieldType mappedFieldType, + BuilderParams builderParams, + IndexSettings indexSettings + ) { super(simpleName, mappedFieldType, builderParams); + this.indexSettings = indexSettings; } @Override @@ -188,7 +211,9 @@ public Iterator iterator() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this); + return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer(), indexSettings).init( + this + ); } @Override @@ -229,7 +254,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio Builder builder = (Builder) new Builder( leafName(), fieldType().indexVersionCreated, - fieldType().getChunksField().bitsetProducer() + fieldType().getChunksField().bitsetProducer(), + indexSettings ).init(this); try { mapper = builder.setModelSettings(field.inference().modelSettings()) @@ -473,19 +499,26 @@ private static ObjectMapper createInferenceField( MapperBuilderContext context, IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE) - .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer)) + .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer, indexSettings)) .build(context); } private static NestedObjectMapper.Builder createChunksField( IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { - NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer); + NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder( + CHUNKS_FIELD, + indexVersionCreated, + bitSetProducer, + indexSettings + ); chunksField.dynamic(ObjectMapper.Dynamic.FALSE); KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false) .docValues(false); From 066f001b514a5f116f4c3ad01456420591464adc Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 17 Sep 2024 10:48:22 -0700 Subject: [PATCH 5/7] Revert most of test changes --- .../mapper/DocCountFieldMapperTests.java | 4 +- .../mapper/DynamicFieldsBuilderTests.java | 10 +- .../mapper/IgnoredSourceFieldMapperTests.java | 234 +++++++++--------- .../index/mapper/KeywordFieldMapperTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 54 ++-- .../index/mapper/RangeFieldMapperTests.java | 2 +- .../index/mapper/SourceLoaderTests.java | 29 +-- .../index/mapper/MapperServiceTestCase.java | 52 ++-- .../index/mapper/MapperTestCase.java | 20 +- .../mapper/HistogramFieldMapperTests.java | 5 +- .../accesscontrol/FieldSubsetReaderTests.java | 18 +- ...AggregateDoubleMetricFieldMapperTests.java | 4 +- .../ConstantKeywordFieldMapperTests.java | 4 +- 13 files changed, 226 insertions(+), 212 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index e57a215a0cf26..b4bc2f23af087 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -84,8 +84,8 @@ public void testInvalidDocument_ArrayDocCount() throws Exception { } public void testSyntheticSource() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> {})); - assertThat(syntheticSource(mapperService, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}")); + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> {})); + assertThat(syntheticSource(mapper, b -> b.field(CONTENT_TYPE, 10)), equalTo("{\"_doc_count\":10}")); } public void testSyntheticSourceMany() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index 246ad204be3bb..bac9d4e16d823 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -9,10 +9,8 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -71,13 +69,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder( - null, - Settings.EMPTY, - false, - true, - new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()) - ).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true, null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index ede1f97c7a9eb..efa12d29f0e57 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -26,7 +26,7 @@ public class IgnoredSourceFieldMapperTests extends MapperServiceTestCase { - private MapperService getMapperServiceWithFieldLimit() throws IOException { + private DocumentMapper getDocumentMapperWithFieldLimit() throws IOException { return createMapperService( Settings.builder() .put("index.mapping.total_fields.limit", 2) @@ -36,17 +36,17 @@ private MapperService getMapperServiceWithFieldLimit() throws IOException { b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "object").endObject(); }) - ); + ).documentMapper(); } private ParsedDocument getParsedDocumentWithFieldLimit(CheckedConsumer build) throws IOException { - MapperService mapperService = getMapperServiceWithFieldLimit(); - return mapperService.documentMapper().parse(source(build)); + DocumentMapper mapper = getDocumentMapperWithFieldLimit(); + return mapper.parse(source(build)); } private String getSyntheticSourceWithFieldLimit(CheckedConsumer build) throws IOException { - MapperService mapperService = getMapperServiceWithFieldLimit(); - return syntheticSource(mapperService, build); + DocumentMapper documentMapper = getDocumentMapperWithFieldLimit(); + return syntheticSource(documentMapper, build); } private MapperService createMapperServiceWithStoredArraySource(XContentBuilder mappings) throws IOException { @@ -236,11 +236,11 @@ public void testIgnoredDynamicArrayNestedInObject() throws IOException { public void testDisabledRootObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(topMapping(b -> { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { b.startObject("_source").field("mode", "synthetic").endObject(); b.field("enabled", false); - })); - var syntheticSource = syntheticSource(mapperService, b -> { b.field("name", name); }); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("name", name); }); assertEquals(String.format(Locale.ROOT, """ {"name":"%s"}""", name), syntheticSource); } @@ -250,11 +250,11 @@ public void testDisabledRootObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(topMapping(b -> { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { b.startObject("_source").field("mode", "synthetic").endObject(); b.field("enabled", false); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -292,10 +292,10 @@ public void testDisabledRootObjectManyFields() throws IOException { public void testDisabledObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.field("name", name); @@ -308,10 +308,10 @@ public void testDisabledObjectSingleField() throws IOException { public void testDisabledObjectContainsArray() throws IOException { String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -328,11 +328,11 @@ public void testDisabledObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("enabled", false).endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -372,7 +372,7 @@ public void testDisabledSubObject() throws IOException { boolean booleanValue = randomBoolean(); int intValue = randomInt(); String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -385,8 +385,8 @@ public void testDisabledSubObject() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -404,7 +404,7 @@ public void testDisabledSubObject() throws IOException { } public void testDisabledSubobjectContainsArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -417,11 +417,11 @@ public void testDisabledSubobjectContainsArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -447,7 +447,7 @@ public void testMixedDisabledEnabledObjects() throws IOException { int intValue = randomInt(); String foo = randomAlphaOfLength(20); String bar = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -477,8 +477,8 @@ public void testMixedDisabledEnabledObjects() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -507,11 +507,11 @@ public void testMixedDisabledEnabledObjects() throws IOException { } public void testIndexStoredArraySourceRootValueArray() throws IOException { - MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.array("int_value", new int[] { 30, 20, 10 }); b.field("bool_value", true); }); @@ -520,7 +520,7 @@ public void testIndexStoredArraySourceRootValueArray() throws IOException { } public void testIndexStoredArraySourceRootObjectArray() throws IOException { - MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -532,8 +532,8 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { } b.endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); b.startObject().field("int_value", 10).endObject(); b.startObject().field("int_value", 20).endObject(); @@ -545,7 +545,7 @@ public void testIndexStoredArraySourceRootObjectArray() throws IOException { } public void testIndexStoredArraySourceNestedValueArray() throws IOException { - MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -557,8 +557,8 @@ public void testIndexStoredArraySourceNestedValueArray() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.array("int_value", new int[] { 30, 20, 10 }); @@ -571,7 +571,7 @@ public void testIndexStoredArraySourceNestedValueArray() throws IOException { } public void testIndexStoredArraySourceNestedObjectArray() throws IOException { - MapperService mapperService = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -592,8 +592,8 @@ public void testIndexStoredArraySourceNestedObjectArray() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.startArray("to"); @@ -609,7 +609,7 @@ public void testIndexStoredArraySourceNestedObjectArray() throws IOException { } public void testRootArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -621,8 +621,8 @@ public void testRootArray() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); b.startObject().field("int_value", 10).endObject(); b.startObject().field("int_value", 20).endObject(); @@ -633,7 +633,7 @@ public void testRootArray() throws IOException { } public void testNestedArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -670,10 +670,10 @@ public void testNestedArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -707,7 +707,7 @@ public void testNestedArray() throws IOException { } public void testArrayWithinArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object").field("store_array_source", true); @@ -726,10 +726,10 @@ public void testArrayWithinArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject(); @@ -760,7 +760,7 @@ public void testArrayWithinArray() throws IOException { } public void testDisabledObjectWithinHigherLevelArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -779,10 +779,10 @@ public void testDisabledObjectWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject(); @@ -803,7 +803,7 @@ public void testDisabledObjectWithinHigherLevelArray() throws IOException { } public void testStoredArrayWithinHigherLevelArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -822,10 +822,10 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject(); @@ -856,7 +856,7 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { } public void testFallbackFieldWithinHigherLevelArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { b.field("type", "object"); @@ -867,10 +867,10 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { @@ -886,7 +886,7 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { } public void testFieldOrdering() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("A").field("type", "integer").endObject(); b.startObject("B").field("type", "object").field("store_array_source", true); { @@ -910,9 +910,9 @@ public void testFieldOrdering() throws IOException { } b.endObject(); b.startObject("E").field("type", "integer").endObject(); - })); + })).documentMapper(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("C", 10); b.startArray("D"); { @@ -934,7 +934,7 @@ public void testFieldOrdering() throws IOException { } public void testNestedObjectWithField() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.field("store_array_source", true); @@ -946,9 +946,9 @@ public void testNestedObjectWithField() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); var syntheticSource = syntheticSource( - mapperService, + documentMapper, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -956,7 +956,7 @@ public void testNestedObjectWithField() throws IOException { } public void testNestedObjectWithArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.field("store_array_source", true); @@ -968,8 +968,8 @@ public void testNestedObjectWithArray() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -982,7 +982,7 @@ public void testNestedObjectWithArray() throws IOException { } public void testNestedSubobjectWithField() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1005,11 +1005,11 @@ public void testNestedSubobjectWithField() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1023,7 +1023,7 @@ public void testNestedSubobjectWithField() throws IOException { } public void testNestedSubobjectWithArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1046,11 +1046,11 @@ public void testNestedSubobjectWithArray() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1072,7 +1072,7 @@ public void testNestedSubobjectWithArray() throws IOException { } public void testNestedObjectIncludeInRoot() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested").field("store_array_source", true).field("include_in_root", true); { b.startObject("properties"); @@ -1083,9 +1083,9 @@ public void testNestedObjectIncludeInRoot() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); var syntheticSource = syntheticSource( - mapperService, + documentMapper, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1094,10 +1094,10 @@ public void testNestedObjectIncludeInRoot() throws IOException { public void testNoDynamicObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.field("name", name); @@ -1113,7 +1113,7 @@ public void testNoDynamicObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("dynamic", "false"); { @@ -1124,9 +1124,9 @@ public void testNoDynamicObjectManyFields() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1157,10 +1157,10 @@ public void testNoDynamicObjectManyFields() throws IOException { } public void testNoDynamicObjectSimpleArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("name", "foo").endObject(); @@ -1173,19 +1173,22 @@ public void testNoDynamicObjectSimpleArray() throws IOException { } public void testNoDynamicObjectSimpleValueArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); }); + })).documentMapper(); + var syntheticSource = syntheticSource( + documentMapper, + b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); } + ); assertEquals(""" {"path":{"name":["A","B","C","D"]}}""", syntheticSource); } public void testNoDynamicObjectNestedArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "false").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().startObject("to").field("foo", "A").field("bar", "B").endObject().endObject(); @@ -1199,10 +1202,10 @@ public void testNoDynamicObjectNestedArray() throws IOException { public void testRuntimeDynamicObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.field("name", name); @@ -1218,7 +1221,7 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { int intValue = randomInt(); String stringValue = randomAlphaOfLength(20); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path").field("type", "object").field("dynamic", "runtime"); { @@ -1229,9 +1232,9 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1262,10 +1265,10 @@ public void testRuntimeDynamicObjectManyFields() throws IOException { } public void testRuntimeDynamicObjectSimpleArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("name", "foo").endObject(); @@ -1278,19 +1281,22 @@ public void testRuntimeDynamicObjectSimpleArray() throws IOException { } public void testRuntimeDynamicObjectSimpleValueArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); }); + })).documentMapper(); + var syntheticSource = syntheticSource( + documentMapper, + b -> { b.startObject("path").array("name", "A", "B", "C", "D").endObject(); } + ); assertEquals(""" {"path":{"name":["A","B","C","D"]}}""", syntheticSource); } public void testRuntimeDynamicObjectNestedArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").field("dynamic", "runtime").endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().startObject("to").field("foo", "A").field("bar", "B").endObject().endObject(); @@ -1303,7 +1309,7 @@ public void testRuntimeDynamicObjectNestedArray() throws IOException { } public void testDisabledSubObjectWithNameOverlappingParentName() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); b.startObject("properties"); { @@ -1311,8 +1317,8 @@ public void testDisabledSubObjectWithNameOverlappingParentName() throws IOExcept } b.endObject(); b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.startObject("at").field("foo", "A").endObject(); @@ -1324,7 +1330,7 @@ public void testDisabledSubObjectWithNameOverlappingParentName() throws IOExcept } public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); b.startObject("properties"); { @@ -1332,8 +1338,8 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx } b.endObject(); b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.startObject("at").field("foo", "A").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 5791e171af215..2d2fad23b3831 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -666,7 +666,7 @@ public void testSyntheticSourceForDisabledField() throws Exception { ) ); String value = randomAlphaOfLengthBetween(1, 20); - assertEquals("{\"field\":\"" + value + "\"}", syntheticSource(mapper, b -> b.field("field", value))); + assertEquals("{\"field\":\"" + value + "\"}", syntheticSource(mapper.documentMapper(), b -> b.field("field", value))); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index d8cbc28fae0c7..0a954115e77f6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1584,7 +1584,7 @@ public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOExceptio } public void testSyntheticNestedWithObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1595,9 +1595,9 @@ public void testSyntheticNestedWithObject() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); var syntheticSource = syntheticSource( - mapperService, + documentMapper, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1605,7 +1605,7 @@ public void testSyntheticNestedWithObject() throws IOException { } public void testSyntheticNestedWithArray() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1616,8 +1616,8 @@ public void testSyntheticNestedWithArray() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("foo", "A").field("bar", "B").endObject(); @@ -1630,7 +1630,7 @@ public void testSyntheticNestedWithArray() throws IOException { } public void testSyntheticNestedWithSubObjects() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1652,11 +1652,11 @@ public void testSyntheticNestedWithSubObjects() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1670,7 +1670,7 @@ public void testSyntheticNestedWithSubObjects() throws IOException { } public void testSyntheticNestedWithSubArrays() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("boolean_value").field("type", "boolean").endObject(); b.startObject("path"); { @@ -1692,11 +1692,11 @@ public void testSyntheticNestedWithSubArrays() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); boolean booleanValue = randomBoolean(); int intValue = randomInt(); - var syntheticSource = syntheticSource(mapperService, b -> { + var syntheticSource = syntheticSource(documentMapper, b -> { b.field("boolean_value", booleanValue); b.startObject("path"); { @@ -1718,7 +1718,7 @@ public void testSyntheticNestedWithSubArrays() throws IOException { } public void testSyntheticNestedWithIncludeInRoot() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested").field("include_in_root", true); { b.startObject("properties"); @@ -1729,9 +1729,9 @@ public void testSyntheticNestedWithIncludeInRoot() throws IOException { b.endObject(); } b.endObject(); - })); + })).documentMapper(); var syntheticSource = syntheticSource( - mapperService, + documentMapper, b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } ); assertEquals(""" @@ -1739,7 +1739,7 @@ public void testSyntheticNestedWithIncludeInRoot() throws IOException { } public void testSyntheticNestedWithEmptyObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1749,14 +1749,14 @@ public void testSyntheticNestedWithEmptyObject() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { b.startObject("path").nullField("foo").endObject(); }); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path").nullField("foo").endObject(); }); assertEquals(""" {"path":{}}""", syntheticSource); } public void testSyntheticNestedWithEmptySubObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1770,8 +1770,8 @@ public void testSyntheticNestedWithEmptySubObject() throws IOException { b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startObject("path"); { b.startObject("to").nullField("foo").endObject(); @@ -1783,7 +1783,7 @@ public void testSyntheticNestedWithEmptySubObject() throws IOException { } public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1793,8 +1793,8 @@ public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOExcepti b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().field("foo", "A").endObject(); @@ -1807,7 +1807,7 @@ public void testSyntheticNestedWithArrayContainingEmptyObject() throws IOExcepti } public void testSyntheticNestedWithArrayContainingOnlyEmptyObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { b.startObject("properties"); @@ -1817,8 +1817,8 @@ public void testSyntheticNestedWithArrayContainingOnlyEmptyObject() throws IOExc b.endObject(); } b.endObject(); - })); - var syntheticSource = syntheticSource(mapperService, b -> { + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { b.startObject().nullField("foo").endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index fcddb329efb0c..7a873bf73c5de 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -384,7 +384,7 @@ protected TestRange randomRangeForSyntheticSourceTest() { } protected Source getSourceFor(CheckedConsumer mapping, List inputValues) throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(mapping)); + var mapperService = createMapperService(syntheticSourceMapping(mapping)); CheckedConsumer input = b -> { b.field("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index 02bae022492ca..c6a4021d8a542 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -25,25 +25,25 @@ public void testNonSynthetic() throws IOException { } public void testEmptyObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertTrue(mapperService.documentMapper().mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); - assertThat(syntheticSource(mapperService, b -> b.field("kwd", "foo")), equalTo(""" + assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); + assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); } public void testDotsInFieldName() throws IOException { - MapperService mapperService = createMapperService( + DocumentMapper mapper = createDocumentMapper( syntheticSourceMapping(b -> b.startObject("foo.bar.baz").field("type", "keyword").endObject()) ); - assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo":{"bar":{"baz":"aaa"}}}""")); } public void testNoSubobjectsIntermediateObject() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("foo"); { b.field("type", "object").field("subobjects", false); @@ -55,7 +55,7 @@ public void testNoSubobjectsIntermediateObject() throws IOException { } b.endObject(); })); - assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo":{"bar.baz":"aaa"}}""")); } @@ -67,35 +67,32 @@ public void testNoSubobjectsRootObject() throws IOException { b.startObject("foo.bar.baz").field("type", "keyword").endObject(); b.endObject(); }); - MapperService mapperService = createMapperService(mappings); - assertThat(syntheticSource(mapperService, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" + DocumentMapper mapper = createDocumentMapper(mappings); + assertThat(syntheticSource(mapper, b -> b.field("foo.bar.baz", "aaa")), equalTo(""" {"foo.bar.baz":"aaa"}""")); } public void testSorted() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "keyword").endObject(); b.startObject("baz").field("type", "keyword").endObject(); })); assertThat( - syntheticSource( - mapperService, - b -> b.field("foo", "over the lazy dog").field("bar", "the quick").field("baz", "brown fox jumped") - ), + syntheticSource(mapper, b -> b.field("foo", "over the lazy dog").field("bar", "the quick").field("baz", "brown fox jumped")), equalTo(""" {"bar":"the quick","baz":"brown fox jumped","foo":"over the lazy dog"}""") ); } public void testArraysPushedToLeaves() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("o").startObject("properties"); b.startObject("foo").field("type", "keyword").endObject(); b.startObject("bar").field("type", "keyword").endObject(); b.endObject().endObject(); })); - assertThat(syntheticSource(mapperService, b -> { + assertThat(syntheticSource(mapper, b -> { b.startArray("o"); b.startObject().field("foo", "a").endObject(); b.startObject().field("bar", "b").endObject(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 0e8a2a68d4bb6..f04ccb5ff843e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -803,16 +803,28 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) throws IOException { + return syntheticSource(mapperService.documentMapper(), mapperService.getIndexSettings(), build); + } + + protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { + return syntheticSource(mapper, new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()), build); + } + + protected final String syntheticSource( + DocumentMapper mapper, + IndexSettings indexSettings, + CheckedConsumer build + ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); - ParsedDocument doc = mapperService.documentMapper().parse(source(build)); + ParsedDocument doc = mapper.parse(source(build)); doc.updateSeqID(0, 0); doc.version().setLongValue(0); iw.addDocuments(doc.docs()); iw.close(); try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); - roundTripSyntheticSource(mapperService, syntheticSource, indexReader); + String syntheticSource = syntheticSource(mapper, indexSettings, indexReader, doc.docs().size() - 1); + roundTripSyntheticSource(mapper, indexSettings, syntheticSource, indexReader); return syntheticSource; } } @@ -828,35 +840,45 @@ protected final String syntheticSource(MapperService mapperService, CheckedConsu * That's the point, really. It'll just be "close enough" for * round tripping. */ - private void roundTripSyntheticSource(MapperService mapperService, String syntheticSource, DirectoryReader reader) throws IOException { + private void roundTripSyntheticSource( + DocumentMapper mapper, + IndexSettings indexSettings, + String syntheticSource, + DirectoryReader reader + ) throws IOException { try (Directory roundTripDirectory = newDirectory()) { RandomIndexWriter roundTripIw = indexWriterForSyntheticSource(roundTripDirectory); - ParsedDocument doc = mapperService.documentMapper() - .parse(new SourceToParse("1", new BytesArray(syntheticSource), XContentType.JSON)); + ParsedDocument doc = mapper.parse(new SourceToParse("1", new BytesArray(syntheticSource), XContentType.JSON)); // Process root and nested documents in the same way as the normal indexing chain (assuming a single document) doc.updateSeqID(0, 0); doc.version().setLongValue(0); roundTripIw.addDocuments(doc.docs()); roundTripIw.close(); try (DirectoryReader roundTripReader = wrapInMockESDirectoryReader(DirectoryReader.open(roundTripDirectory))) { - String roundTripSyntheticSource = syntheticSource(mapperService, roundTripReader, doc.docs().size() - 1); + String roundTripSyntheticSource = syntheticSource(mapper, indexSettings, roundTripReader, doc.docs().size() - 1); assertThat(roundTripSyntheticSource, equalTo(syntheticSource)); validateRoundTripReader(syntheticSource, reader, roundTripReader); } } } - protected static String syntheticSource(MapperService mapperService, IndexReader reader, int docId) throws IOException { + protected static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { + return syntheticSource( + mapper, + new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()), + reader, + docId + ); + } + + protected static String syntheticSource(DocumentMapper mapper, IndexSettings indexSettings, IndexReader reader, int docId) + throws IOException { LeafReader leafReader = getOnlyLeafReader(reader); final String synthetic1; final XContent xContent; { - SourceProvider provider = SourceProvider.fromSyntheticSource( - mapperService.documentMapper().mapping(), - mapperService.getIndexSettings(), - SourceFieldMetrics.NOOP - ); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), indexSettings, SourceFieldMetrics.NOOP); var source = provider.getSource(leafReader.getContext(), docId); synthetic1 = source.internalSourceRef().utf8ToString(); xContent = source.sourceContentType().xContent(); @@ -866,8 +888,8 @@ protected static String syntheticSource(MapperService mapperService, IndexReader { int[] docIds = new int[] { docId }; SourceLoader sourceLoader = new SourceLoader.Synthetic( - mapperService.documentMapper().mapping()::syntheticFieldLoader, - mapperService.getIndexSettings(), + mapper.mapping()::syntheticFieldLoader, + indexSettings, SourceFieldMetrics.NOOP ); var sourceLeafLoader = sourceLoader.leaf(getOnlyLeafReader(reader), docIds); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 23c73b2e5a365..e0768b4a66885 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1133,12 +1133,12 @@ public void testSyntheticSourceIgnoreMalformedExamples() throws IOException { } private void assertSyntheticSource(SyntheticSourceExample example) throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); example.mapping().accept(b); b.endObject(); })); - assertThat(syntheticSource(mapperService, example::buildInput), equalTo(example.expected())); + assertThat(syntheticSource(mapper, example::buildInput), equalTo(example.expected())); } protected boolean supportsEmptyInputArray() { @@ -1218,24 +1218,24 @@ public final void testSyntheticSourceMany() throws IOException { public final void testNoSyntheticSourceForScript() throws IOException { // Fetch the ingest script support to eagerly assumeFalse if the mapper doesn't support ingest scripts ingestScriptSupport(); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); minimalMapping(b); b.field("script", randomBoolean() ? "empty" : "non-empty"); b.endObject(); })); - assertThat(syntheticSource(mapperService, b -> {}), equalTo("{}")); + assertThat(syntheticSource(mapper, b -> {}), equalTo("{}")); } public final void testSyntheticSourceInObject() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceExample syntheticSourceExample = syntheticSourceSupport(ignoreMalformed).example(5); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("obj").startObject("properties").startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject().endObject().endObject(); })); - assertThat(syntheticSource(mapperService, b -> { + assertThat(syntheticSource(mapper, b -> { b.startObject("obj"); syntheticSourceExample.buildInput(b); b.endObject(); @@ -1247,14 +1247,14 @@ public final void testSyntheticEmptyList() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceSupport support = syntheticSourceSupport(ignoreMalformed); SyntheticSourceExample syntheticSourceExample = support.example(5); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject(); })); var expected = support.preservesExactSource() ? "{\"field\":[]}" : "{}"; - assertThat(syntheticSource(mapperService, b -> b.startArray("field").endArray()), equalTo(expected)); + assertThat(syntheticSource(mapper, b -> b.startArray("field").endArray()), equalTo(expected)); } private boolean shouldUseIgnoreMalformed() { @@ -1508,12 +1508,12 @@ public final void testSyntheticSourceInvalid() throws IOException { public final void testSyntheticSourceInNestedObject() throws IOException { boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceExample syntheticSourceExample = syntheticSourceSupport(ignoreMalformed).example(5); - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("obj").field("type", "nested").startObject("properties").startObject("field"); syntheticSourceExample.mapping().accept(b); b.endObject().endObject().endObject(); })); - assertThat(syntheticSource(mapperService, b -> { + assertThat(syntheticSource(mapper, b -> { b.startObject("obj"); syntheticSourceExample.buildInput(b); b.endObject(); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 7a859d27dec1b..6fcbf20b8657f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -375,7 +374,7 @@ protected IngestScriptSupport ingestScriptSupport() { } public void testArrayValueSyntheticSource() throws Exception { - MapperService mapperService = createMapperService( + DocumentMapper mapper = createDocumentMapper( syntheticSourceFieldMapping(b -> b.field("type", "histogram").field("ignore_malformed", "true")) ); @@ -402,7 +401,7 @@ public void testArrayValueSyntheticSource() throws Exception { } expected.endObject(); - var syntheticSource = syntheticSource(mapperService, arrayValue); + var syntheticSource = syntheticSource(mapper, arrayValue); assertEquals(Strings.toString(expected), syntheticSource); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 344dd412bd5cd..560dee9b5843c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -67,7 +67,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -718,7 +717,7 @@ public void testSourceFilteringIntegration() throws Exception { } public void testIgnoredSourceFilteringIntegration() throws Exception { - MapperService mapperService = createMapperService( + DocumentMapper mapper = createMapperService( Settings.builder() .put("index.mapping.total_fields.limit", 1) .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) @@ -726,8 +725,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { syntheticSourceMapping(b -> { b.startObject("foo").field("type", "keyword").endObject(); }) - ); - DocumentMapper mapper = mapperService.documentMapper(); + ).documentMapper(); try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); @@ -750,7 +748,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals("{\"fieldA\":\"testA\"}", syntheticSource); } } @@ -767,7 +765,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}],"fieldB":"testB","obj":{"fieldC":"testC"}}""", syntheticSource); } @@ -781,7 +779,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals(""" {"obj":{"fieldC":"testC"}}""", syntheticSource); } @@ -799,7 +797,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}],"fieldA":"testA","fieldB":"testB"}""", syntheticSource); } @@ -813,7 +811,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{"fieldD":"testD"}]}""", syntheticSource); } @@ -831,7 +829,7 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { new CharacterRunAutomaton(automaton) ) ) { - String syntheticSource = syntheticSource(mapperService, indexReader, doc.docs().size() - 1); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); assertEquals(""" {"arr":[{}],"fieldA":"testA","fieldB":"testB","obj":{"fieldC":"testC"}}""", syntheticSource); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 665339f41c6b7..5fbc25eb037a7 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -527,7 +527,7 @@ protected IngestScriptSupport ingestScriptSupport() { } public void testArrayValueSyntheticSource() throws Exception { - MapperService mapperService = createMapperService( + DocumentMapper mapper = createDocumentMapper( syntheticSourceFieldMapping( b -> b.field("type", CONTENT_TYPE) .array("metrics", "min", "max") @@ -559,7 +559,7 @@ public void testArrayValueSyntheticSource() throws Exception { } expected.endObject(); - var syntheticSource = syntheticSource(mapperService, arrayValue); + var syntheticSource = syntheticSource(mapper, arrayValue); assertEquals(Strings.toString(expected), syntheticSource); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index e7bb0c6c9c743..92aac7897bcfd 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -325,12 +325,12 @@ protected Function loadBlockExpected() { } public void testNullValueSyntheticSource() throws IOException { - MapperService mapperService = createMapperService(syntheticSourceMapping(b -> { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); b.field("type", "constant_keyword"); b.endObject(); })); - assertThat(syntheticSource(mapperService, b -> {}), equalTo("{}")); + assertThat(syntheticSource(mapper, b -> {}), equalTo("{}")); } @Override From ccb03c5ea636b9e9ce8083c52296632d48db19a6 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 17 Sep 2024 11:33:00 -0700 Subject: [PATCH 6/7] Simpler approach --- .../index/get/ShardGetService.java | 6 +- .../mapper/IgnoredSourceFieldMapper.java | 67 ++++++++++++++ .../index/mapper/NestedObjectMapper.java | 10 +-- .../index/mapper/SourceFieldMapper.java | 89 +++++++------------ .../index/mapper/SourceLoader.java | 6 +- .../index/query/SearchExecutionContext.java | 8 +- .../search/lookup/SourceProvider.java | 5 +- .../mapper/DynamicFieldsBuilderTests.java | 2 +- ...edSourceFieldMapperConfigurationTests.java | 28 +++++- .../index/mapper/RangeFieldMapperTests.java | 10 +-- .../index/mapper/SourceFieldMetricsTests.java | 7 +- .../query/SearchExecutionContextTests.java | 8 +- .../index/mapper/MapperServiceTestCase.java | 44 ++------- 13 files changed, 150 insertions(+), 140 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 528762883600a..41879c64c3338 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -307,11 +307,7 @@ private GetResult innerGetFetch( Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic( - mappingLookup.getMapping()::syntheticFieldLoader, - indexSettings, - mapperMetrics.sourceFieldMetrics() - ) + ? new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()) : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index eae644db1a34c..59579944172ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Setting; @@ -29,6 +30,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; /** @@ -56,6 +59,9 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); + /* + + */ public static final Setting SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting( "index.mapping.synthetic_source.skip_ignored_source_write", false, @@ -63,6 +69,9 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { Setting.Property.IndexScope ); + /* + + */ public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( "index.mapping.synthetic_source.skip_ignored_source_read", false, @@ -167,6 +176,64 @@ static NameValue decode(Object field) { return new NameValue(name, parentOffset, value, null); } + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + public static Set ensureLoaded(Set fieldsToLoadForSyntheticSource, IndexSettings indexSettings) { + if (indexSettings.getSkipIgnoredSourceRead() == false) { + fieldsToLoadForSyntheticSource.add(NAME); + } + + return fieldsToLoadForSyntheticSource; + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + // This loader controls if this field is loaded in scope of synthetic source constructions. + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() { + @Override + public Stream> storedFieldLoaders() { + if (indexSettings.getSkipIgnoredSourceRead()) { + return Stream.empty(); + } + + // Values are handled in `SourceLoader`. + return Stream.of(Map.entry(NAME, (v) -> {})); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + + } + + @Override + public String fieldName() { + // Does not really matter. + return NAME; + } + + @Override + public void reset() { + + } + }); + } + public record MappedNameValue(NameValue nameValue, XContentType type, Map map) {} /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index b23e6fbf0e369..fc5f28dd51c9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -398,12 +398,10 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return SourceLoader.SyntheticFieldLoader.NOTHING; } - SourceLoader sourceLoader = new SourceLoader.Synthetic( - () -> super.syntheticFieldLoader(mappers.values().stream(), true), - indexSettings, - NOOP - ); - var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()); + SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); + // Some synthetic source use cases require using _ignored_source field + var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings); + var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields); return new NestedSyntheticFieldLoader( storedFieldLoader, sourceLoader, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 583a0be70a903..3318595ed7129 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -23,7 +23,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; @@ -37,7 +36,6 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING; @@ -65,90 +63,81 @@ private enum Mode { SYNTHETIC } - private static final Function DEFAULT = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper DEFAULT = new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, - true, - settings + true ); - private static final Function DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, null, - false, - settings + false ); - private static final Function TSDB_DEFAULT = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper TSDB_DEFAULT = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - true, - settings + true ); - private static final Function TSDB_DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper TSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - false, - settings + false ); - private static final Function LOGSDB_DEFAULT = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper LOGSDB_DEFAULT = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.LOGSDB, - true, - settings + true ); - private static final Function LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper LOGSDB_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( Mode.SYNTHETIC, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.LOGSDB, - false, - settings + false ); /* * Synthetic source was added as the default for TSDB in v.8.7. The legacy field mapper below * is used in bwc tests and mixed clusters containing time series indexes created in an earlier version. */ - private static final Function TSDB_LEGACY_DEFAULT = (settings) -> new SourceFieldMapper( + private static final SourceFieldMapper TSDB_LEGACY_DEFAULT = new SourceFieldMapper( null, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, IndexMode.TIME_SERIES, - true, - settings + true ); - private static final Function TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE = ( - settings) -> new SourceFieldMapper( - null, - Explicit.IMPLICIT_TRUE, - Strings.EMPTY_ARRAY, - Strings.EMPTY_ARRAY, - IndexMode.TIME_SERIES, - false, - settings - ); + private static final SourceFieldMapper TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper( + null, + Explicit.IMPLICIT_TRUE, + Strings.EMPTY_ARRAY, + Strings.EMPTY_ARRAY, + IndexMode.TIME_SERIES, + false + ); public static class Defaults { public static final String NAME = SourceFieldMapper.NAME; @@ -208,21 +197,17 @@ public static class Builder extends MetadataFieldMapper.Builder { private final boolean enableRecoverySource; - private final IndexSettings indexSettings; - public Builder( IndexMode indexMode, final Settings settings, boolean supportsCheckForNonDefaultParams, - boolean enableRecoverySource, - IndexSettings indexSettings + boolean enableRecoverySource ) { super(Defaults.NAME); this.indexMode = indexMode; this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true); this.enableRecoverySource = enableRecoverySource; - this.indexSettings = indexSettings; } public Builder setSynthetic() { @@ -255,12 +240,11 @@ public SourceFieldMapper build() { } } if (isDefault()) { - var factory = switch (indexMode) { + return switch (indexMode) { case TIME_SERIES -> enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; case LOGSDB -> enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; default -> enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; }; - return factory.apply(indexSettings); } if (supportsNonDefaultParameterValues == false) { List disallowed = new ArrayList<>(); @@ -290,8 +274,7 @@ public SourceFieldMapper build() { includes.getValue().toArray(Strings.EMPTY_ARRAY), excludes.getValue().toArray(Strings.EMPTY_ARRAY), indexMode, - enableRecoverySource, - indexSettings + enableRecoverySource ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -307,24 +290,21 @@ public SourceFieldMapper build() { if (indexMode.isSyntheticSourceEnabled()) { if (indexMode == IndexMode.TIME_SERIES) { if (c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.V_8_7_0)) { - return (enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); + return enableRecoverySource ? TSDB_DEFAULT : TSDB_DEFAULT_NO_RECOVERY_SOURCE; } else { - return (enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE).apply( - c.getIndexSettings() - ); + return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE; } } else if (indexMode == IndexMode.LOGSDB) { - return (enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); + return enableRecoverySource ? LOGSDB_DEFAULT : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE; } } - return (enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE).apply(c.getIndexSettings()); + return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE; }, c -> new Builder( c.getIndexSettings().getMode(), c.getSettings(), c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK), - INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()), - c.getIndexSettings() + INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(c.getSettings()) ) ); @@ -378,7 +358,6 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { private final IndexMode indexMode; private final boolean enableRecoverySource; - private final IndexSettings indexSettings; private SourceFieldMapper( Mode mode, @@ -386,8 +365,7 @@ private SourceFieldMapper( String[] includes, String[] excludes, IndexMode indexMode, - boolean enableRecoverySource, - IndexSettings indexSettings + boolean enableRecoverySource ) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; @@ -402,7 +380,6 @@ private SourceFieldMapper( this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; this.enableRecoverySource = enableRecoverySource; - this.indexSettings = indexSettings; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -475,7 +452,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, Settings.EMPTY, false, enableRecoverySource, indexSettings).init(this); + return new Builder(indexMode, Settings.EMPTY, false, enableRecoverySource).init(this); } /** @@ -483,7 +460,7 @@ public FieldMapper.Builder getMergeBuilder() { */ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) { if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, indexSettings, metrics); + return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics); } return SourceLoader.FROM_STORED_SOURCE; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 2c1c3a8a343b3..baff3835d104b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.LeafReader; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentBuilder; @@ -113,15 +112,12 @@ class Synthetic implements SourceLoader { * @param fieldLoaderSupplier A supplier to create {@link SyntheticFieldLoader}, one for each leaf. * @param metrics Metrics for profiling. */ - public Synthetic(Supplier fieldLoaderSupplier, IndexSettings indexSettings, SourceFieldMetrics metrics) { + public Synthetic(Supplier fieldLoaderSupplier, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = fieldLoaderSupplier; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - if (indexSettings.getSkipIgnoredSourceRead() == false) { - this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); - } this.metrics = metrics; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 4a3f16a0579dc..b07112440d3c2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -439,11 +439,7 @@ public boolean isSourceSynthetic() { */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { if (forceSyntheticSource) { - return new SourceLoader.Synthetic( - mappingLookup.getMapping()::syntheticFieldLoader, - indexSettings, - mapperMetrics.sourceFieldMetrics() - ); + return new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()); } return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); } @@ -498,7 +494,7 @@ public boolean containsBrokenAnalysis(String field) { public SearchLookup lookup() { if (this.lookup == null) { SourceProvider sourceProvider = isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), indexSettings, mapperMetrics.sourceFieldMetrics()) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) : SourceProvider.fromStoredFields(); setLookupProviders(sourceProvider, LeafFieldLookupProvider.fromStoredFields()); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 8654e95bb3c30..e232aec5d1f6c 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,7 +10,6 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceFieldMetrics; @@ -49,7 +48,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping, IndexSettings indexSettings, SourceFieldMetrics metrics) { - return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, indexSettings, metrics)); + static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) { + return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java index bac9d4e16d823..358ad07664870 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java @@ -69,7 +69,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti XContentParser parser = createParser(JsonXContent.jsonXContent, source); SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON); - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true, null).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE) ).build(MapperBuilderContext.root(false, false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 5836c975cf7b5..5a101c26310eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -53,9 +53,21 @@ public void testDisableIgnoredSourceRead() throws IOException { // Field was written. assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); - String syntheticSource = syntheticSource(mapperService, inputDocument); + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are not loaded. assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); } public void testDisableIgnoredSourceWrite() throws IOException { @@ -93,9 +105,21 @@ public void testDisableIgnoredSourceWrite() throws IOException { // Field is not written. assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); - String syntheticSource = syntheticSource(mapperService, inputDocument); + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are not loaded. assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); } private MapperService mapperServiceWithCustomSettings( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 7a873bf73c5de..5676c5c92e5a8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -384,7 +384,7 @@ protected TestRange randomRangeForSyntheticSourceTest() { } protected Source getSourceFor(CheckedConsumer mapping, List inputValues) throws IOException { - var mapperService = createMapperService(syntheticSourceMapping(mapping)); + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(mapping)); CheckedConsumer input = b -> { b.field("field"); @@ -401,15 +401,11 @@ protected Source getSourceFor(CheckedConsumer mapp try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = mapperService.documentMapper().parse(source(input)).rootDoc(); + LuceneDocument doc = mapper.parse(source(input)).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource( - mapperService.documentMapper().mapping(), - mapperService.getIndexSettings(), - SourceFieldMetrics.NOOP - ); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index ac605c1ae31d3..81532114a7050 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -37,17 +37,16 @@ public void testFieldHasValueWithEmptyFieldInfos() {} public void testSyntheticSourceLoadLatency() throws IOException { var mapping = syntheticSourceMapping(b -> b.startObject("kwd").field("type", "keyword").endObject()); - var mapperService = createMapperService(mapping); + var mapper = createDocumentMapper(mapping); try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); - LuceneDocument doc = mapperService.documentMapper().parse(source(b -> b.field("kwd", "foo"))).rootDoc(); + LuceneDocument doc = mapper.parse(source(b -> b.field("kwd", "foo"))).rootDoc(); iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { SourceProvider provider = SourceProvider.fromSyntheticSource( - mapperService.documentMapper().mapping(), - mapperService.getIndexSettings(), + mapper.mapping(), createTestMapperMetrics().sourceFieldMetrics() ); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 0b53a73337981..658176b1d31dd 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -384,13 +384,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder( - null, - Settings.EMPTY, - false, - true, - new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()) - ).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, true).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index f04ccb5ff843e..a9ee0317ce1ee 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -801,20 +801,7 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t return new RandomIndexWriter(random(), directory, new StandardAnalyzer()); } - protected final String syntheticSource(MapperService mapperService, CheckedConsumer build) - throws IOException { - return syntheticSource(mapperService.documentMapper(), mapperService.getIndexSettings(), build); - } - protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { - return syntheticSource(mapper, new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()), build); - } - - protected final String syntheticSource( - DocumentMapper mapper, - IndexSettings indexSettings, - CheckedConsumer build - ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); ParsedDocument doc = mapper.parse(source(build)); @@ -823,8 +810,8 @@ protected final String syntheticSource( iw.addDocuments(doc.docs()); iw.close(); try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { - String syntheticSource = syntheticSource(mapper, indexSettings, indexReader, doc.docs().size() - 1); - roundTripSyntheticSource(mapper, indexSettings, syntheticSource, indexReader); + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + roundTripSyntheticSource(mapper, syntheticSource, indexReader); return syntheticSource; } } @@ -840,12 +827,7 @@ protected final String syntheticSource( * That's the point, really. It'll just be "close enough" for * round tripping. */ - private void roundTripSyntheticSource( - DocumentMapper mapper, - IndexSettings indexSettings, - String syntheticSource, - DirectoryReader reader - ) throws IOException { + private void roundTripSyntheticSource(DocumentMapper mapper, String syntheticSource, DirectoryReader reader) throws IOException { try (Directory roundTripDirectory = newDirectory()) { RandomIndexWriter roundTripIw = indexWriterForSyntheticSource(roundTripDirectory); ParsedDocument doc = mapper.parse(new SourceToParse("1", new BytesArray(syntheticSource), XContentType.JSON)); @@ -855,7 +837,7 @@ private void roundTripSyntheticSource( roundTripIw.addDocuments(doc.docs()); roundTripIw.close(); try (DirectoryReader roundTripReader = wrapInMockESDirectoryReader(DirectoryReader.open(roundTripDirectory))) { - String roundTripSyntheticSource = syntheticSource(mapper, indexSettings, roundTripReader, doc.docs().size() - 1); + String roundTripSyntheticSource = syntheticSource(mapper, roundTripReader, doc.docs().size() - 1); assertThat(roundTripSyntheticSource, equalTo(syntheticSource)); validateRoundTripReader(syntheticSource, reader, roundTripReader); } @@ -863,22 +845,12 @@ private void roundTripSyntheticSource( } protected static String syntheticSource(DocumentMapper mapper, IndexReader reader, int docId) throws IOException { - return syntheticSource( - mapper, - new IndexSettings(IndexMetadata.builder("my-index").build(), indexSettings(1, 0).build()), - reader, - docId - ); - } - - protected static String syntheticSource(DocumentMapper mapper, IndexSettings indexSettings, IndexReader reader, int docId) - throws IOException { LeafReader leafReader = getOnlyLeafReader(reader); final String synthetic1; final XContent xContent; { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), indexSettings, SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); var source = provider.getSource(leafReader.getContext(), docId); synthetic1 = source.internalSourceRef().utf8ToString(); xContent = source.sourceContentType().xContent(); @@ -887,11 +859,7 @@ protected static String syntheticSource(DocumentMapper mapper, IndexSettings ind final String synthetic2; { int[] docIds = new int[] { docId }; - SourceLoader sourceLoader = new SourceLoader.Synthetic( - mapper.mapping()::syntheticFieldLoader, - indexSettings, - SourceFieldMetrics.NOOP - ); + SourceLoader sourceLoader = new SourceLoader.Synthetic(mapper.mapping()::syntheticFieldLoader, SourceFieldMetrics.NOOP); var sourceLeafLoader = sourceLoader.leaf(getOnlyLeafReader(reader), docIds); var storedFieldLoader = StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()) .getLoader(leafReader.getContext(), docIds); From d08ad4283e199fbe51c2d57804fd6a5657181c8d Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 17 Sep 2024 12:04:32 -0700 Subject: [PATCH 7/7] style --- .../index/mapper/IgnoredSourceFieldMapper.java | 6 ++++-- .../mapper/IgnoredSourceFieldMapperConfigurationTests.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 59579944172ee..94d862d5cc516 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -60,7 +60,8 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); /* - + Setting to disable encoding and writing values for this field. + This is needed to unblock index functionality in case there is a bug on this code path. */ public static final Setting SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting( "index.mapping.synthetic_source.skip_ignored_source_write", @@ -70,7 +71,8 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { ); /* - + Setting to disable reading and decoding values stored in this field. + This is needed to unblock search functionality in case there is a bug on this code path. */ public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( "index.mapping.synthetic_source.skip_ignored_source_read", diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 5a101c26310eb..e08ace01e88e8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -108,7 +108,7 @@ public void testDisableIgnoredSourceWrite() throws IOException { String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); // Values are not loaded. assertEquals("{}", syntheticSource); - + mapperService.getIndexSettings() .getScopedSettings() .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build());