Skip to content

Commit

Permalink
Upgrade to lucene snapshot 978eef5459c (elastic#86852)
Browse files Browse the repository at this point in the history
Final (hopefully!) snapshot before the 9.2.0 release

* Update test to expect persian tokenfilter - will be exposed later
* Fix KnnVectorQueryBuilderTests::doAssertLuceneQuery

Co-authored-by: Mayya Sharipova <[email protected]>
  • Loading branch information
romseygeek and mayya-sharipova authored May 17, 2022
1 parent b7f4934 commit 0418e8a
Show file tree
Hide file tree
Showing 59 changed files with 74 additions and 61 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats

org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
org.junit.Ignore @ Use AwaitsFix instead
org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
Expand Down
2 changes: 1 addition & 1 deletion build-tools-internal/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 8.3.0
lucene = 9.2.0-snapshot-efa5d6f4d43
lucene = 9.2.0-snapshot-978eef5459c

bundled_jdk_vendor = openjdk
bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
Expand Down
5 changes: 5 additions & 0 deletions docs/changelog/86852.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 86852
summary: Upgrade to lucene snapshot 978eef5459c
area: Search
type: upgrade
issues: []
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f19903613908590bb237683064c82a4834e64ba5

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
050d713d9dc35226d040c580d48a157c21953e6f

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
071d4e394dc5b9135e815054710466673092c725

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3b3084be8519720c80c910062219370934673169

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d46b121b3da6d87e37470e462b3acf000cc7e8fc

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b649f34cb35d2007f9460ab808493a0908698e18

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
80cf1edcd52219348441fde9b6535577ff6b5055

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
34549e8867baa25bbc5c88160bbe1490520496ff

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7dac3ff3c24683c8fc057a930ed5cd346b33b587

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
54a32bd0b8410fa1793b5fb9aa37070a50d34406

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1f9e1809056e553590f82d2fab240516dba163a9

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2a5725388d220bb62fbfe24c0bd42e00142dfb4a

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a059178643306db25088c02bce8c4b6ff51c157a

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
38c3310e99d31e621b431fe48bda5489055ee5c2

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
95fe8d6803fb8caf7097b50a6e15fd823600aa49

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ed27757832d5188a0a7fed5646cb670cffc3e526

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b851d0d5123d6f61ccbf0cfd9274b174baf6f670

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5eb5de43f515f29d9c15e3dead7278d4edb60890

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e7ade1e53845c76917a523dde6038c82e4b50e26

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8fc04d2909ca3649d34a9b4e75c02370f5414bdf

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
36e70112545743795e81c507ccf9d2214621f9ea

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4b46de9ca199c319a9807c63a728776f96dfc22f

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
import java.util.Map;

public class Lucene {
public static final String LATEST_CODEC = "Lucene91";
public static final String LATEST_CODEC = "Lucene92";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
package org.elasticsearch.index.codec;

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.mapper.MapperService;

Expand All @@ -34,11 +34,11 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService) {
final var codecs = new HashMap<String, Codec>();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene91Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene91Codec(Lucene91Codec.Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Lucene92Codec.Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene91Codec.Mode.BEST_SPEED, mapperService));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene91Codec.Mode.BEST_COMPRESSION, mapperService));
codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_SPEED, mapperService));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene92Codec.Mode.BEST_COMPRESSION, mapperService));
}
codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
for (String codec : Codec.availableCodecs()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.MapperService;

Expand All @@ -25,7 +25,7 @@
* per index in real time via the mapping API. If no specific postings format or vector format is
* configured for a specific field the default postings or vector format is used.
*/
public class PerFieldMapperCodec extends Lucene91Codec {
public class PerFieldMapperCodec extends Lucene92Codec {
private final MapperService mapperService;

private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
Expand Down Expand Up @@ -266,7 +266,7 @@ public void testTriangle() throws Exception {
public void testCompletionField() throws Exception {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(false)
.setCodec(new Lucene91Codec(Lucene91Codec.Mode.BEST_SPEED) {
.setCodec(new Lucene92Codec(Lucene92Codec.Mode.BEST_SPEED) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.startsWith("suggest_")) {
Expand Down Expand Up @@ -353,25 +353,25 @@ private static void addFieldsToDoc(Document doc, IndexableField[] fields) {
enum CodecMode {
BEST_SPEED {
@Override
Lucene91Codec.Mode mode() {
return Lucene91Codec.Mode.BEST_SPEED;
Lucene92Codec.Mode mode() {
return Lucene92Codec.Mode.BEST_SPEED;
}
},

BEST_COMPRESSION {
@Override
Lucene91Codec.Mode mode() {
return Lucene91Codec.Mode.BEST_COMPRESSION;
Lucene92Codec.Mode mode() {
return Lucene92Codec.Mode.BEST_COMPRESSION;
}
};

abstract Lucene91Codec.Mode mode();
abstract Lucene92Codec.Mode mode();
}

static void indexRandomly(Directory directory, CodecMode codecMode, int numDocs, Consumer<Document> addFields) throws IOException {
IndexWriterConfig config = new IndexWriterConfig().setCommitOnClose(true)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene91Codec(codecMode.mode()));
.setCodec(new Lucene92Codec(codecMode.mode()));
try (IndexWriter writer = new IndexWriter(directory, config)) {
for (int i = 0; i < numDocs; i++) {
final Document doc = new Document();
Expand Down Expand Up @@ -549,7 +549,7 @@ static void rewriteIndexWithPerFieldCodec(Directory source, CodecMode mode, Dire
try (DirectoryReader reader = DirectoryReader.open(source)) {
IndexWriterConfig config = new IndexWriterConfig().setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
.setUseCompoundFile(randomBoolean())
.setCodec(new Lucene91Codec(mode.mode()) {
.setCodec(new Lucene92Codec(mode.mode()) {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return new Lucene90PostingsFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
Expand Down Expand Up @@ -41,21 +41,21 @@ public class CodecTests extends ESTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene91Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class));
}

public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec);
}

public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene91Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec);
}

// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
Expand All @@ -67,7 +67,7 @@ private void assertStoredFieldsCompressionEquals(Lucene91Codec.Mode expected, Co
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene91Codec.Mode.valueOf(v));
assertEquals(expected, Lucene92Codec.Mode.valueOf(v));
ir.close();
dir.close();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
package org.elasticsearch.index.engine;

import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
Expand Down Expand Up @@ -45,7 +45,7 @@ public void testExceptionsAreNotCached() {
public void testCompletionStatsCache() throws IOException, InterruptedException {
final IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
final PostingsFormat postingsFormat = new Completion90PostingsFormat();
indexWriterConfig.setCodec(new Lucene91Codec() {
indexWriterConfig.setCodec(new Lucene92Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormat; // all fields are suggest fields
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,8 @@ public abstract class AnalysisFactoryTestCase extends ESTestCase {
// LUCENE-10248
entry("spanishpluralstem", Void.class),
// LUCENE-10352
entry("daitchmokotoffsoundex", Void.class)
entry("daitchmokotoffsoundex", Void.class),
entry("persianstem", Void.class)
);

static final Map<String, Class<?>> KNOWN_CHARFILTERS = Map.of(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
071d4e394dc5b9135e815054710466673092c725

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
package org.elasticsearch.xpack.vectors.mapper;

import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnVectorField;
Expand Down Expand Up @@ -528,7 +528,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField() {
return null; // use default format
} else {
HnswIndexOptions hnswIndexOptions = (HnswIndexOptions) indexOptions;
return new Lucene91HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
return new Lucene92HnswVectorsFormat(hnswIndexOptions.m, hnswIndexOptions.efConstruction);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.KnnVectorField;
import org.apache.lucene.index.IndexableField;
Expand Down Expand Up @@ -42,8 +42,8 @@
import java.util.Collection;
import java.util.List;

import static org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
Expand Down Expand Up @@ -469,8 +469,8 @@ public void testKnnVectorsFormat() throws IOException {
Codec codec = codecService.codec("default");
assertThat(codec, instanceOf(PerFieldMapperCodec.class));
KnnVectorsFormat knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field");
assertThat(knnVectorsFormat, instanceOf(Lucene91HnswVectorsFormat.class));
String expectedString = "Lucene91HnswVectorsFormat(name = Lucene91HnswVectorsFormat, maxConn = "
assertThat(knnVectorsFormat, instanceOf(Lucene92HnswVectorsFormat.class));
String expectedString = "lucene92HnswVectorsFormat(name = lucene92HnswVectorsFormat, maxConn = "
+ m
+ ", beamWidth="
+ efConstruction
Expand Down
Loading

0 comments on commit 0418e8a

Please sign in to comment.