Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
…into pit2.x
  • Loading branch information
bharath-techie committed Oct 6, 2022
2 parents a4b1c4e + a12feea commit 41fd719
Show file tree
Hide file tree
Showing 69 changed files with 135 additions and 67 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
### Dependencies
- Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
- Bumps `xmlbeans` from 5.1.0 to 5.1.1
- Bumps `hadoop-hdfs` from 3.3.3 to 3.3.4

### Changed
- Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
- Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
- Add index specific setting for remote repository ([#4253](https://github.com/opensearch-project/OpenSearch/pull/4253))
- [Segment Replication] Update replicas to commit SegmentInfos instead of relying on SIS files from primary shards. ([#4402](https://github.com/opensearch-project/OpenSearch/pull/4402))
- Change the version to remove deprecated code of adding node name into log pattern of log4j property file ([#4569](https://github.com/opensearch-project/OpenSearch/pull/4569))
- Update to Apache Lucene 9.4.0 ([#4661](https://github.com/opensearch-project/OpenSearch/pull/4661))

### Deprecated

Expand All @@ -47,6 +49,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Fixed flaky test `ResourceAwareTasksTests.testTaskIdPersistsInThreadContext` ([#4484](https://github.com/opensearch-project/OpenSearch/pull/4484))
- Fixed the ignore_malformed setting to also ignore objects ([#4494](https://github.com/opensearch-project/OpenSearch/pull/4494))
- Fixed the `_cat/shards/10_basic.yml` test cases fix.
- Updated jackson to 2.13.4 and snakeyml to 1.32 ([#4556](https://github.com/opensearch-project/OpenSearch/pull/4556))
- Fixed day of year defaulting for round up parser ([#4627](https://github.com/opensearch-project/OpenSearch/pull/4627))
- Fixed the SnapshotsInProgress error during index deletion ([#4570](https://github.com/opensearch-project/OpenSearch/pull/4570))
- [Bug]: Fixed invalid location of JDK dependency for arm64 architecture([#4613](https://github.com/opensearch-project/OpenSearch/pull/4613))

### Security
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 2.4.0
lucene = 9.3.0
lucene = 9.4.0

bundled_jdk_vendor = adoptium
bundled_jdk = 17.0.4+8
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
19749e264805171009836cbedecc5494b13cd920

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
aa0f250558375922f3091820361156e514fe1842

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
32eb1ad367ab1289804aeed95ea7216711a7764d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
63661714be65f882a921d281965b0779fd487b90

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1034d876551fc21f7835b456dab01db21b9a4af6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f704ee4b14e2fe2622bb983f04b36a32df8fd4a7

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a95ff17b51da6b3da641fa4053e5ee9ea2ff5daf

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
13e1ae2c760d8c0d7990ffe3296e46d9d8e6f842
2 changes: 1 addition & 1 deletion plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ opensearchplugin {
}

versions << [
'hadoop3': '3.3.3'
'hadoop3': '3.3.4'
]

testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs"
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6339a8f7279310c8b1f7ef314b592d8c71ca72ef

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
21f7a9a2da446f1e5b3e5af16ebf956d3ee43ee0

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
036ef2f86dc44410d2bb5d54ce40435d2484d9a5
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-analysis-common-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
02fbd4e87241411fcf5d34e92a50bee46ab164dc
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-backward-codecs-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
259863dfd107645de6146b3c87b4ecee66a4d43d
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-core-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cca1116f813c0f0c63acfac4c952baf29d46d76b
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-grouping-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
51bec1d5acc8ecaf9f50e047d3f86d60c7a958f4
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-highlighter-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c8cf8c9308d8fb18a927c7ed267a14ace3990a5f
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-join-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
99b2d3c8e137a6853a2503456897d47d4f18974b
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-memory-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
881cb214e79da14de35cb0e8e6779d2722828a96
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-misc-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a126123e482e6bf2e7aea670d221a2a39d3277dc
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queries-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fe74dbfe9dba9ee9ee2cb80f151fde97fb4efd12
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-queryparser-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
13f108a8572fcf0670c7df3ba8dbe1076d0e0dbe
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-sandbox-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e7a676a12ea50dcbf64564f4e4022f939f0a627d
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial-extras-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
84d956d1cb1458c51967af1c4acadd2a1f92634d
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-spatial3d-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
76887ca708f23b13613e45fb9e307c548b22c6da
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.3.0.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions server/licenses/lucene-suggest-9.4.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
406c9c539f262449d3b1e57e7bc4302efeecaf6c
Original file line number Diff line number Diff line change
Expand Up @@ -1475,6 +1475,31 @@ public void testSnapshotDeleteRelocatingPrimaryIndex() throws Exception {
logger.info("--> done");
}

public void testIndexDeletionDuringSnapshotCreationInQueue() throws Exception {
assertAcked(prepareCreate("test-idx", 1, indexSettingsNoReplicas(1)));
ensureGreen();
indexRandomDocs("test-idx", 100);
createRepository("test-repo", "fs");
createSnapshot("test-repo", "test-snap", Collections.singletonList("test-idx"));

logger.info("--> create snapshot to be deleted and then delete");
createSnapshot("test-repo", "test-snap-delete", Collections.singletonList("test-idx"));
clusterAdmin().prepareDeleteSnapshot("test-repo", "test-snap-delete").execute();

logger.info("--> create snapshot before index deletion during above snapshot deletion");
clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-2")
.setWaitForCompletion(false)
.setPartial(true)
.setIndices("test-idx")
.get();

logger.info("delete index during snapshot creation");
assertAcked(admin().indices().prepareDelete("test-idx"));

clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap").get();
ensureGreen("test-idx");
}

private long calculateTotalFilesSize(List<Path> files) {
return files.stream().mapToLong(f -> {
try {
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_2_2 = new Version(2020299, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_3_1 = new Version(2030199, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_4_0);
public static final Version CURRENT = V_2_4_0;

public static Version readVersion(StreamInput in) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene92";
public static final String LATEST_CODEC = "Lucene94";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,15 +259,15 @@ public long getFrom(TemporalAccessor temporal) {
static final DateFormatter SECONDS_FORMATTER = new JavaDateFormatter(
"epoch_second",
SECONDS_FORMATTER1,
builder -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L),
(builder, parser) -> builder.parseDefaulting(ChronoField.NANO_OF_SECOND, 999_999_999L),
SECONDS_FORMATTER1,
SECONDS_FORMATTER2
);

static final DateFormatter MILLIS_FORMATTER = new JavaDateFormatter(
"epoch_millis",
MILLISECONDS_FORMATTER1,
builder -> builder.parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L),
(builder, parser) -> builder.parseDefaulting(EpochTime.NANOS_OF_MILLI, 999_999L),
MILLISECONDS_FORMATTER1,
MILLISECONDS_FORMATTER2
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,19 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.function.BiConsumer;
import java.util.stream.Collectors;

class JavaDateFormatter implements DateFormatter {

// base fields which should be used for default parsing, when we round up for date math
private static final Map<TemporalField, Long> ROUND_UP_BASE_FIELDS = new HashMap<>(6);
private static final Map<TemporalField, Long> ROUND_UP_GENERIC_BASE_FIELDS = new HashMap<>(4);

{
ROUND_UP_BASE_FIELDS.put(ChronoField.MONTH_OF_YEAR, 1L);
ROUND_UP_BASE_FIELDS.put(ChronoField.DAY_OF_MONTH, 1L);
ROUND_UP_BASE_FIELDS.put(ChronoField.HOUR_OF_DAY, 23L);
ROUND_UP_BASE_FIELDS.put(ChronoField.MINUTE_OF_HOUR, 59L);
ROUND_UP_BASE_FIELDS.put(ChronoField.SECOND_OF_MINUTE, 59L);
ROUND_UP_BASE_FIELDS.put(ChronoField.NANO_OF_SECOND, 999_999_999L);
ROUND_UP_GENERIC_BASE_FIELDS.put(ChronoField.HOUR_OF_DAY, 23L);
ROUND_UP_GENERIC_BASE_FIELDS.put(ChronoField.MINUTE_OF_HOUR, 59L);
ROUND_UP_GENERIC_BASE_FIELDS.put(ChronoField.SECOND_OF_MINUTE, 59L);
ROUND_UP_GENERIC_BASE_FIELDS.put(ChronoField.NANO_OF_SECOND, 999_999_999L);
}

private final String format;
Expand Down Expand Up @@ -96,14 +94,25 @@ JavaDateFormatter getRoundupParser() {

// named formatters use default roundUpParser
JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
this(format, printer, builder -> ROUND_UP_BASE_FIELDS.forEach(builder::parseDefaulting), parsers);
this(format, printer, ROUND_UP_BASE_FIELDS, parsers);
}

private static final BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> ROUND_UP_BASE_FIELDS = (builder, parser) -> {
String parserString = parser.toString();
if (parserString.contains(ChronoField.DAY_OF_YEAR.toString())) {
builder.parseDefaulting(ChronoField.DAY_OF_YEAR, 1L);
} else {
builder.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1L);
builder.parseDefaulting(ChronoField.DAY_OF_MONTH, 1L);
}
ROUND_UP_GENERIC_BASE_FIELDS.forEach(builder::parseDefaulting);
};

// subclasses override roundUpParser
JavaDateFormatter(
String format,
DateTimeFormatter printer,
Consumer<DateTimeFormatterBuilder> roundupParserConsumer,
BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
DateTimeFormatter... parsers
) {
if (printer == null) {
Expand Down Expand Up @@ -138,13 +147,16 @@ JavaDateFormatter getRoundupParser() {
* <code>DateFormatters</code>.
* This means that we need to also have multiple RoundUp parsers.
*/
private List<DateTimeFormatter> createRoundUpParser(String format, Consumer<DateTimeFormatterBuilder> roundupParserConsumer) {
private List<DateTimeFormatter> createRoundUpParser(
String format,
BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer
) {
if (format.contains("||") == false) {
List<DateTimeFormatter> roundUpParsers = new ArrayList<>();
for (DateTimeFormatter parser : this.parsers) {
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
builder.append(parser);
roundupParserConsumer.accept(builder);
roundupParserConsumer.accept(builder, parser);
roundUpParsers.add(builder.toFormatter(locale()));
}
return roundUpParsers;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.mapper.MapperService;
Expand All @@ -62,8 +62,8 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService, Logger logger) {
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene92Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene94Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene92.Lucene92Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.mapper.CompletionFieldMapper;
Expand All @@ -53,7 +53,7 @@
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene92Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene94Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
Expand Down
Loading

0 comments on commit 41fd719

Please sign in to comment.