Merge branch '2.x' into agg

Signed-off-by: Heemin Kim <[email protected]>

CHANGELOG.md

@@ -18,7 +18,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
   - Addition of Doc values on the GeoShape Field
   - Addition of GeoShape ValueSource level code interfaces for accessing the DocValues.
   - Addition of Missing Value feature in the GeoShape Aggregations.
+- Enable Point based optimization for custom comparators ([#8168](https://github.com/opensearch-project/OpenSearch/pull/8168))
 - Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
+
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)
 - Bump `com.google.guava:guava` from 30.1.1-jre to 32.0.0-jre (#7565, #7811, #7807, #7808)
@@ -49,6 +51,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256)
 - [Upgrade] Lucene 9.7.0 release (#8272)
 - Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209))
+- Bump `com.google.api-client:google-api-client` from 1.34.0 to 2.2.0 ([#8276](https://github.com/opensearch-project/OpenSearch/pull/8276))
+- Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
+- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
+- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
@@ -68,6 +74,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Refactor] Metadata members from ImmutableOpenMap to j.u.Map ([#7165](https://github.com/opensearch-project/OpenSearch/pull/7165))
 - [Refactor] more ImmutableOpenMap to jdk Map in cluster package ([#7301](https://github.com/opensearch-project/OpenSearch/pull/7301))
 - [Refactor] ImmutableOpenMap to j.u.Map in IndexMetadata ([#7306](https://github.com/opensearch-project/OpenSearch/pull/7306))
+- Check UTF16 string size before converting to String to avoid OOME ([#7963](https://github.com/opensearch-project/OpenSearch/pull/7963))
 
 ### Deprecated
 
@@ -79,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Enforce 512 byte document ID limit in bulk updates ([#8039](https://github.com/opensearch-project/OpenSearch/pull/8039))
 - With only GlobalAggregation in request causes unnecessary wrapping with MultiCollector ([#8125](https://github.com/opensearch-project/OpenSearch/pull/8125))
 - Fix mapping char_filter when mapping a hashtag ([#7591](https://github.com/opensearch-project/OpenSearch/pull/7591))
+- Fix NPE in multiterms aggregations involving empty buckets ([#7318](https://github.com/opensearch-project/OpenSearch/pull/7318))
 
 ### Security
 

buildSrc/version.properties

@@ -44,7 +44,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.70
+bouncycastle=1.75
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2

plugins/identity-shiro/build.gradle

@@ -28,7 +28,7 @@ dependencies {
 
   implementation 'org.passay:passay:1.6.3'
 
-  implementation "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
+  implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
 
   testImplementation project(path: ':modules:transport-netty4') // for http
   testImplementation project(path: ':plugins:transport-nio') // for http

plugins/identity-shiro/licenses/bcprov-jdk15to18-1.75.jar.sha1

@@ -0,0 +1 @@
+df22e1b6a9f6b218913f5b68dd16641344397fe0

plugins/ingest-attachment/build.gradle

@@ -71,9 +71,9 @@ dependencies {
   api "org.apache.pdfbox:fontbox:${versions.pdfbox}"
   api "org.apache.pdfbox:jempbox:1.8.17"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}"
-  api "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
-  api "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcmail-jdk15to18:${versions.bouncycastle}"
+  api "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
+  api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
   // OpenOffice
   api "org.apache.poi:poi-ooxml:${versions.poi}"
   api "org.apache.poi:poi:${versions.poi}"

plugins/ingest-attachment/licenses/bcmail-jdk15to18-1.75.jar.sha1

@@ -0,0 +1 @@
+b316bcd094e3917b1ece93a6edbab93f8315fb3b

plugins/ingest-attachment/licenses/bcpkix-jdk15to18-1.75.jar.sha1

@@ -0,0 +1 @@
+f16e5252ad7a46d5eaf255231b0a5da307599082

plugins/ingest-attachment/licenses/bcprov-jdk15to18-1.75.jar.sha1

@@ -0,0 +1 @@
+df22e1b6a9f6b218913f5b68dd16641344397fe0

plugins/ingest-attachment/src/main/plugin-metadata/plugin-security.policy

@@ -35,9 +35,6 @@ grant {
   // needed to apply additional sandboxing to tika parsing
   permission java.security.SecurityPermission "createAccessControlContext";
 
-  // TODO: fix PDFBox not to actually install bouncy castle like this
-  permission java.security.SecurityPermission "putProviderProperty.BC";
-  permission java.security.SecurityPermission "insertProvider";
   // TODO: fix POI XWPF to not do this: https://bz.apache.org/bugzilla/show_bug.cgi?id=58597
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed by xmlbeans, as part of POI for MS xml docs

plugins/repository-gcs/build.gradle

@@ -54,33 +54,43 @@ versions << [
 ]
 
 dependencies {
-  api 'com.google.cloud:google-cloud-storage:1.113.1'
-  api 'com.google.cloud:google-cloud-core:2.5.10'
-  runtimeOnly 'com.google.guava:guava:32.0.1-jre'
-  api 'com.google.guava:failureaccess:1.0.1'
-  api 'com.google.http-client:google-http-client:1.35.0'
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'com.google.api:api-common:1.8.1'
   api 'com.google.api:gax:2.27.0'
-  api 'org.threeten:threetenbp:1.4.4'
-  api 'com.google.code.gson:gson:2.9.0'
+  api 'com.google.api:gax-httpjson:0.103.1'
+
+  api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0'
+
+  api 'com.google.api-client:google-api-client:2.2.0'
+
   api 'com.google.api.grpc:proto-google-common-protos:2.10.0'
   api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
-  api 'com.google.cloud:google-cloud-core-http:2.17.0'
+
   api "com.google.auth:google-auth-library-credentials:${versions.google_auth}"
   api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
-  api 'com.google.oauth-client:google-oauth-client:1.33.3'
-  api 'com.google.api-client:google-api-client:1.34.0'
-  api 'com.google.http-client:google-http-client-appengine:1.41.8'
-  api 'com.google.http-client:google-http-client-jackson2:1.42.3'
-  api 'com.google.http-client:google-http-client-gson:1.41.4'
-  api 'com.google.api:gax-httpjson:0.62.0'
-  api 'io.grpc:grpc-context:1.45.1'
-  api 'io.opencensus:opencensus-api:0.18.0'
-  api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
-  api 'com.google.apis:google-api-services-storage:v1-rev20200814-1.30.10'
+
+  api 'com.google.cloud:google-cloud-core:2.5.10'
+  api 'com.google.cloud:google-cloud-core-http:2.17.0'
+  api 'com.google.cloud:google-cloud-storage:1.113.1'
+
+  api 'com.google.code.gson:gson:2.9.0'
+
+  runtimeOnly "com.google.guava:guava:${versions.guava}"
+  api 'com.google.guava:failureaccess:1.0.1'
+
+  api 'com.google.http-client:google-http-client:1.43.2'
+  api 'com.google.http-client:google-http-client-appengine:1.43.2'
+  api 'com.google.http-client:google-http-client-gson:1.43.2'
+  api 'com.google.http-client:google-http-client-jackson2:1.43.2'
+
+  api 'com.google.oauth-client:google-oauth-client:1.34.1'
+
+  api "commons-logging:commons-logging:${versions.commonslogging}"
+  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  api "commons-codec:commons-codec:${versions.commonscodec}"
+  api 'org.threeten:threetenbp:1.4.4'
+  api 'io.grpc:grpc-context:1.46.0'
+  api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
 
   testImplementation project(':test:fixtures:gcs-fixture')
 }
@@ -121,7 +131,6 @@ thirdPartyAudit {
 
   ignoreMissingClasses(
     'com.google.api.client.http.apache.v2.ApacheHttpTransport',
-    'com.google.api.gax.tracing.NoopApiTracer',
     'com.google.appengine.api.datastore.Blob',
     'com.google.appengine.api.datastore.DatastoreService',
     'com.google.appengine.api.datastore.DatastoreServiceFactory',
@@ -141,6 +150,9 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
+    'com.google.protobuf.util.JsonFormat',
+    'com.google.protobuf.util.JsonFormat$Parser',
+    'com.google.protobuf.util.JsonFormat$Printer',
     'com.google.protobuf.util.Timestamps',
     // commons-logging optional dependencies
     'org.apache.avalon.framework.logger.Logger',
@@ -167,8 +179,6 @@ thirdPartyAudit {
     'org.apache.http.client.methods.HttpRequestBase',
     'org.apache.http.config.Registry',
     'org.apache.http.config.RegistryBuilder',
-    'org.apache.http.config.SocketConfig',
-    'org.apache.http.config.SocketConfig$Builder',
     'org.apache.http.conn.ClientConnectionManager',
     'org.apache.http.conn.ConnectionKeepAliveStrategy',
     'org.apache.http.conn.params.ConnManagerParams',

plugins/repository-gcs/licenses/gax-httpjson-0.103.1.jar.sha1

@@ -0,0 +1 @@
+041d99172fda933bc879bdfd8de9420c5c34107e

plugins/repository-gcs/licenses/google-api-client-2.2.0.jar.sha1

@@ -0,0 +1 @@
+10e53fd4d987e37190432e896bdaa62e8ea2c628

plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20230617-2.0.0.jar.sha1

@@ -0,0 +1 @@
+fc3f225b405303fe7cb760d578348b6b07e7ea8b

plugins/repository-gcs/licenses/google-http-client-1.43.2.jar.sha1

@@ -0,0 +1 @@
+2520469ebd8c0675f0d2aeafd2da665228320fcf

plugins/repository-gcs/licenses/google-http-client-appengine-1.43.2.jar.sha1

@@ -0,0 +1 @@
+9fb548c5264227813fd83991b94a705b0841c15f

plugins/repository-gcs/licenses/google-http-client-gson-1.43.2.jar.sha1

@@ -0,0 +1 @@
+b1c2e3e89804c113dba7b342aa8e0fc2cf3d9378

plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.2.jar.sha1

@@ -0,0 +1 @@
+5e52a9967ebd8246fc4cca64df5f03608db5ac6e

plugins/repository-gcs/licenses/google-oauth-client-1.34.1.jar.sha1

@@ -0,0 +1 @@
+4a4f88c5e13143f882268c98239fb85c3b2c6cb2

plugins/repository-gcs/licenses/grpc-context-1.46.0.jar.sha1

@@ -0,0 +1 @@
+621d765a459758cdcd9aa4b4795392a4353e3576

plugins/repository-gcs/licenses/opencensus-api-0.31.1.jar.sha1

@@ -0,0 +1 @@
+66a60c7201c2b8b20ce495f0295b32bb0ccbbc57

plugins/repository-gcs/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1

@@ -0,0 +1 @@
+3c13fc5715231fadb16a9b74a44d9d59c460cfa8

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml

@@ -712,3 +712,51 @@ setup:
   - match: { aggregations.m_terms.buckets.0.key: ["a", 1] }
   - match: { aggregations.m_terms.buckets.0.key_as_string: "a|1" }
   - match: { aggregations.m_terms.buckets.0.doc_count: 4 }
+
+---
+"aggregate over multi-terms test":
+  - skip:
+      version: "- 2.8.99"
+      reason: "multi_terms aggregation was introduced in 2.1.0, NPE bug checked by this test case will manifest in any version < 3.0"
+
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"str": "a", "ip": "127.0.0.1", "date": "2022-03-23"}'
+          - '{"index": {}}'
+          - '{"str": "a", "ip": "127.0.0.1", "date": "2022-03-25"}'
+          - '{"index": {}}'
+          - '{"str": "b", "ip": "127.0.0.1", "date": "2022-03-23"}'
+          - '{"index": {}}'
+          - '{"str": "b", "ip": "127.0.0.1", "date": "2022-03-25"}'
+
+  - do:
+      search:
+        index: test_1
+        size: 0
+        body:
+          aggs:
+            histo:
+              date_histogram:
+                field: date
+                calendar_interval: day
+              aggs:
+                m_terms:
+                  multi_terms:
+                    terms:
+                      - field: str
+                      - field: ip
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 3 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2022-03-23T00:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.m_terms.buckets.0.key: ["a", "127.0.0.1"] }
+  - match: { aggregations.histo.buckets.0.m_terms.buckets.1.key: ["b", "127.0.0.1"] }
+  - match: { aggregations.histo.buckets.1.key_as_string: "2022-03-24T00:00:00.000Z" }
+  - length: { aggregations.histo.buckets.1.m_terms.buckets: 0 }
+  - match: { aggregations.histo.buckets.2.key_as_string: "2022-03-25T00:00:00.000Z" }
+  - match: { aggregations.histo.buckets.2.m_terms.buckets.0.key: [ "a", "127.0.0.1" ] }
+  - match: { aggregations.histo.buckets.2.m_terms.buckets.1.key: [ "b", "127.0.0.1" ] }

server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java

@@ -63,4 +63,10 @@ public void setup() {
     public void teardown() {
         assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME));
     }
+
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7592")
+    @Override
+    public void testPressureServiceStats() throws Exception {
+        super.testPressureServiceStats();
+    }
 }

server/src/main/java/org/opensearch/common/bytes/AbstractBytesReference.java

@@ -33,6 +33,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.UnicodeUtil;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.core.xcontent.XContentBuilder;
 
@@ -49,6 +50,7 @@
 public abstract class AbstractBytesReference implements BytesReference {
 
     private Integer hash = null; // we cache the hash of this reference since it can be quite costly to re-calculated it
+    private static final int MAX_UTF16_LENGTH = Integer.MAX_VALUE >> 1;
 
     @Override
     public int getInt(int index) {
@@ -80,9 +82,19 @@ public void writeTo(OutputStream os) throws IOException {
         }
     }
 
+    protected int getMaxUTF16Length() {
+        return MAX_UTF16_LENGTH;
+    }
+
     @Override
     public String utf8ToString() {
-        return toBytesRef().utf8ToString();
+        BytesRef bytesRef = toBytesRef();
+        final char[] ref = new char[bytesRef.length];
+        final int len = UnicodeUtil.UTF8toUTF16(bytesRef, ref);
+        if (len > getMaxUTF16Length()) {
+            throw new IllegalArgumentException("UTF16 String size is " + len + ", should be less than " + getMaxUTF16Length());
+        }
+        return new String(ref, 0, len);
     }
 
     @Override

server/src/main/java/org/opensearch/index/IndexService.java

@@ -603,7 +603,10 @@ public synchronized void removeShard(int shardId, String reason) {
     private void closeShard(String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) {
         final int shardId = sId.id();
         final Settings indexSettings = this.getIndexSettings().getSettings();
-        Store remoteStore = indexShard.remoteStore();
+        Store remoteStore = null;
+        if (indexShard != null) {
+            remoteStore = indexShard.remoteStore();
+        }
         if (store != null) {
             store.beforeClose();
         }

server/src/main/java/org/opensearch/index/fielddata/IndexFieldData.java

@@ -120,11 +120,13 @@ abstract class XFieldComparatorSource extends FieldComparatorSource {
         protected final MultiValueMode sortMode;
         protected final Object missingValue;
         protected final Nested nested;
+        protected boolean enableSkipping;
 
         public XFieldComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
             this.sortMode = sortMode;
             this.missingValue = missingValue;
             this.nested = nested;
+            this.enableSkipping = true; // true by default
         }
 
         public MultiValueMode sortMode() {
@@ -135,6 +137,10 @@ public Nested nested() {
             return this.nested;
         }
 
+        public void disableSkipping() {
+            this.enableSkipping = false;
+        }
+
         /**
          * Simple wrapper class around a filter that matches parent documents
          * and a filter that matches child documents. For every root document R,