[Backport 2.x] support termQueryCaseInsensitive/termQuery can search from doc_value in flat_object/keyword field

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
 - Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
 - Introduce framework for auxiliary transports and an experimental gRPC transport plugin ([#16534](https://github.com/opensearch-project/OpenSearch/pull/16534))
+- Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml

@@ -44,6 +44,8 @@ setup:
           {"order":"order7","issue":{"labels":{"number":7,"name":"abc7","status":1}}}
           {"index":{"_index":"flat_object_doc_values_test","_id":"8"}}
           {"order":"order8","issue":{"labels":{"number":8,"name":"abc8","status":1}}}
+          {"index":{"_index":"flat_object_doc_values_test","_id":"9"}}
+          {"order":"order9","issue":{"labels":{"number":9,"name":"abC8","status":1}}}
 
 ---
 # Delete Index when connection is teardown
@@ -67,7 +69,53 @@ teardown:
           }
         }
 
-  - length: { hits.hits: 9 }
+  - length: { hits.hits: 10 }
+
+  # Case Insensitive Term Query with exact dot path.
+  - do:
+      search:
+        body: {
+          _source: true,
+          query: {
+            bool: {
+              must: [
+                {
+                  term: {
+                    issue.labels.name: {
+                      value: "abc8",
+                      case_insensitive: "true"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+
+  - length: { hits.hits: 2 }
+
+  # Case Insensitive Term Query with no path.
+  - do:
+      search:
+        body: {
+          _source: true,
+          query: {
+            bool: {
+              must: [
+                {
+                  term: {
+                    issue.labels: {
+                      value: "abc8",
+                      case_insensitive: "true"
+                    }
+                  }
+                }
+              ]
+            }
+          }
+        }
+
+  - length: { hits.hits: 2 }
 
   # Term Query with exact dot path.
   - do:

rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml

@@ -1126,8 +1126,8 @@ setup:
 "search on fields with only doc_values enabled":
   - skip:
       features: [ "headers" ]
-      version: " - 2.18.99"
-      reason: "searching with only doc_values was finally added in 2.19.0"
+      version: " - 2.99.99"
+      reason: "searching with only doc_values was finally added in 3.0.0"
   - do:
       indices.create:
         index: test-doc-values
@@ -1198,6 +1198,37 @@ setup:
           - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "3" } }'
           - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }'
+          - '{ "index": { "_index": "test-doc-values", "_id": "4" } }'
+          - '{ "some_keyword": "Keyword1" }'
+          - '{ "index": { "_index": "test-doc-values", "_id": "5" } }'
+          - '{ "some_keyword": "keyword1" }'
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            term: {
+              "some_keyword": {
+                "value": "Keyword1"
+              } }
+
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            term: {
+              "some_keyword": {
+                "value": "keyword1",
+                "case_insensitive": "true"
+              } }
+
+  - match: { hits.total: 2 }
 
   - do:
       search:

server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java

@@ -15,7 +15,6 @@
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
@@ -336,23 +335,17 @@ private KeywordFieldType valueFieldType() {
             return (mappedFieldTypeName == null) ? valueFieldType : valueAndPathFieldType;
         }
 
+        @Override
+        public Query termQueryCaseInsensitive(Object value, QueryShardContext context) {
+            return valueFieldType().termQueryCaseInsensitive(rewriteValue(inputToString(value)), context);
+        }
+
         /**
          * redirect queries with rewrite value to rewriteSearchValue and directSubFieldName
          */
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-
-            String searchValueString = inputToString(value);
-            String directSubFieldName = directSubfield();
-            String rewriteSearchValue = rewriteValue(searchValueString);
-
-            failIfNotIndexed();
-            Query query;
-            query = new TermQuery(new Term(directSubFieldName, indexedValueForSearch(rewriteSearchValue)));
-            if (boost() != 1f) {
-                query = new BoostQuery(query, boost());
-            }
-            return query;
+            return valueFieldType().termQuery(rewriteValue(inputToString(value)), context);
         }
 
         @Override

server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java

@@ -39,6 +39,7 @@
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MultiTermQuery;
@@ -391,6 +392,46 @@
             return value;
         }
 
+        @Override
+        public Query termQueryCaseInsensitive(Object value, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            if (isSearchable()) {
+                return super.termQueryCaseInsensitive(value, context);
+            } else {
+                BytesRef bytesRef = indexedValueForSearch(rewriteForDocValue(value));
+                Term term = new Term(name(), bytesRef);
+                Query query = AutomatonQueries.createAutomatonQuery(
+                    term,
+                    AutomatonQueries.toCaseInsensitiveString(bytesRef.utf8ToString(), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT),
+                    MultiTermQuery.DOC_VALUES_REWRITE
+                );
+                if (boost() != 1f) {
+                    query = new BoostQuery(query, boost());
+                }
+                return query;
+            }
+        }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            if (isSearchable()) {
+                return super.termQuery(value, context);
+            } else {
+                Query query = SortedSetDocValuesField.newSlowRangeQuery(
+                    name(),
+                    indexedValueForSearch(rewriteForDocValue(value)),
+                    indexedValueForSearch(rewriteForDocValue(value)),
+                    true,
+                    true
+                );
+                if (boost() != 1f) {
+                    query = new BoostQuery(query, boost());
+                }
+                return query;
+            }
+        }
+
         @Override
         public Query termsQuery(List<?> values, QueryShardContext context) {
             failIfNotIndexedAndNoDocValues();