Make hits.total an object in the search response

This commit changes the format of the `hits.total` in the search response to be an object with
a `value` and a `relation`. The `value` indicates the number of hits that match the query and the
`relation` indicates whether the number is accurate (in which case the relation is equals to `eq`)
or a lower bound of the total (in which case it is equals to `gte`).
This change also adds a parameter called `rest_total_hits_as_int` that can be used in the
search APIs to opt out from this change (retrieve the total hits as a number in the rest response).
Note that currently all search responses are accurate (`track_total_hits: true`) or they don't contain
`hits.total` (`track_total_hits: true`). We'll add a way to get a lower bound of the total hits in a
follow up (to allow numbers to be passed to `track_total_hits`).

Relates elastic#33028

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -402,8 +402,7 @@ private SearchHits getHits(ReducedQueryPhase reducedQueryPhase, boolean ignoreFr
                 hits.add(searchHit);
             }
         }
-        return new SearchHits(hits.toArray(new SearchHit[hits.size()]), reducedQueryPhase.totalHits,
-            reducedQueryPhase.maxScore);
+        return new SearchHits(hits.toArray(new SearchHit[hits.size()]), reducedQueryPhase.totalHits, reducedQueryPhase.maxScore);
     }
 
     /**
@@ -443,7 +442,8 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
         boolean timedOut = false;
         Boolean terminatedEarly = null;
         if (queryResults.isEmpty()) { // early terminate we have nothing to reduce
-            return new ReducedQueryPhase(topDocsStats.totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
+            final TotalHits totalHits = topDocsStats.getTotalHits();
+            return new ReducedQueryPhase(totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
                 timedOut, terminatedEarly, null, null, null, EMPTY_DOCS, null,
                 null, numReducePhases, false, 0, 0, true);
         }
@@ -508,7 +508,8 @@ private ReducedQueryPhase reducedQueryPhase(Collection<? extends SearchPhaseResu
             firstResult.pipelineAggregators(), reduceContext);
         final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
         final SortedTopDocs scoreDocs = this.sortDocs(isScrollRequest, queryResults, bufferedTopDocs, topDocsStats, from, size);
-        return new ReducedQueryPhase(topDocsStats.totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
+        final TotalHits totalHits = topDocsStats.getTotalHits();
+        return new ReducedQueryPhase(totalHits, topDocsStats.fetchHits, topDocsStats.maxScore,
             timedOut, terminatedEarly, suggest, aggregations, shardResults, scoreDocs.scoreDocs, scoreDocs.sortFields,
             firstResult != null ? firstResult.sortValueFormats() : null,
             numReducePhases, scoreDocs.isSortedByField, size, from, firstResult == null);
@@ -543,7 +544,7 @@ private InternalAggregations reduceAggs(List<InternalAggregations> aggregationsL
 
     public static final class ReducedQueryPhase {
         // the sum of all hits across all reduces shards
-        final long totalHits;
+        final TotalHits totalHits;
         // the number of returned hits (doc IDs) across all reduces shards
         final long fetchHits;
         // the max score across all reduces hits or {@link Float#NaN} if no hits returned
@@ -575,7 +576,7 @@ public static final class ReducedQueryPhase {
         // sort value formats used to sort / format the result
         final DocValueFormat[] sortValueFormats;
 
-        ReducedQueryPhase(long totalHits, long fetchHits, float maxScore, boolean timedOut, Boolean terminatedEarly, Suggest suggest,
+        ReducedQueryPhase(TotalHits totalHits, long fetchHits, float maxScore, boolean timedOut, Boolean terminatedEarly, Suggest suggest,
                           InternalAggregations aggregations, SearchProfileShardResults shardResults, ScoreDoc[] scoreDocs,
                           SortField[] sortFields, DocValueFormat[] sortValueFormats, int numReducePhases, boolean isSortedByField, int size,
                           int from, boolean isEmptyResult) {
@@ -748,8 +749,8 @@ public ReducedQueryPhase reduce() {
 
     static final class TopDocsStats {
         final boolean trackTotalHits;
-        long totalHits;
-        TotalHits.Relation totalHitsRelation = TotalHits.Relation.EQUAL_TO;
+        private long totalHits;
+        private TotalHits.Relation totalHitsRelation;
         long fetchHits;
         float maxScore = Float.NEGATIVE_INFINITY;
 
@@ -759,7 +760,12 @@ static final class TopDocsStats {
 
         TopDocsStats(boolean trackTotalHits) {
             this.trackTotalHits = trackTotalHits;
-            this.totalHits = trackTotalHits ? 0 : -1;
+            this.totalHits = 0;
+            this.totalHitsRelation = trackTotalHits ? Relation.EQUAL_TO : Relation.GREATER_THAN_OR_EQUAL_TO;
+        }
+
+        TotalHits getTotalHits() {
+            return trackTotalHits ? new TotalHits(totalHits, totalHitsRelation) : null;
         }
 
         void add(TopDocsAndMaxScore topDocs) {

server/src/main/java/org/elasticsearch/index/SearchSlowLog.java

@@ -163,7 +163,13 @@ public String toString() {
                     .append(" ")
                     .append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], ")
                     .append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ")
-                    .append("total_hits[").append(context.queryResult().getTotalHits()).append("], ");
+                    .append("total_hits[");
+            if (context.queryResult().getTotalHits() != null) {
+                sb.append(context.queryResult().getTotalHits());
+            } else {
+                sb.append("-1");
+            }
+            sb.append("], ");
             if (context.getQueryShardContext().getTypes() == null) {
                 sb.append("types[], ");
             } else {

server/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java

@@ -198,7 +198,8 @@ private Response wrap(SearchResponse response) {
             }
             hits = unmodifiableList(hits);
         }
-        return new Response(response.isTimedOut(), failures, response.getHits().getTotalHits(),
+        long total = response.getHits().getTotalHits().value;
+        return new Response(response.isTimedOut(), failures, total,
                 hits, response.getScrollId());
     }
 

server/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest.action.cat;
 
+import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -60,7 +61,7 @@ protected void documentation(StringBuilder sb) {
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         SearchRequest countRequest = new SearchRequest(indices);
-        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true);
         countRequest.source(searchSourceBuilder);
         try {
             request.withContentOrSourceParamParserOrNull(parser -> {
@@ -79,6 +80,7 @@ public RestChannelConsumer doCatRequest(final RestRequest request, final NodeCli
         return channel -> client.search(countRequest, new RestResponseListener<SearchResponse>(channel) {
             @Override
             public RestResponse buildResponse(SearchResponse countResponse) throws Exception {
+                assert countResponse.getHits().getTotalHits().relation == TotalHits.Relation.EQUAL_TO;
                 return RestTable.buildResponse(buildTable(request, countResponse), channel);
             }
         });
@@ -96,7 +98,7 @@ protected Table getTableWithHeader(final RestRequest request) {
     private Table buildTable(RestRequest request, SearchResponse response) {
         Table table = getTableWithHeader(request);
         table.startRow();
-        table.addCell(response.getHits().getTotalHits());
+        table.addCell(response.getHits().getTotalHits().value);
         table.endRow();
 
         return table;

server/src/main/java/org/elasticsearch/rest/action/search/RestCountAction.java

@@ -108,7 +108,7 @@ public RestResponse buildResponse(SearchResponse response, XContentBuilder build
                 if (terminateAfter != DEFAULT_TERMINATE_AFTER) {
                     builder.field("terminated_early", response.isTerminatedEarly());
                 }
-                builder.field("count", response.getHits().getTotalHits());
+                builder.field("count", response.getHits().getTotalHits().value);
                 buildBroadcastShardsHeader(builder, request, response.getTotalShards(), response.getSuccessfulShards(),
                     0, response.getFailedShards(), response.getShardFailures());
 

server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java

@@ -40,15 +40,24 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestMultiSearchAction extends BaseRestHandler {
-    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>(
+            Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM)
+        );
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(
         LogManager.getLogger(RestMultiSearchAction.class));

server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java

@@ -45,6 +45,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Set;
 import java.util.function.IntConsumer;
 
@@ -54,8 +55,18 @@
 import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
 
 public class RestSearchAction extends BaseRestHandler {
+    /**
+     * Indicates whether hits.total should be rendered as an integer or an object
+     * in the rest search response.
+     */
+    public static final String TOTAL_HIT_AS_INT_PARAM = "rest_total_hits_as_int";
     public static final String TYPED_KEYS_PARAM = "typed_keys";
-    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
+    private static final Set<String> RESPONSE_PARAMS;
+
+    static {
+        final Set<String> responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HIT_AS_INT_PARAM));
+        RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
+    }
 
     private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestSearchAction.class));
     static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" +

server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java

@@ -29,12 +29,16 @@
 import org.elasticsearch.search.Scroll;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestSearchScrollAction extends BaseRestHandler {
+    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM);
+
     public RestSearchScrollAction(Settings settings, RestController controller) {
         super(settings);
 
@@ -70,4 +74,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             }});
         return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel));
     }
+
+    @Override
+    protected Set<String> responseParams() {
+        return RESPONSE_PARAMS;
+    }
 }

server/src/main/java/org/elasticsearch/search/SearchHit.java

@@ -117,7 +117,7 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable<D
 
     private Map<String, SearchHits> innerHits;
 
-    private SearchHit() {
+    SearchHit() {
 
     }
 
@@ -792,6 +792,8 @@ public void readFrom(StreamInput in) throws IOException {
                 SearchHits value = SearchHits.readSearchHits(in);
                 innerHits.put(key, value);
             }
+        } else {
+            innerHits = null;
         }
     }