Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Include supported aggregations in field caps api response #93884

Draft
wants to merge 6 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.index.mapper.TimeSeriesParams;
import org.elasticsearch.xcontent.ConstructingObjectParser;
import org.elasticsearch.xcontent.InstantiatingObjectParser;
Expand Down Expand Up @@ -57,6 +58,7 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
private static final ParseField NON_DIMENSION_INDICES_FIELD = new ParseField("non_dimension_indices");
private static final ParseField METRIC_CONFLICTS_INDICES_FIELD = new ParseField("metric_conflicts_indices");
private static final ParseField META_FIELD = new ParseField("meta");
private static final ParseField SUPPORTED_AGGREGATIONS = new ParseField("supported_aggregations");

private final String name;
private final String type;
Expand All @@ -74,24 +76,28 @@ public class FieldCapabilities implements Writeable, ToXContentObject {

private final Map<String, Set<String>> meta;

private final Set<String> supportedAggregations;

/**
* Constructor for a set of indices.
* @param name The name of the field
* @param type The type associated with the field.
* @param isMetadataField Whether this field is a metadata field.
* @param isSearchable Whether this field is indexed for search.
* @param isAggregatable Whether this field can be aggregated on.
* @param isDimension Whether this field can be used as dimension
* @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
* @param indices The list of indices where this field name is defined as {@code type},
* or null if all indices have the same {@code type} for the field.
* @param nonSearchableIndices The list of indices where this field is not searchable,
* or null if the field is searchable in all indices.
*
* @param name The name of the field
* @param type The type associated with the field.
* @param isMetadataField Whether this field is a metadata field.
* @param isSearchable Whether this field is indexed for search.
* @param isAggregatable Whether this field can be aggregated on.
* @param isDimension Whether this field can be used as dimension
* @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
* @param indices The list of indices where this field name is defined as {@code type},
* or null if all indices have the same {@code type} for the field.
* @param nonSearchableIndices The list of indices where this field is not searchable,
* or null if the field is searchable in all indices.
* @param nonAggregatableIndices The list of indices where this field is not aggregatable,
* or null if the field is aggregatable in all indices.
* @param nonDimensionIndices The list of indices where this field is not a dimension
* @param nonDimensionIndices The list of indices where this field is not a dimension
* @param metricConflictsIndices The list of indices where this field is has different metric types or not mark as a metric
* @param meta Merged metadata across indices.
* @param meta Merged metadata across indices.
* @param supportedAggregations If the field is aggregatable then the list of supported aggregations
*/
public FieldCapabilities(
String name,
Expand All @@ -106,7 +112,8 @@ public FieldCapabilities(
String[] nonAggregatableIndices,
String[] nonDimensionIndices,
String[] metricConflictsIndices,
Map<String, Set<String>> meta
Map<String, Set<String>> meta,
Set<String> supportedAggregations
) {
this.name = name;
this.type = type;
Expand All @@ -121,6 +128,7 @@ public FieldCapabilities(
this.nonDimensionIndices = nonDimensionIndices;
this.metricConflictsIndices = metricConflictsIndices;
this.meta = Objects.requireNonNull(meta);
this.supportedAggregations = Objects.requireNonNull(supportedAggregations);
}

/**
Expand Down Expand Up @@ -163,9 +171,9 @@ public FieldCapabilities(
nonAggregatableIndices,
null,
null,
meta
meta,
Set.of()
);

}

/**
Expand Down Expand Up @@ -202,7 +210,8 @@ public FieldCapabilities(
List<String> nonAggregatableIndices,
List<String> nonDimensionIndices,
List<String> metricConflictsIndices,
Map<String, Set<String>> meta
Map<String, Set<String>> meta,
List<String> supportedAggregations
) {
this(
name,
Expand All @@ -217,7 +226,8 @@ public FieldCapabilities(
nonAggregatableIndices != null ? nonAggregatableIndices.toArray(new String[0]) : null,
nonDimensionIndices != null ? nonDimensionIndices.toArray(new String[0]) : null,
metricConflictsIndices != null ? metricConflictsIndices.toArray(new String[0]) : null,
meta != null ? meta : Collections.emptyMap()
meta != null ? meta : Collections.emptyMap(),
supportedAggregations != null ? new HashSet<>(supportedAggregations) : Set.of()
);
}

Expand Down Expand Up @@ -245,6 +255,9 @@ public FieldCapabilities(
this.metricConflictsIndices = null;
}
meta = in.readMap(StreamInput::readString, i -> i.readSet(StreamInput::readString));
supportedAggregations = in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)
? in.readSet(StreamInput::readString)
: Set.of();
}

@Override
Expand All @@ -266,6 +279,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalStringArray(metricConflictsIndices);
}
out.writeMap(meta, StreamOutput::writeString, (o, set) -> o.writeCollection(set, StreamOutput::writeString));
if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
out.writeCollection(supportedAggregations, StreamOutput::writeString);
}
}

@Override
Expand Down Expand Up @@ -307,6 +323,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endObject();
}
builder.startArray(SUPPORTED_AGGREGATIONS.getPreferredName());
for (String supportedAggregation : supportedAggregations) {
builder.value(supportedAggregation);
}
builder.endArray();
builder.endObject();
return builder;
}
Expand Down Expand Up @@ -340,6 +361,7 @@ public static FieldCapabilities fromXContent(String name, XContentParser parser)
(p, context) -> p.map(HashMap::new, v -> Set.copyOf(v.list())),
META_FIELD
);
parser.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), SUPPORTED_AGGREGATIONS);
PARSER = parser.build();
}

Expand Down Expand Up @@ -437,6 +459,10 @@ public Map<String, Set<String>> meta() {
return meta;
}

public Set<String> getSupportedAggregations() {
return supportedAggregations;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -454,12 +480,23 @@ public boolean equals(Object o) {
&& Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices)
&& Arrays.equals(nonDimensionIndices, that.nonDimensionIndices)
&& Arrays.equals(metricConflictsIndices, that.metricConflictsIndices)
&& Objects.equals(meta, that.meta);
&& Objects.equals(meta, that.meta)
&& Objects.equals(supportedAggregations, that.supportedAggregations);
}

@Override
public int hashCode() {
int result = Objects.hash(name, type, isMetadataField, isSearchable, isAggregatable, isDimension, metricType, meta);
int result = Objects.hash(
name,
type,
isMetadataField,
isSearchable,
isAggregatable,
isDimension,
metricType,
meta,
supportedAggregations
);
result = 31 * result + Arrays.hashCode(indices);
result = 31 * result + Arrays.hashCode(nonSearchableIndices);
result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
Expand All @@ -474,7 +511,7 @@ public String toString() {
}

static FieldCapabilities buildBasic(String field, String type, String[] indices) {
return new FieldCapabilities(field, type, false, false, false, false, null, indices, null, null, null, null, Map.of());
return new FieldCapabilities(field, type, false, false, false, false, null, indices, null, null, null, null, Map.of(), Set.of());
}

static class Builder {
Expand Down Expand Up @@ -522,7 +559,8 @@ void add(
boolean agg,
boolean isDimension,
TimeSeriesParams.MetricType metricType,
Map<String, String> meta
Map<String, String> meta,
Set<String> supportedAggregations
) {
assert assertIndicesSorted(indices);
totalIndices += indices.length;
Expand All @@ -544,7 +582,7 @@ void add(
hasConflictMetricType = true;
this.metricType = null;
}
indicesList.add(new IndexCaps(indices, search, agg, isDimension, metricType));
indicesList.add(new IndexCaps(indices, search, agg, isDimension, metricType, supportedAggregations));
for (Map.Entry<String, String> entry : meta.entrySet()) {
this.meta.computeIfAbsent(entry.getKey(), key -> new HashSet<>()).add(entry.getValue());
}
Expand Down Expand Up @@ -613,6 +651,14 @@ FieldCapabilities build(boolean withIndices) {
Map<String, Set<String>> immutableMeta = meta.entrySet()
.stream()
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, entryValueFunction.andThen(Set::copyOf)));

var supportedAggregations = indicesList.get(0).supportedAggregations();
if (indicesList.size() > 1) {
for (int i = 1; i < indicesList.size(); i++) {
supportedAggregations = Sets.intersection(supportedAggregations, indicesList.get(i).supportedAggregations());
}
}

return new FieldCapabilities(
name,
type,
Expand All @@ -626,7 +672,8 @@ FieldCapabilities build(boolean withIndices) {
nonAggregatableIndices,
nonDimensionIndices,
metricConflictsIndices,
immutableMeta
immutableMeta,
supportedAggregations
);
}
}
Expand All @@ -636,6 +683,7 @@ private record IndexCaps(
boolean isSearchable,
boolean isAggregatable,
boolean isDimension,
TimeSeriesParams.MetricType metricType
TimeSeriesParams.MetricType metricType,
Set<String> supportedAggregations
) {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,13 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
for (String field : fieldNames) {
MappedFieldType ft = context.getFieldType(field);
if (filter.test(ft)) {
Set<String> supportedAggregations;
if (ft.isAggregatable()) {
var valuesSourceType = context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH).getValuesSourceType();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe not load index field data? The only reason that index field data is loaded, is to figure out the value source type. Maybe getValueSourceType() method can be added to MappedFieldType? This avoids having to load field data. For some fields like _id this causes warnings (see note in FieldCapabilitiesFilterTests). On the other hand verifying that index field data can be loaded is a good test that the field is actually aggregatable?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I vaguely recall that there was some difficulty with putting getValuesSourceType() on MappedFieldType, but I don't recall what it was. I think it's a good idea if it's not a ton of work.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, if it isn't too much then I think we should try it.

It does avoid many deprecation warning when asking field caps for all fields, since it will try to load Index field data for _id field and that triggers deprecation warning (I had to alter a test to take into account these deprecation warnings).

supportedAggregations = context.getValuesSourceRegistry().getSupportedAggregations(valuesSourceType);
} else {
supportedAggregations = Set.of();
}
IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(
field,
ft.familyTypeName(),
Expand All @@ -125,6 +132,7 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
ft.isAggregatable(),
isTimeSeriesIndex ? ft.isDimension() : false,
isTimeSeriesIndex ? ft.getMetricType() : null,
supportedAggregations,
ft.meta()
);
responseMap.put(field, fieldCap);
Expand Down Expand Up @@ -155,6 +163,7 @@ static Map<String, IndexFieldCapabilities> retrieveFieldCaps(
false,
false,
null,
Set.of(),
Collections.emptyMap()
);
responseMap.put(parentField, fieldCap);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import java.io.IOException;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

/**
* Describes the capabilities of a field in a single index.
Expand All @@ -33,14 +34,16 @@ public class IndexFieldCapabilities implements Writeable {
private final boolean isAggregatable;
private final boolean isDimension;
private final TimeSeriesParams.MetricType metricType;
private final Set<String> supportedAggregations;
private final Map<String, String> meta;

/**
* @param name The name of the field.
* @param type The type associated with the field.
* @param isSearchable Whether this field is indexed for search.
* @param isAggregatable Whether this field can be aggregated on.
* @param meta Metadata about the field.
* @param name The name of the field.
* @param type The type associated with the field.
* @param isSearchable Whether this field is indexed for search.
* @param isAggregatable Whether this field can be aggregated on.
* @param supportedAggregations
* @param meta Metadata about the field.
*/
IndexFieldCapabilities(
String name,
Expand All @@ -50,6 +53,7 @@ public class IndexFieldCapabilities implements Writeable {
boolean isAggregatable,
boolean isDimension,
TimeSeriesParams.MetricType metricType,
Set<String> supportedAggregations,
Map<String, String> meta
) {
this.name = name;
Expand All @@ -59,6 +63,7 @@ public class IndexFieldCapabilities implements Writeable {
this.isAggregatable = isAggregatable;
this.isDimension = isDimension;
this.metricType = metricType;
this.supportedAggregations = Objects.requireNonNull(supportedAggregations);
this.meta = meta;
}

Expand All @@ -75,6 +80,11 @@ public class IndexFieldCapabilities implements Writeable {
this.isDimension = false;
this.metricType = null;
}
if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
this.supportedAggregations = in.readSet(StreamInput::readString);
} else {
this.supportedAggregations = Set.of();
}
this.meta = in.readMap(StreamInput::readString, StreamInput::readString);
}

Expand All @@ -89,6 +99,9 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(isDimension);
out.writeOptionalEnum(metricType);
}
if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
out.writeCollection(supportedAggregations, StreamOutput::writeString);
}
out.writeMap(meta, StreamOutput::writeString, StreamOutput::writeString);
}

Expand Down Expand Up @@ -124,6 +137,10 @@ public Map<String, String> meta() {
return meta;
}

public Set<String> getSupportedAggregations() {
return supportedAggregations;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -136,11 +153,22 @@ public boolean equals(Object o) {
&& Objects.equals(metricType, that.metricType)
&& Objects.equals(name, that.name)
&& Objects.equals(type, that.type)
&& Objects.equals(supportedAggregations, that.supportedAggregations)
&& Objects.equals(meta, that.meta);
}

@Override
public int hashCode() {
return Objects.hash(name, type, isMetadatafield, isSearchable, isAggregatable, isDimension, metricType, meta);
return Objects.hash(
name,
type,
isMetadatafield,
isSearchable,
isAggregatable,
isDimension,
metricType,
supportedAggregations,
meta
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,8 @@ private void innerMerge(
fieldCap.isAggregatable(),
fieldCap.isDimension(),
fieldCap.getMetricType(),
fieldCap.meta()
fieldCap.meta(),
fieldCap.getSupportedAggregations()
);
}
}
Expand Down
Loading