diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 9d6bc57a771a5..f3e88a9199e58 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -16,7 +16,7 @@ Provides usage information about the installed {xpack} features. === {api-description-title} This API provides information about which features are currently enabled and -available under the current license and some usage statistics. +available under the current license and some usage statistics. [discrete] [[usage-api-query-parms]] @@ -263,7 +263,14 @@ GET /_xpack/usage }, "analytics" : { "available" : true, - "enabled" : true + "enabled" : true, + "stats": { + "boxplot_usage" : 0, + "top_metrics_usage" : 0, + "cumulative_cardinality_usage" : 0, + "t_test_usage" : 0, + "string_stats_usage" : 0 + } } } ------------------------------------------------------------ diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java index 7d006efefa2d5..176ac498b7d0d 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java @@ -75,7 +75,7 @@ public List getPipelineAggregations() { CumulativeCardinalityPipelineAggregationBuilder.NAME, CumulativeCardinalityPipelineAggregationBuilder::new, CumulativeCardinalityPipelineAggregator::new, - usage.track(AnalyticsUsage.Item.CUMULATIVE_CARDINALITY, + usage.track(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY, checkLicense(CumulativeCardinalityPipelineAggregationBuilder.PARSER))) ); } @@ -86,24 +86,24 @@ public List getAggregations() { new AggregationSpec( StringStatsAggregationBuilder.NAME, StringStatsAggregationBuilder::new, - usage.track(AnalyticsUsage.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER))) + usage.track(AnalyticsStatsAction.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER))) .addResultReader(InternalStringStats::new) .setAggregatorRegistrar(StringStatsAggregationBuilder::registerAggregators), new AggregationSpec( BoxplotAggregationBuilder.NAME, BoxplotAggregationBuilder::new, - usage.track(AnalyticsUsage.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER))) + usage.track(AnalyticsStatsAction.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER))) .addResultReader(InternalBoxplot::new) .setAggregatorRegistrar(BoxplotAggregationBuilder::registerAggregators), new AggregationSpec( TopMetricsAggregationBuilder.NAME, TopMetricsAggregationBuilder::new, - usage.track(AnalyticsUsage.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER))) + usage.track(AnalyticsStatsAction.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER))) .addResultReader(InternalTopMetrics::new), new AggregationSpec( TTestAggregationBuilder.NAME, TTestAggregationBuilder::new, - usage.track(AnalyticsUsage.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER))) + usage.track(AnalyticsStatsAction.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER))) .addResultReader(InternalTTest::new) ); } @@ -137,7 +137,7 @@ public Collection createComponents(Client client, ClusterService cluster ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver) { - return singletonList(new AnalyticsUsage()); + return singletonList(usage); } @Override diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java index 02ca990a08f61..6b5d72e3c2f1a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java @@ -8,54 +8,32 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.xpack.core.analytics.EnumCounters; import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; -import java.util.EnumMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - /** * Tracks usage of the Analytics aggregations. */ public class AnalyticsUsage { - /** - * Items to track. - */ - public enum Item { - BOXPLOT, - CUMULATIVE_CARDINALITY, - STRING_STATS, - TOP_METRICS, - T_TEST; - } - private final Map trackers = new EnumMap<>(Item.class); + private final EnumCounters counters = new EnumCounters<>(AnalyticsStatsAction.Item.class); public AnalyticsUsage() { - for (Item item: Item.values()) { - trackers.put(item, new AtomicLong(0)); - } } /** * Track successful parsing. */ - public ContextParser track(Item item, ContextParser realParser) { - AtomicLong usage = trackers.get(item); + public ContextParser track(AnalyticsStatsAction.Item item, ContextParser realParser) { return (parser, context) -> { T value = realParser.parse(parser, context); // Intentionally doesn't count unless the parser returns cleanly. - usage.incrementAndGet(); + counters.inc(item); return value; }; } public AnalyticsStatsAction.NodeResponse stats(DiscoveryNode node) { - return new AnalyticsStatsAction.NodeResponse(node, - trackers.get(Item.BOXPLOT).get(), - trackers.get(Item.CUMULATIVE_CARDINALITY).get(), - trackers.get(Item.STRING_STATS).get(), - trackers.get(Item.TOP_METRICS).get(), - trackers.get(Item.T_TEST).get()); + return new AnalyticsStatsAction.NodeResponse(node, counters); } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java index 998ac6b3f0b56..8e0727c014c34 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/action/AnalyticsUsageTransportAction.java @@ -7,6 +7,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; @@ -20,26 +21,49 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; +import org.elasticsearch.xpack.core.analytics.EnumCounters; +import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; public class AnalyticsUsageTransportAction extends XPackUsageFeatureTransportAction { private final XPackLicenseState licenseState; + private final Client client; @Inject public AnalyticsUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - XPackLicenseState licenseState) { + XPackLicenseState licenseState, Client client) { super(XPackUsageFeatureAction.ANALYTICS.name(), transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver); this.licenseState = licenseState; + this.client = client; } @Override protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, ActionListener listener) { boolean available = licenseState.isDataScienceAllowed(); + if (available) { + AnalyticsStatsAction.Request statsRequest = new AnalyticsStatsAction.Request(); + statsRequest.setParentTask(clusterService.localNode().getId(), task.getId()); + client.execute(AnalyticsStatsAction.INSTANCE, statsRequest, ActionListener.wrap(r -> + listener.onResponse(new XPackUsageFeatureResponse(usageFeatureResponse(true, true, r))), + listener::onFailure)); + } else { + AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(false, true, Collections.emptyMap()); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } + } - AnalyticsFeatureSetUsage usage = - new AnalyticsFeatureSetUsage(available, true); - listener.onResponse(new XPackUsageFeatureResponse(usage)); + static AnalyticsFeatureSetUsage usageFeatureResponse(boolean available, boolean enabled, AnalyticsStatsAction.Response r) { + List> countersPerNode = r.getNodes() + .stream() + .map(AnalyticsStatsAction.NodeResponse::getStats) + .collect(Collectors.toList()); + EnumCounters mergedCounters = EnumCounters.merge(AnalyticsStatsAction.Item.class, countersPerNode); + return new AnalyticsFeatureSetUsage(available, enabled, mergedCounters.toMap()); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java index 9eebdd03c1162..ba76d96c66213 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsInfoTransportActionTests.java @@ -5,28 +5,54 @@ */ package org.elasticsearch.xpack.analytics.action; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; +import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; import org.junit.Before; +import org.mockito.stubbing.Answer; + +import java.util.Collections; import static org.hamcrest.core.Is.is; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; public class AnalyticsInfoTransportActionTests extends ESTestCase { private XPackLicenseState licenseState; + private Task task; + private ClusterService clusterService; + private ClusterName clusterName; @Before public void init() { licenseState = mock(XPackLicenseState.class); + task = mock(Task.class); + when(task.getId()).thenReturn(randomLong()); + clusterService = mock(ClusterService.class); + DiscoveryNode discoveryNode = mock(DiscoveryNode.class); + when(discoveryNode.getId()).thenReturn(randomAlphaOfLength(10)); + when(clusterService.localNode()).thenReturn(discoveryNode); + clusterName = mock(ClusterName.class); } public void testAvailable() throws Exception { @@ -35,11 +61,11 @@ public void testAvailable() throws Exception { boolean available = randomBoolean(); when(licenseState.isDataScienceAllowed()).thenReturn(available); assertThat(featureSet.available(), is(available)); - - AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), null, null, - mock(ActionFilters.class), null, licenseState); + Client client = mockClient(); + AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), clusterService, null, + mock(ActionFilters.class), null, licenseState, client); PlainActionFuture future = new PlainActionFuture<>(); - usageAction.masterOperation(null, null, null, future); + usageAction.masterOperation(task, null, null, future); XPackFeatureSet.Usage usage = future.get().getUsage(); assertThat(usage.available(), is(available)); @@ -47,6 +73,10 @@ public void testAvailable() throws Exception { usage.writeTo(out); XPackFeatureSet.Usage serializedUsage = new AnalyticsFeatureSetUsage(out.bytes().streamInput()); assertThat(serializedUsage.available(), is(available)); + if (available) { + verify(client, times(1)).execute(any(), any(), any()); + } + verifyNoMoreInteractions(client); } public void testEnabled() throws Exception { @@ -54,11 +84,13 @@ public void testEnabled() throws Exception { mock(TransportService.class), mock(ActionFilters.class), licenseState); assertThat(featureSet.enabled(), is(true)); assertTrue(featureSet.enabled()); - + boolean available = randomBoolean(); + when(licenseState.isDataScienceAllowed()).thenReturn(available); + Client client = mockClient(); AnalyticsUsageTransportAction usageAction = new AnalyticsUsageTransportAction(mock(TransportService.class), - null, null, mock(ActionFilters.class), null, licenseState); + clusterService, null, mock(ActionFilters.class), null, licenseState, client); PlainActionFuture future = new PlainActionFuture<>(); - usageAction.masterOperation(null, null, null, future); + usageAction.masterOperation(task, null, null, future); XPackFeatureSet.Usage usage = future.get().getUsage(); assertTrue(usage.enabled()); @@ -66,6 +98,22 @@ public void testEnabled() throws Exception { usage.writeTo(out); XPackFeatureSet.Usage serializedUsage = new AnalyticsFeatureSetUsage(out.bytes().streamInput()); assertTrue(serializedUsage.enabled()); + if (available) { + verify(client, times(1)).execute(any(), any(), any()); + } + verifyNoMoreInteractions(client); + } + + private Client mockClient() { + Client client = mock(Client.class); + doAnswer((Answer) invocation -> { + @SuppressWarnings("unchecked") + ActionListener listener = + (ActionListener) invocation.getArguments()[2]; + listener.onResponse(new AnalyticsStatsAction.Response(clusterName, Collections.emptyList(), Collections.emptyList())); + return null; + }).when(client).execute(eq(AnalyticsStatsAction.INSTANCE), any(), any()); + return client; } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java new file mode 100644 index 0000000000000..d1c03e373a67d --- /dev/null +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.analytics.action; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.core.analytics.EnumCounters; +import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; + +import static org.hamcrest.Matchers.equalTo; + +public class AnalyticsStatsActionNodeResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return AnalyticsStatsAction.NodeResponse::new; + } + + @Override + protected AnalyticsStatsAction.NodeResponse createTestInstance() { + String nodeName = randomAlphaOfLength(10); + DiscoveryNode node = new DiscoveryNode(nodeName, buildNewFakeTransportAddress(), Version.CURRENT); + EnumCounters counters = new EnumCounters<>(AnalyticsStatsAction.Item.class); + for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) { + if (randomBoolean()) { + counters.inc(item, randomLongBetween(0, 1000)); + } + } + return new AnalyticsStatsAction.NodeResponse(node, counters); + } + + public void testItemEnum() { + int i = 0; + // We rely on the ordinals for serialization, so they shouldn't change between version + assertThat(AnalyticsStatsAction.Item.BOXPLOT.ordinal(), equalTo(i++)); + assertThat(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY.ordinal(), equalTo(i++)); + assertThat(AnalyticsStatsAction.Item.STRING_STATS.ordinal(), equalTo(i++)); + assertThat(AnalyticsStatsAction.Item.TOP_METRICS.ordinal(), equalTo(i++)); + assertThat(AnalyticsStatsAction.Item.T_TEST.ordinal(), equalTo(i++)); + // Please add tests for newly added items here + assertThat(AnalyticsStatsAction.Item.values().length, equalTo(i)); + } +} diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java index 388171d3a082f..49047d41de190 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.analytics.AnalyticsUsage; +import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage; import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction; import java.io.IOException; @@ -33,6 +34,7 @@ import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.analytics.action.AnalyticsUsageTransportAction.usageFeatureResponse; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -58,20 +60,18 @@ public TransportAnalyticsStatsAction action(AnalyticsUsage usage) { } public void test() throws IOException { - for (AnalyticsUsage.Item item : AnalyticsUsage.Item.values()) { + for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) { AnalyticsUsage realUsage = new AnalyticsUsage(); AnalyticsUsage emptyUsage = new AnalyticsUsage(); ContextParser parser = realUsage.track(item, (p, c) -> c); ObjectPath unused = run(realUsage, emptyUsage); - assertThat(unused.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0)); - assertThat(unused.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0)); + assertThat(unused.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0)); int count = between(1, 10000); for (int i = 0; i < count; i++) { assertNull(parser.parse(null, null)); } ObjectPath used = run(realUsage, emptyUsage); - assertThat(item.name(), used.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count)); - assertThat(item.name(), used.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0)); + assertThat(item.name(), used.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count)); } } @@ -83,11 +83,9 @@ private ObjectPath run(AnalyticsUsage... nodeUsages) throws IOException { AnalyticsStatsAction.Response response = new AnalyticsStatsAction.Response( new ClusterName("cluster_name"), nodeResponses, emptyList()); + AnalyticsFeatureSetUsage usage = usageFeatureResponse(true, true, response); try (XContentBuilder builder = jsonBuilder()) { - builder.startObject(); - response.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - + usage.toXContent(builder, ToXContent.EMPTY_PARAMS); return ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java index 9913df74cc314..8d6fd97db8bae 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureResponse.java @@ -34,4 +34,4 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteable(usage); } - } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java index b39ca2c8ed8fa..29be3866e39ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java @@ -43,4 +43,4 @@ public void writeTo(StreamOutput out) throws IOException { } } - } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java index f7805703ad58a..30b9119a5dd29 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/AnalyticsFeatureSetUsage.java @@ -6,26 +6,47 @@ package org.elasticsearch.xpack.core.analytics; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.XPackFeatureSet; import org.elasticsearch.xpack.core.XPackField; import java.io.IOException; +import java.util.Collections; +import java.util.Map; import java.util.Objects; public class AnalyticsFeatureSetUsage extends XPackFeatureSet.Usage { - public AnalyticsFeatureSetUsage(boolean available, boolean enabled) { + private final Map stats; + + public AnalyticsFeatureSetUsage(boolean available, boolean enabled, Map stats) { super(XPackField.ANALYTICS, available, enabled); + this.stats = stats; } public AnalyticsFeatureSetUsage(StreamInput input) throws IOException { super(input); + if (input.getVersion().onOrAfter(Version.V_7_8_0)) { + stats = input.readMap(); + } else { + stats = Collections.emptyMap(); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeMap(stats); + } } @Override public int hashCode() { - return Objects.hash(available, enabled); + return Objects.hash(available, enabled, stats); } @Override @@ -38,6 +59,19 @@ public boolean equals(Object obj) { } AnalyticsFeatureSetUsage other = (AnalyticsFeatureSetUsage) obj; return Objects.equals(available, other.available) && - Objects.equals(enabled, other.enabled); + Objects.equals(enabled, other.enabled) && + Objects.equals(stats, other.stats); + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + if (enabled) { + builder.startObject("stats"); + for (Map.Entry entry : stats.entrySet()) { + builder.field(entry.getKey() + "_usage", entry.getValue()); + } + builder.endObject(); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java new file mode 100644 index 0000000000000..29d3799377206 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.analytics; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLongArray; + +/** + * Utility class similar to org.elasticsearch.xpack.core.watcher.common.stats.Counters, but it is using Enum instead + * of string to identify the counter. The serialization happens using enum ordinals similar to + * {@link StreamOutput#writeEnum(Enum)}, which means that ordinal for existing enums should remain the same for backward + * and forward compatibility of the serialization protocol. + */ +public class EnumCounters> implements Writeable { + private final AtomicLongArray counters; + private final E[] enums; + + public EnumCounters(Class enumClass) { + counters = new AtomicLongArray(enumClass.getEnumConstants().length); + enums = enumClass.getEnumConstants(); + } + + public EnumCounters(StreamInput in, Class enumClass) throws IOException { + int size = in.readVInt(); + enums = enumClass.getEnumConstants(); + long[] vals = new long[enums.length]; + for (int i = 0; i < size; i++) { + long val = in.readVLong(); + if (i < vals.length) { + vals[i] = val; + } + } + counters = new AtomicLongArray(vals); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(counters.length()); + for (int i = 0; i < counters.length(); i++) { + out.writeVLong(counters.get(i)); + } + } + + public void set(E name) { + counters.set(name.ordinal(), 0); + } + + public void inc(E name) { + counters.incrementAndGet(name.ordinal()); + } + + public void inc(E name, long count) { + counters.addAndGet(name.ordinal(), count); + } + + public long get(E name) { + return counters.get(name.ordinal()); + } + + public long size() { + return counters.length(); + } + + public boolean hasCounters() { + return size() > 0; + } + + public Map toMap() { + Map map = new HashMap<>(); + for (E e : enums) { + map.put(e.name().toLowerCase(Locale.ROOT), counters.get(e.ordinal())); + } + return map; + } + + public static > EnumCounters merge(Class enumClass, List> counters) { + EnumCounters result = new EnumCounters<>(enumClass); + E[] enums = enumClass.getEnumConstants(); + for (EnumCounters c : counters) { + for (E e : enums) { + result.inc(e, c.get(e)); + } + } + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EnumCounters that = (EnumCounters) o; + return Arrays.equals(toArray(), that.toArray()) && + Arrays.equals(enums, that.enums); + } + + @Override + public int hashCode() { + int result = Arrays.hashCode(toArray()); + result = 31 * result + Arrays.hashCode(enums); + return result; + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder("["); + boolean first = true; + for (E e : enums) { + buf.append(e.name().toLowerCase(Locale.ROOT)).append(": ").append(get(e)); + if (first) { + buf.append(", "); + first = false; + } + } + buf.append("]"); + return buf.toString(); + } + + private long[] toArray() { + long[] res = new long[enums.length]; + for (int i = 0; i < res.length; i++) { + res[i] = counters.get(i); + } + return res; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java index bbd20b3a6d36e..edd3025ec2fa9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java @@ -13,13 +13,13 @@ import org.elasticsearch.action.support.nodes.BaseNodesResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.analytics.EnumCounters; import java.io.IOException; import java.util.List; @@ -33,6 +33,17 @@ private AnalyticsStatsAction() { super(NAME, Response::new); } + /** + * Items to track. Serialized by ordinals. Append only, don't remove or change order of items in this list. + */ + public enum Item { + BOXPLOT, + CUMULATIVE_CARDINALITY, + STRING_STATS, + TOP_METRICS, + T_TEST; + } + public static class Request extends BaseNodesRequest implements ToXContentObject { public Request() { @@ -78,7 +89,7 @@ public NodeRequest(Request request) { } } - public static class Response extends BaseNodesResponse implements Writeable, ToXContentObject { + public static class Response extends BaseNodesResponse implements Writeable { public Response(StreamInput in) throws IOException { super(in); } @@ -96,110 +107,66 @@ protected List readNodesFrom(StreamInput in) throws IOException { protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { out.writeList(nodes); } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startArray("stats"); - for (NodeResponse node : getNodes()) { - node.toXContent(builder, params); - } - builder.endArray(); - - return builder; - } } - public static class NodeResponse extends BaseNodeResponse implements ToXContentObject { - static final ParseField BOXPLOT_USAGE = new ParseField("boxplot_usage"); - static final ParseField CUMULATIVE_CARDINALITY_USAGE = new ParseField("cumulative_cardinality_usage"); - static final ParseField STRING_STATS_USAGE = new ParseField("string_stats_usage"); - static final ParseField TOP_METRICS_USAGE = new ParseField("top_metrics_usage"); - static final ParseField T_TEST_USAGE = new ParseField("t_test_usage"); - - private final long boxplotUsage; - private final long cumulativeCardinalityUsage; - private final long stringStatsUsage; - private final long topMetricsUsage; - private final long ttestUsage; - - public NodeResponse(DiscoveryNode node, long boxplotUsage, long cumulativeCardinalityUsage, long stringStatsUsage, - long topMetricsUsage, long ttestUsage) { + public static class NodeResponse extends BaseNodeResponse { + private final EnumCounters counters; + + public NodeResponse(DiscoveryNode node, EnumCounters counters) { super(node); - this.boxplotUsage = boxplotUsage; - this.cumulativeCardinalityUsage = cumulativeCardinalityUsage; - this.stringStatsUsage = stringStatsUsage; - this.topMetricsUsage = topMetricsUsage; - this.ttestUsage = ttestUsage; + this.counters = counters; } public NodeResponse(StreamInput in) throws IOException { super(in); - if (in.getVersion().onOrAfter(Version.V_7_7_0)) { - boxplotUsage = in.readVLong(); - } else { - boxplotUsage = 0; - } - cumulativeCardinalityUsage = in.readZLong(); - if (in.getVersion().onOrAfter(Version.V_7_7_0)) { - stringStatsUsage = in.readVLong(); - topMetricsUsage = in.readVLong(); - } else { - stringStatsUsage = 0; - topMetricsUsage = 0; - } if (in.getVersion().onOrAfter(Version.V_7_8_0)) { - ttestUsage = in.readVLong(); + counters = new EnumCounters<>(in, Item.class); } else { - ttestUsage = 0; + counters = new EnumCounters<>(Item.class); + if (in.getVersion().onOrAfter(Version.V_7_7_0)) { + counters.inc(Item.BOXPLOT, in.readVLong()); + } + counters.inc(Item.CUMULATIVE_CARDINALITY, in.readZLong()); + if (in.getVersion().onOrAfter(Version.V_7_7_0)) { + counters.inc(Item.STRING_STATS, in.readVLong()); + counters.inc(Item.TOP_METRICS, in.readVLong()); + } } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_7_0)) { - out.writeVLong(boxplotUsage); - } - out.writeVLong(cumulativeCardinalityUsage); - if (out.getVersion().onOrAfter(Version.V_7_7_0)) { - out.writeVLong(stringStatsUsage); - out.writeVLong(topMetricsUsage); - } if (out.getVersion().onOrAfter(Version.V_7_8_0)) { - out.writeVLong(ttestUsage); + counters.writeTo(out); + } else { + if (out.getVersion().onOrAfter(Version.V_7_7_0)) { + out.writeVLong(counters.get(Item.BOXPLOT)); + } + out.writeZLong(counters.get(Item.CUMULATIVE_CARDINALITY)); + if (out.getVersion().onOrAfter(Version.V_7_7_0)) { + out.writeVLong(counters.get(Item.STRING_STATS)); + out.writeVLong(counters.get(Item.TOP_METRICS)); + } } } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(BOXPLOT_USAGE.getPreferredName(), boxplotUsage); - builder.field(CUMULATIVE_CARDINALITY_USAGE.getPreferredName(), cumulativeCardinalityUsage); - builder.field(STRING_STATS_USAGE.getPreferredName(), stringStatsUsage); - builder.field(TOP_METRICS_USAGE.getPreferredName(), topMetricsUsage); - builder.field(T_TEST_USAGE.getPreferredName(), ttestUsage); - builder.endObject(); - return builder; - } - - public long getBoxplotUsage() { - return boxplotUsage; + public EnumCounters getStats() { + return counters; } - public long getCumulativeCardinalityUsage() { - return cumulativeCardinalityUsage; - } - - public long getStringStatsUsage() { - return stringStatsUsage; - } - - public long getTopMetricsUsage() { - return topMetricsUsage; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeResponse that = (NodeResponse) o; + return counters.equals(that.counters) && + getNode().equals(that.getNode()); } - public long getTTestUsage() { - return topMetricsUsage; + @Override + public int hashCode() { + return Objects.hash(counters, getNode()); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/analytics/EnumCountersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/analytics/EnumCountersTests.java new file mode 100644 index 0000000000000..feafc438388c8 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/analytics/EnumCountersTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.analytics; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireTestCase; +import org.elasticsearch.xpack.core.analytics.EnumCounters; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class EnumCountersTests extends AbstractWireTestCase> { + + enum TestV1 {A, B, C} + + enum TestV2 {A, B, C, D} + + @Override + protected EnumCounters createTestInstance() { + EnumCounters inst = new EnumCounters<>(TestV2.class); + inst.inc(TestV2.A, randomNonNegativeLong()); + inst.inc(TestV2.B, randomNonNegativeLong()); + inst.inc(TestV2.C, randomNonNegativeLong()); + inst.inc(TestV2.D, randomNonNegativeLong()); + return inst; + } + + @Override + protected EnumCounters copyInstance(EnumCounters instance, Version version) throws IOException { + return serialize(instance, in -> new EnumCounters<>(in, TestV2.class)); + } + + public void testIncrements() { + EnumCounters counters = new EnumCounters<>(TestV1.class); + int a = randomIntBetween(0, 100); + int b = randomIntBetween(0, 100); + int c = randomIntBetween(0, 100); + incrementRandomly(counters, TestV1.A, a); + incrementRandomly(counters, TestV1.B, b); + incrementRandomly(counters, TestV1.C, c); + assertEquals(a, counters.get(TestV1.A)); + assertEquals(b, counters.get(TestV1.B)); + assertEquals(c, counters.get(TestV1.C)); + Map map = counters.toMap(); + assertThat(map.keySet(), hasSize(3)); + assertThat(map.get("a"), equalTo((long) a)); + assertThat(map.get("b"), equalTo((long) b)); + assertThat(map.get("c"), equalTo((long) c)); + } + + public void testBackwardCompatibility() throws Exception { + EnumCounters counters = new EnumCounters<>(TestV2.class); + counters.inc(TestV2.A, 1); + counters.inc(TestV2.B, 2); + counters.inc(TestV2.C, 3); + counters.inc(TestV2.D, 4); + EnumCounters oldCounters = serialize(counters, in -> new EnumCounters<>(in, TestV1.class)); + assertEquals(counters.get(TestV2.A), oldCounters.get(TestV1.A)); + assertEquals(counters.get(TestV2.B), oldCounters.get(TestV1.B)); + assertEquals(counters.get(TestV2.C), oldCounters.get(TestV1.C)); + } + + + public void testForwardCompatibility() throws Exception { + EnumCounters counters = new EnumCounters<>(TestV1.class); + counters.inc(TestV1.A, 1); + counters.inc(TestV1.B, 2); + counters.inc(TestV1.C, 3); + EnumCounters newCounters = serialize(counters, in -> new EnumCounters<>(in, TestV2.class)); + assertEquals(counters.get(TestV1.A), newCounters.get(TestV2.A)); + assertEquals(counters.get(TestV1.B), newCounters.get(TestV2.B)); + assertEquals(counters.get(TestV1.C), newCounters.get(TestV2.C)); + assertEquals(0, newCounters.get(TestV2.D)); + } + + private , E2 extends Enum> EnumCounters serialize( + EnumCounters source, Writeable.Reader> targetReader) throws IOException { + + try (BytesStreamOutput output = new BytesStreamOutput()) { + source.writeTo(output); + try (StreamInput in = output.bytes().streamInput()) { + return targetReader.read(in); + } + } + } + + private > void incrementRandomly(EnumCounters counters, E e, int inc) { + int single = randomIntBetween(0, inc); + if (randomBoolean()) { + for (int i = 0; i < single; i++) { + counters.inc(e); + } + counters.inc(e, inc - single); + } else { + counters.inc(e, inc - single); + for (int i = 0; i < single; i++) { + counters.inc(e); + } + } + } + +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/usage.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/usage.yml new file mode 100644 index 0000000000000..64b316fbc716b --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/usage.yml @@ -0,0 +1,158 @@ +--- +setup: + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 1, "v1": 3.1415, "v2": 2.1415, "str": "a"}' + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 2, "v1": 1.0, "v2": 2.0, "str": "a"}' + - '{"index": {}}' + - '{"timestamp": "2017-01-01T05:00:00Z", "s": 3, "v1": 2.71828, "v2": 3.71828, "str": "b"}' + +--- +"Usage stats on analytics indices": + - skip: + version: " - 7.99.99" + reason: "stats is not working in earlier versions at the moment" + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - set: {analytics.stats.boxplot_usage: boxplot_usage} + - set: {analytics.stats.top_metrics_usage: top_metrics_usage} + - set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage} + - set: {analytics.stats.t_test_usage: t_test_usage} + - set: {analytics.stats.string_stats_usage: string_stats_usage} + + # use boxplot agg + - do: + search: + index: "test" + body: + size: 0 + aggs: + plot: + boxplot: + field: "s" + + - match: { aggregations.plot.q2: 2.0 } + + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - gt: { analytics.stats.boxplot_usage: $boxplot_usage } + - set: {analytics.stats.boxplot_usage: boxplot_usage} + - match: {analytics.stats.top_metrics_usage: $top_metrics_usage} + - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage} + - match: {analytics.stats.t_test_usage: $t_test_usage} + - match: {analytics.stats.string_stats_usage: $string_stats_usage} + + + # use top_metrics agg + - do: + search: + index: "test" + size: 0 + body: + aggs: + tm: + top_metrics: + metrics: + field: v1 + sort: + s: desc + - match: { aggregations.tm.top.0.metrics.v1: 2.718280076980591 } + - match: { aggregations.tm.top.0.sort: [3] } + + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - match: {analytics.stats.boxplot_usage: $boxplot_usage} + - gt: { analytics.stats.top_metrics_usage: $top_metrics_usage } + - set: {analytics.stats.top_metrics_usage: top_metrics_usage} + - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage} + - match: {analytics.stats.t_test_usage: $t_test_usage} + - match: {analytics.stats.string_stats_usage: $string_stats_usage} + + + # use cumulative_cardinality agg + - do: + search: + index: "test" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "timestamp" + calendar_interval: "day" + aggs: + distinct_s: + cardinality: + field: "s" + total_users: + cumulative_cardinality: + buckets_path: "distinct_s" + + - length: { aggregations.histo.buckets: 1 } + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - match: {analytics.stats.boxplot_usage: $boxplot_usage} + - match: {analytics.stats.top_metrics_usage: $top_metrics_usage} + - gt: { analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage } + - set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage} + - match: {analytics.stats.t_test_usage: $t_test_usage} + - match: {analytics.stats.string_stats_usage: $string_stats_usage} + + # use t-test agg + - do: + search: + size: 0 + index: "test" + body: + aggs: + ttest: + t_test: + a: + field: v1 + b: + field: v2 + - match: { aggregations.ttest.value: 0.7172402682151968 } + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - match: {analytics.stats.boxplot_usage: $boxplot_usage} + - match: {analytics.stats.top_metrics_usage: $top_metrics_usage} + - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage} + - gt: { analytics.stats.t_test_usage: $t_test_usage } + - set: {analytics.stats.t_test_usage: t_test_usage} + - match: {analytics.stats.string_stats_usage: $string_stats_usage} + + - do: + search: + size: 0 + index: "test" + body: + aggs: + my_agg: + string_stats: + field: str.keyword + - match: { aggregations.my_agg.count: 3 } + + - do: {xpack.usage: {}} + - match: { analytics.available: true } + - match: { analytics.enabled: true } + - match: {analytics.stats.boxplot_usage: $boxplot_usage} + - match: {analytics.stats.top_metrics_usage: $top_metrics_usage} + - match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage} + - match: {analytics.stats.t_test_usage: $t_test_usage} + - gt: { analytics.stats.string_stats_usage: $string_stats_usage } + - set: {analytics.stats.string_stats_usage: string_stats_usage}