Skip to content

Commit

Permalink
[ML] Add created_by info to usage stats
Browse files Browse the repository at this point in the history 
This change adds information about which UI path
(if any) created ML anomaly detector jobs to the
stats returned by the _xpack/usage endpoint.

Counts for the following possibilities are expected:

* ml_module_apache_access
* ml_module_apm_transaction
* ml_module_auditbeat_process_docker
* ml_module_auditbeat_process_hosts
* ml_module_nginx_access
* ml_module_sample
* multi_metric_wizard
* population_wizard
* single_metric_wizard
* unknown

The "unknown" count is for jobs that do not have a
created_by setting in their custom_settings.

Closes elastic#38403
  • Loading branch information
@droberts195
droberts195 committed Mar 27, 2019
1 parent 05c7afb commit 1fffc33
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 6 deletions.
1 change: 1 addition & 0 deletions ...in/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class MachineLearningFeatureSetUsage extends XPackFeatureSet.Usage {
public static final String DETECTORS = "detectors";
public static final String FORECASTS = "forecasts";
public static final String MODEL_SIZE = "model_size";
public static final String CREATED_BY = "created_by";
public static final String NODE_COUNT = "node_count";

private final Map<String, Object> jobsUsage;
Expand Down
27 changes: 22 additions & 5 deletions x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,13 +216,16 @@ private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs)
Map<JobState, StatsAccumulator> detectorStatsByState = new HashMap<>();
Map<JobState, StatsAccumulator> modelSizeStatsByState = new HashMap<>();
Map<JobState, ForecastStats> forecastStatsByState = new HashMap<>();
Map<JobState, Map<String, Long>> createdByByState = new HashMap<>();

List<GetJobsStatsAction.Response.JobStats> jobsStats = response.getResponse().results();
Map<String, Job> jobMap = jobs.stream().collect(Collectors.toMap(Job::getId, item -> item));
Map<String, Long> allJobsCreatedBy = jobs.stream().map(this::jobCreatedBy)
.collect(Collectors.groupingBy(item -> item, Collectors.counting()));;
for (GetJobsStatsAction.Response.JobStats jobStats : jobsStats) {
ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
int detectorsCount = jobMap.get(jobStats.getJobId()).getAnalysisConfig()
.getDetectors().size();
Job job = jobMap.get(jobStats.getJobId());
int detectorsCount = job.getAnalysisConfig().getDetectors().size();
double modelSize = modelSizeStats == null ? 0.0
: jobStats.getModelSizeStats().getModelBytes();

Expand All @@ -237,27 +240,41 @@ private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs)
modelSizeStatsByState.computeIfAbsent(jobState,
js -> new StatsAccumulator()).add(modelSize);
forecastStatsByState.merge(jobState, jobStats.getForecastStats(), (f1, f2) -> f1.merge(f2));
createdByByState.computeIfAbsent(jobState, js -> new HashMap<>())
.compute(jobCreatedBy(job), (k, v) -> (v == null) ? 1L : (v + 1));
}

jobsUsage.put(MachineLearningFeatureSetUsage.ALL, createJobUsageEntry(jobs.size(), allJobsDetectorsStats,
allJobsModelSizeStats, allJobsForecastStats));
allJobsModelSizeStats, allJobsForecastStats, allJobsCreatedBy));
for (JobState jobState : jobCountByState.keySet()) {
jobsUsage.put(jobState.name().toLowerCase(Locale.ROOT), createJobUsageEntry(
jobCountByState.get(jobState).get(),
detectorStatsByState.get(jobState),
modelSizeStatsByState.get(jobState),
forecastStatsByState.get(jobState)));
forecastStatsByState.get(jobState),
createdByByState.get(jobState)));
}
}

private String jobCreatedBy(Job job) {
Map<String, Object> customSettings = job.getCustomSettings();
if (customSettings == null || customSettings.containsKey(MachineLearningFeatureSetUsage.CREATED_BY) == false) {
return "unknown";
}
// Replace non-alpha-numeric characters with underscores because
// the values from custom settings become keys in the usage data
return customSettings.get(MachineLearningFeatureSetUsage.CREATED_BY).toString().replaceAll("\\W", "_");
}

private Map<String, Object> createJobUsageEntry(long count, StatsAccumulator detectorStats,
StatsAccumulator modelSizeStats,
ForecastStats forecastStats) {
ForecastStats forecastStats, Map<String, Long> createdBy) {
Map<String, Object> usage = new HashMap<>();
usage.put(MachineLearningFeatureSetUsage.COUNT, count);
usage.put(MachineLearningFeatureSetUsage.DETECTORS, detectorStats.asMap());
usage.put(MachineLearningFeatureSetUsage.MODEL_SIZE, modelSizeStats.asMap());
usage.put(MachineLearningFeatureSetUsage.FORECASTS, forecastStats.asMap());
usage.put(MachineLearningFeatureSetUsage.CREATED_BY, createdBy);
return usage;
}

Expand Down
16 changes: 15 additions & 1 deletion ...ck/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,8 @@ public void testUsage() throws Exception {
Settings.Builder settings = Settings.builder().put(commonSettings);
settings.put("xpack.ml.enabled", true);

Job opened1 = buildJob("opened1", Arrays.asList(buildMinDetector("foo")));
Job opened1 = buildJob("opened1", Collections.singletonList(buildMinDetector("foo")),
Collections.singletonMap("created_by", randomFrom("a-cool-module", "a_cool_module", "a cool module")));
GetJobsStatsAction.Response.JobStats opened1JobStats = buildJobStats("opened1", JobState.OPENED, 100L, 3L);
Job opened2 = buildJob("opened2", Arrays.asList(buildMinDetector("foo"), buildMinDetector("bar")));
GetJobsStatsAction.Response.JobStats opened2JobStats = buildJobStats("opened2", JobState.OPENED, 200L, 8L);
Expand Down Expand Up @@ -200,6 +201,8 @@ public void testUsage() throws Exception {
assertThat(source.getValue("jobs._all.model_size.max"), equalTo(300.0));
assertThat(source.getValue("jobs._all.model_size.total"), equalTo(600.0));
assertThat(source.getValue("jobs._all.model_size.avg"), equalTo(200.0));
assertThat(source.getValue("jobs._all.created_by.a_cool_module"), equalTo(1));
assertThat(source.getValue("jobs._all.created_by.unknown"), equalTo(2));

assertThat(source.getValue("jobs.opened.count"), equalTo(2));
assertThat(source.getValue("jobs.opened.detectors.min"), equalTo(1.0));
Expand All @@ -210,6 +213,8 @@ public void testUsage() throws Exception {
assertThat(source.getValue("jobs.opened.model_size.max"), equalTo(200.0));
assertThat(source.getValue("jobs.opened.model_size.total"), equalTo(300.0));
assertThat(source.getValue("jobs.opened.model_size.avg"), equalTo(150.0));
assertThat(source.getValue("jobs.opened.created_by.a_cool_module"), equalTo(1));
assertThat(source.getValue("jobs.opened.created_by.unknown"), equalTo(1));

assertThat(source.getValue("jobs.closed.count"), equalTo(1));
assertThat(source.getValue("jobs.closed.detectors.min"), equalTo(3.0));
Expand All @@ -220,6 +225,8 @@ public void testUsage() throws Exception {
assertThat(source.getValue("jobs.closed.model_size.max"), equalTo(300.0));
assertThat(source.getValue("jobs.closed.model_size.total"), equalTo(300.0));
assertThat(source.getValue("jobs.closed.model_size.avg"), equalTo(300.0));
assertThat(source.getValue("jobs.closed.created_by.a_cool_module"), is(nullValue()));
assertThat(source.getValue("jobs.closed.created_by.unknown"), equalTo(1));

assertThat(source.getValue("jobs.opening"), is(nullValue()));
assertThat(source.getValue("jobs.closing"), is(nullValue()));
Expand Down Expand Up @@ -359,6 +366,7 @@ private void givenJobs(List<Job> jobs, List<GetJobsStatsAction.Response.JobStats
}).when(jobManager).expandJobs(eq(MetaData.ALL), eq(true), any(ActionListener.class));

doAnswer(invocationOnMock -> {
@SuppressWarnings("unchecked")
ActionListener<GetJobsStatsAction.Response> listener =
(ActionListener<GetJobsStatsAction.Response>) invocationOnMock.getArguments()[2];
listener.onResponse(new GetJobsStatsAction.Response(
Expand Down Expand Up @@ -400,6 +408,7 @@ private void givenNodeCount(int nodeCount) {

private void givenDatafeeds(List<GetDatafeedsStatsAction.Response.DatafeedStats> datafeedStats) {
doAnswer(invocationOnMock -> {
@SuppressWarnings("unchecked")
ActionListener<GetDatafeedsStatsAction.Response> listener =
(ActionListener<GetDatafeedsStatsAction.Response>) invocationOnMock.getArguments()[2];
listener.onResponse(new GetDatafeedsStatsAction.Response(
Expand All @@ -416,10 +425,15 @@ private static Detector buildMinDetector(String fieldName) {
}

private static Job buildJob(String jobId, List<Detector> detectors) {
return buildJob(jobId, detectors, null);
}

private static Job buildJob(String jobId, List<Detector> detectors, Map<String, Object> customSettings) {
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors);
return new Job.Builder(jobId)
.setAnalysisConfig(analysisConfig)
.setDataDescription(new DataDescription.Builder())
.setCustomSettings(customSettings)
.build(new Date(randomNonNegativeLong()));
}

Expand Down

0 comments on commit 1fffc33

Please sign in to comment.