Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
suppport HC detector in profile api (#274)
Browse files Browse the repository at this point in the history
* suppport HC detector in profile api

* address comments
  • Loading branch information
ylwu-amzn authored Oct 19, 2020
1 parent 439404d commit 3a9d6c9
Show file tree
Hide file tree
Showing 27 changed files with 898 additions and 110 deletions.
6 changes: 6 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,12 @@ List<String> jacocoExclusions = [
'com.amazon.opendistroforelasticsearch.ad.transport.AnomalyResultTransportAction.EntityResultListener',
'com.amazon.opendistroforelasticsearch.ad.NodeStateManager',
'com.amazon.opendistroforelasticsearch.ad.transport.handler.MultiEntityResultHandler',
'com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileTransportAction*',
'com.amazon.opendistroforelasticsearch.ad.AnomalyDetectorProfileRunner',
'com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileResponse',
'com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileRequest',
'com.amazon.opendistroforelasticsearch.ad.util.MultiResponsesDelegateActionListener',
'com.amazon.opendistroforelasticsearch.ad.transport.GetAnomalyDetectorRequest',
]

jacocoTestCoverageVerification {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@
import com.amazon.opendistroforelasticsearch.ad.transport.DeleteAnomalyDetectorTransportAction;
import com.amazon.opendistroforelasticsearch.ad.transport.DeleteModelAction;
import com.amazon.opendistroforelasticsearch.ad.transport.DeleteModelTransportAction;
import com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileAction;
import com.amazon.opendistroforelasticsearch.ad.transport.EntityProfileTransportAction;
import com.amazon.opendistroforelasticsearch.ad.transport.EntityResultAction;
import com.amazon.opendistroforelasticsearch.ad.transport.EntityResultTransportAction;
import com.amazon.opendistroforelasticsearch.ad.transport.GetAnomalyDetectorAction;
Expand Down Expand Up @@ -442,6 +444,14 @@ public Collection<Object> createComponents(
StatNames.MODELS_CHECKPOINT_INDEX_STATUS.getName(),
new ADStat<>(true, new IndexStatusSupplier(indexUtils, CommonName.CHECKPOINT_INDEX_NAME))
)
.put(
StatNames.ANOMALY_DETECTION_JOB_INDEX_STATUS.getName(),
new ADStat<>(true, new IndexStatusSupplier(indexUtils, AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX))
)
.put(
StatNames.ANOMALY_DETECTION_STATE_STATUS.getName(),
new ADStat<>(true, new IndexStatusSupplier(indexUtils, DetectorInternalState.DETECTOR_STATE_INDEX))
)
.put(StatNames.DETECTOR_COUNT.getName(), new ADStat<>(true, new SettableSupplier()))
.build();

Expand Down Expand Up @@ -595,7 +605,8 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
new ActionHandler<>(IndexAnomalyDetectorAction.INSTANCE, IndexAnomalyDetectorTransportAction.class),
new ActionHandler<>(AnomalyDetectorJobAction.INSTANCE, AnomalyDetectorJobTransportAction.class),
new ActionHandler<>(ADResultBulkAction.INSTANCE, ADResultBulkTransportAction.class),
new ActionHandler<>(EntityResultAction.INSTANCE, EntityResultTransportAction.class)
new ActionHandler<>(EntityResultAction.INSTANCE, EntityResultTransportAction.class),
new ActionHandler<>(EntityProfileAction.INSTANCE, EntityProfileTransportAction.class)
);
}

Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ public interface EntityCache extends MaintenanceState, CleanState {
/**
* Whether an entity is active or not
* @param detectorId The Id of the detector that an entity belongs to
* @param entityId Entity Id
* @param entityModelId Entity model Id
* @return Whether an entity is active or not
*/
boolean isActive(String detectorId, String entityId);
boolean isActive(String detectorId, String entityModelId);

/**
* Get total updates of detector's most active entity's RCF model.
Expand All @@ -68,8 +68,8 @@ public interface EntityCache extends MaintenanceState, CleanState {
* Get RCF model total updates of specific entity
*
* @param detectorId detector id
* @param entityId entity id
* @param entityModelId entity model id
* @return RCF model total updates of specific entity
*/
long getTotalUpdates(String detectorId, String entityId);
long getTotalUpdates(String detectorId, String entityModelId);
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ public class DetectorProfile implements Writeable, ToXContentObject, Mergeable {
private String coordinatingNode;
private long totalSizeInBytes;
private InitProgressProfile initProgress;
private Long totalEntities;
private Long activeEntities;

public XContentBuilder toXContent(XContentBuilder builder) throws IOException {
return toXContent(builder, ToXContent.EMPTY_PARAMS);
Expand All @@ -62,6 +64,8 @@ public static class Builder {
private String coordinatingNode = null;
private long totalSizeInBytes = -1;
private InitProgressProfile initProgress = null;
private Long totalEntities;
private Long activeEntities;

public Builder() {}

Expand Down Expand Up @@ -100,6 +104,16 @@ public Builder initProgress(InitProgressProfile initProgress) {
return this;
}

public Builder totalEntities(Long totalEntities) {
this.totalEntities = totalEntities;
return this;
}

public Builder activeEntities(Long activeEntities) {
this.activeEntities = activeEntities;
return this;
}

public DetectorProfile build() {
DetectorProfile profile = new DetectorProfile();
profile.state = this.state;
Expand All @@ -109,6 +123,8 @@ public DetectorProfile build() {
profile.coordinatingNode = coordinatingNode;
profile.totalSizeInBytes = totalSizeInBytes;
profile.initProgress = initProgress;
profile.totalEntities = totalEntities;
profile.activeEntities = activeEntities;

return profile;
}
Expand Down Expand Up @@ -154,6 +170,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (initProgress != null) {
xContentBuilder.field(CommonName.INIT_PROGRESS, initProgress);
}
if (totalEntities != null) {
xContentBuilder.field(CommonName.TOTAL_ENTITIES, totalEntities);
}
if (activeEntities != null) {
xContentBuilder.field(CommonName.ACTIVE_ENTITIES, activeEntities);
}
return xContentBuilder.endObject();
}

Expand Down Expand Up @@ -213,6 +235,22 @@ public void setInitProgress(InitProgressProfile initProgress) {
this.initProgress = initProgress;
}

public Long getTotalEntities() {
return totalEntities;
}

public void setTotalEntities(Long totalEntities) {
this.totalEntities = totalEntities;
}

public Long getActiveEntities() {
return activeEntities;
}

public void setActiveEntities(Long activeEntities) {
this.activeEntities = activeEntities;
}

@Override
public void merge(Mergeable other) {
if (this == other || other == null || getClass() != other.getClass()) {
Expand Down Expand Up @@ -240,6 +278,12 @@ public void merge(Mergeable other) {
if (otherProfile.getInitProgress() != null) {
this.initProgress = otherProfile.getInitProgress();
}
if (otherProfile.getTotalEntities() != null) {
this.totalEntities = otherProfile.getTotalEntities();
}
if (otherProfile.getActiveEntities() != null) {
this.activeEntities = otherProfile.getActiveEntities();
}
}

@Override
Expand Down Expand Up @@ -275,6 +319,12 @@ public boolean equals(Object obj) {
if (initProgress != null) {
equalsBuilder.append(initProgress, other.initProgress);
}
if (totalEntities != null) {
equalsBuilder.append(totalEntities, other.totalEntities);
}
if (activeEntities != null) {
equalsBuilder.append(activeEntities, other.activeEntities);
}
return equalsBuilder.isEquals();
}
return false;
Expand All @@ -290,6 +340,8 @@ public int hashCode() {
.append(coordinatingNode)
.append(totalSizeInBytes)
.append(initProgress)
.append(totalEntities)
.append(activeEntities)
.toHashCode();
}

Expand Down Expand Up @@ -318,6 +370,12 @@ public String toString() {
if (initProgress != null) {
toStringBuilder.append(CommonName.INIT_PROGRESS, initProgress);
}
if (totalEntities != null) {
toStringBuilder.append(CommonName.TOTAL_ENTITIES, totalEntities);
}
if (activeEntities != null) {
toStringBuilder.append(CommonName.ACTIVE_ENTITIES, activeEntities);
}
return toStringBuilder.toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.ad.model;

import java.io.IOException;

import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;

/**
* Profile output for detector entity.
*/
public class EntityProfile implements Writeable, ToXContent {
// field name in toXContent
public static final String CATEGORY_FIELD = "category_field";
public static final String ENTITY_VALUE = "value";
public static final String IS_ACTIVE = "is_active";

private final String categoryField;
private final String value;
private final Boolean isActive;

public EntityProfile(String categoryField, String value, Boolean isActive) {
super();
this.categoryField = categoryField;
this.value = value;
this.isActive = isActive;
}

public EntityProfile(StreamInput in) throws IOException {
categoryField = in.readString();
value = in.readString();
isActive = in.readBoolean();
}

public String getCategoryField() {
return categoryField;
}

public String getValue() {
return value;
}

public Boolean getActive() {
return isActive;
}

public XContentBuilder toXContent(XContentBuilder builder) throws IOException {
return toXContent(builder, ToXContent.EMPTY_PARAMS);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(CATEGORY_FIELD, categoryField);
builder.field(ENTITY_VALUE, value);
builder.field(IS_ACTIVE, isActive);
builder.endObject();
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(categoryField);
out.writeString(value);
out.writeBoolean(isActive);
}

@Override
public String toString() {
ToStringBuilder builder = new ToStringBuilder(this);
builder.append(CATEGORY_FIELD, categoryField);
builder.append(ENTITY_VALUE, value);
builder.append(IS_ACTIVE, isActive);
return builder.toString();
}

@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
if (obj instanceof EntityProfile) {
EntityProfile other = (EntityProfile) obj;
EqualsBuilder equalsBuilder = new EqualsBuilder();
equalsBuilder.append(categoryField, other.categoryField);
equalsBuilder.append(value, other.value);
equalsBuilder.append(isActive, other.isActive);

return equalsBuilder.isEquals();
}
return false;
}

@Override
public int hashCode() {
return new HashCodeBuilder().append(categoryField).append(value).append(isActive).toHashCode();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ public enum ProfileName {
SHINGLE_SIZE(CommonName.SHINGLE_SIZE),
TOTAL_SIZE_IN_BYTES(CommonName.TOTAL_SIZE_IN_BYTES),
MODELS(CommonName.MODELS),
INIT_PROGRESS(CommonName.INIT_PROGRESS);
INIT_PROGRESS(CommonName.INIT_PROGRESS),
TOTAL_ENTITIES(CommonName.TOTAL_ENTITIES),
ACTIVE_ENTITIES(CommonName.ACTIVE_ENTITIES);

private String name;

Expand Down Expand Up @@ -61,6 +63,10 @@ public static ProfileName getName(String name) {
return MODELS;
case CommonName.INIT_PROGRESS:
return INIT_PROGRESS;
case CommonName.TOTAL_ENTITIES:
return TOTAL_ENTITIES;
case CommonName.ACTIVE_ENTITIES:
return ACTIVE_ENTITIES;
default:
throw new IllegalArgumentException("Unsupported profile types");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.amazon.opendistroforelasticsearch.ad.rest;

import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.DETECTOR_ID;
import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.ENTITY;
import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.PROFILE;
import static com.amazon.opendistroforelasticsearch.ad.util.RestHandlerUtils.TYPE;

Expand Down Expand Up @@ -60,6 +61,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
}
String detectorId = request.param(DETECTOR_ID);
String typesStr = request.param(TYPE);
String entityValue = request.param(ENTITY);
String rawPath = request.rawPath();
boolean returnJob = request.paramAsBoolean("job", false);
boolean all = request.paramAsBoolean("_all", false);
Expand All @@ -69,7 +71,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
returnJob,
typesStr,
rawPath,
all
all,
entityValue
);

return channel -> client
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ public enum StatNames {
ANOMALY_DETECTORS_INDEX_STATUS("anomaly_detectors_index_status"),
ANOMALY_RESULTS_INDEX_STATUS("anomaly_results_index_status"),
MODELS_CHECKPOINT_INDEX_STATUS("models_checkpoint_index_status"),
ANOMALY_DETECTION_JOB_INDEX_STATUS("anomaly_detection_job_index_status"),
ANOMALY_DETECTION_STATE_STATUS("anomaly_detection_state_status"),
MODEL_INFORMATION("models");

private String name;
Expand Down
Loading

0 comments on commit 3a9d6c9

Please sign in to comment.