Skip to content

Commit

Permalink
Merge branch 'master' into feature/reader-context
Browse files Browse the repository at this point in the history
  • Loading branch information
dnhatn committed Mar 2, 2020
2 parents cdbfb67 + 0b2a628 commit f921838
Show file tree
Hide file tree
Showing 166 changed files with 4,288 additions and 1,043 deletions.
9 changes: 9 additions & 0 deletions benchmarks/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,12 @@ thirdPartyAudit.ignoreViolations(
'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
'org.openjdk.jmh.util.Utils'
)

spotless {
java {
// IDEs can sometimes run annotation processors that leave files in
// here, causing Spotless to complain. Even though this path ought not
// to exist, exclude it anyway in order to avoid spurious failures.
targetExclude 'src/main/generated/**/*.java'
}
}
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ task verifyVersions {
* after the backport of the backcompat code is complete.
*/

boolean bwc_tests_enabled = true
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
boolean bwc_tests_enabled = false
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/52998" /* place a PR link here when committing bwc changes */
if (bwc_tests_enabled == false) {
if (bwc_tests_disabled_issue.isEmpty()) {
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 8.0.0
lucene = 8.5.0-snapshot-b01d7cb
lucene = 8.5.0-snapshot-c4475920b08

bundled_jdk_vendor = adoptopenjdk
bundled_jdk = 13.0.2+8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,32 +36,32 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
private String[] indices;
private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, false);

private QueryBuilder query = null;
private QueryBuilder filter = null;
private String timestampField = "@timestamp";
private String eventTypeField = "event_type";
private String implicitJoinKeyField = "agent.id";
private int fetchSize = 50;
private SearchAfterBuilder searchAfterBuilder;
private String rule;
private String query;

static final String KEY_QUERY = "query";
static final String KEY_FILTER = "filter";
static final String KEY_TIMESTAMP_FIELD = "timestamp_field";
static final String KEY_EVENT_TYPE_FIELD = "event_type_field";
static final String KEY_IMPLICIT_JOIN_KEY_FIELD = "implicit_join_key_field";
static final String KEY_SIZE = "size";
static final String KEY_SEARCH_AFTER = "search_after";
static final String KEY_RULE = "rule";
static final String KEY_QUERY = "query";

public EqlSearchRequest(String indices, String rule) {
public EqlSearchRequest(String indices, String query) {
indices(indices);
rule(rule);
query(query);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject();
if (query != null) {
builder.field(KEY_QUERY, query);
if (filter != null) {
builder.field(KEY_FILTER, filter);
}
builder.field(KEY_TIMESTAMP_FIELD, timestampField());
builder.field(KEY_EVENT_TYPE_FIELD, eventTypeField());
Expand All @@ -74,7 +74,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
builder.array(KEY_SEARCH_AFTER, searchAfterBuilder.getSortValues());
}

builder.field(KEY_RULE, rule);
builder.field(KEY_QUERY, query);
builder.endObject();
return builder;
}
Expand All @@ -88,12 +88,12 @@ public EqlSearchRequest indices(String... indices) {
return this;
}

public QueryBuilder query() {
return this.query;
public QueryBuilder filter() {
return this.filter;
}

public EqlSearchRequest query(QueryBuilder query) {
this.query = query;
public EqlSearchRequest filter(QueryBuilder filter) {
this.filter = filter;
return this;
}

Expand Down Expand Up @@ -156,13 +156,13 @@ private EqlSearchRequest setSearchAfter(SearchAfterBuilder builder) {
return this;
}

public String rule() {
return this.rule;
public String query() {
return this.query;
}

public EqlSearchRequest rule(String rule) {
Objects.requireNonNull(rule, "rule must not be null");
this.rule = rule;
public EqlSearchRequest query(String query) {
Objects.requireNonNull(query, "query must not be null");
this.query = query;
return this;
}

Expand All @@ -175,30 +175,29 @@ public boolean equals(Object o) {
return false;
}
EqlSearchRequest that = (EqlSearchRequest) o;
return
fetchSize == that.fetchSize &&
return fetchSize == that.fetchSize &&
Arrays.equals(indices, that.indices) &&
Objects.equals(indicesOptions, that.indicesOptions) &&
Objects.equals(query, that.query) &&
Objects.equals(filter, that.filter) &&
Objects.equals(timestampField, that.timestampField) &&
Objects.equals(eventTypeField, that.eventTypeField) &&
Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) &&
Objects.equals(searchAfterBuilder, that.searchAfterBuilder) &&
Objects.equals(rule, that.rule);
Objects.equals(query, that.query);
}

@Override
public int hashCode() {
return Objects.hash(
Arrays.hashCode(indices),
indicesOptions,
query,
filter,
fetchSize,
timestampField,
eventTypeField,
implicitJoinKeyField,
searchAfterBuilder,
rule);
query);
}

public String[] indices() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
static final ParseField STATE = new ParseField("state");
static final ParseField FAILURE_REASON = new ParseField("failure_reason");
static final ParseField PROGRESS = new ParseField("progress");
static final ParseField MEMORY_USAGE = new ParseField("memory_usage");
static final ParseField NODE = new ParseField("node");
static final ParseField ASSIGNMENT_EXPLANATION = new ParseField("assignment_explanation");

Expand All @@ -55,8 +56,9 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
(DataFrameAnalyticsState) args[1],
(String) args[2],
(List<PhaseProgress>) args[3],
(NodeAttributes) args[4],
(String) args[5]));
(MemoryUsage) args[4],
(NodeAttributes) args[5],
(String) args[6]));

static {
PARSER.declareString(constructorArg(), ID);
Expand All @@ -68,6 +70,7 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
}, STATE, ObjectParser.ValueType.STRING);
PARSER.declareString(optionalConstructorArg(), FAILURE_REASON);
PARSER.declareObjectArray(optionalConstructorArg(), PhaseProgress.PARSER, PROGRESS);
PARSER.declareObject(optionalConstructorArg(), MemoryUsage.PARSER, MEMORY_USAGE);
PARSER.declareObject(optionalConstructorArg(), NodeAttributes.PARSER, NODE);
PARSER.declareString(optionalConstructorArg(), ASSIGNMENT_EXPLANATION);
}
Expand All @@ -76,16 +79,18 @@ public static DataFrameAnalyticsStats fromXContent(XContentParser parser) throws
private final DataFrameAnalyticsState state;
private final String failureReason;
private final List<PhaseProgress> progress;
private final MemoryUsage memoryUsage;
private final NodeAttributes node;
private final String assignmentExplanation;

public DataFrameAnalyticsStats(String id, DataFrameAnalyticsState state, @Nullable String failureReason,
@Nullable List<PhaseProgress> progress, @Nullable NodeAttributes node,
@Nullable String assignmentExplanation) {
@Nullable List<PhaseProgress> progress, @Nullable MemoryUsage memoryUsage,
@Nullable NodeAttributes node, @Nullable String assignmentExplanation) {
this.id = id;
this.state = state;
this.failureReason = failureReason;
this.progress = progress;
this.memoryUsage = memoryUsage;
this.node = node;
this.assignmentExplanation = assignmentExplanation;
}
Expand All @@ -106,6 +111,11 @@ public List<PhaseProgress> getProgress() {
return progress;
}

@Nullable
public MemoryUsage getMemoryUsage() {
return memoryUsage;
}

public NodeAttributes getNode() {
return node;
}
Expand All @@ -124,13 +134,14 @@ public boolean equals(Object o) {
&& Objects.equals(state, other.state)
&& Objects.equals(failureReason, other.failureReason)
&& Objects.equals(progress, other.progress)
&& Objects.equals(memoryUsage, other.memoryUsage)
&& Objects.equals(node, other.node)
&& Objects.equals(assignmentExplanation, other.assignmentExplanation);
}

@Override
public int hashCode() {
return Objects.hash(id, state, failureReason, progress, node, assignmentExplanation);
return Objects.hash(id, state, failureReason, progress, memoryUsage, node, assignmentExplanation);
}

@Override
Expand All @@ -140,6 +151,7 @@ public String toString() {
.add("state", state)
.add("failureReason", failureReason)
.add("progress", progress)
.add("memoryUsage", memoryUsage)
.add("node", node)
.add("assignmentExplanation", assignmentExplanation)
.toString();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ml.dataframe;

import org.elasticsearch.client.common.TimeUtil;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.inject.internal.ToStringBuilder;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;

import java.io.IOException;
import java.time.Instant;
import java.util.Objects;

public class MemoryUsage implements ToXContentObject {

static final ParseField TIMESTAMP = new ParseField("timestamp");
static final ParseField PEAK_USAGE_BYTES = new ParseField("peak_usage_bytes");

public static final ConstructingObjectParser<MemoryUsage, Void> PARSER = new ConstructingObjectParser<>("analytics_memory_usage",
true, a -> new MemoryUsage((Instant) a[0], (long) a[1]));

static {
PARSER.declareField(ConstructingObjectParser.constructorArg(),
p -> TimeUtil.parseTimeFieldToInstant(p, TIMESTAMP.getPreferredName()),
TIMESTAMP,
ObjectParser.ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.constructorArg(), PEAK_USAGE_BYTES);
}

private final Instant timestamp;
private final long peakUsageBytes;

public MemoryUsage(Instant timestamp, long peakUsageBytes) {
this.timestamp = Instant.ofEpochMilli(Objects.requireNonNull(timestamp).toEpochMilli());
this.peakUsageBytes = peakUsageBytes;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.toEpochMilli());
builder.field(PEAK_USAGE_BYTES.getPreferredName(), peakUsageBytes);
builder.endObject();
return builder;
}

@Override
public boolean equals(Object o) {
if (o == this) return true;
if (o == null || getClass() != o.getClass()) return false;

MemoryUsage other = (MemoryUsage) o;
return Objects.equals(timestamp, other.timestamp)
&& peakUsageBytes == other.peakUsageBytes;
}

@Override
public int hashCode() {
return Objects.hash(timestamp, peakUsageBytes);
}

@Override
public String toString() {
return new ToStringBuilder(getClass())
.add(TIMESTAMP.getPreferredName(), timestamp.getEpochSecond())
.add(PEAK_USAGE_BYTES.getPreferredName(), peakUsageBytes)
.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1506,6 +1506,7 @@ public void testGetDataFrameAnalyticsStats() throws Exception {
assertThat(progress.get(1), equalTo(new PhaseProgress("loading_data", 0)));
assertThat(progress.get(2), equalTo(new PhaseProgress("analyzing", 0)));
assertThat(progress.get(3), equalTo(new PhaseProgress("writing_results", 0)));
assertThat(stats.getMemoryUsage(), is(nullValue()));
}

public void testStartDataFrameAnalyticsConfig() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ protected EqlSearchRequest createClientTestInstance() {
EqlSearchRequest.eventTypeField(randomAlphaOfLength(10));
}
if (randomBoolean()) {
EqlSearchRequest.rule(randomAlphaOfLength(10));
EqlSearchRequest.query(randomAlphaOfLength(10));
}
if (randomBoolean()) {
EqlSearchRequest.timestampField(randomAlphaOfLength(10));
Expand All @@ -56,9 +56,9 @@ protected EqlSearchRequest createClientTestInstance() {
}
if (randomBoolean()) {
if (randomBoolean()) {
EqlSearchRequest.query(QueryBuilders.matchAllQuery());
EqlSearchRequest.filter(QueryBuilders.matchAllQuery());
} else {
EqlSearchRequest.query(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100)));
EqlSearchRequest.filter(QueryBuilders.termQuery(randomAlphaOfLength(10), randomInt(100)));
}
}
return EqlSearchRequest;
Expand All @@ -75,8 +75,8 @@ protected void assertInstances(org.elasticsearch.xpack.eql.action.EqlSearchReque
assertThat(serverInstance.eventTypeField(), equalTo(clientTestInstance.eventTypeField()));
assertThat(serverInstance.implicitJoinKeyField(), equalTo(clientTestInstance.implicitJoinKeyField()));
assertThat(serverInstance.timestampField(), equalTo(clientTestInstance.timestampField()));
assertThat(serverInstance.filter(), equalTo(clientTestInstance.filter()));
assertThat(serverInstance.query(), equalTo(clientTestInstance.query()));
assertThat(serverInstance.rule(), equalTo(clientTestInstance.rule()));
assertThat(serverInstance.searchAfter(), equalTo(clientTestInstance.searchAfter()));
assertThat(serverInstance.indicesOptions(), equalTo(clientTestInstance.indicesOptions()));
assertThat(serverInstance.indices(), equalTo(clientTestInstance.indices()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public static DataFrameAnalyticsStats randomDataFrameAnalyticsStats() {
randomFrom(DataFrameAnalyticsState.values()),
randomBoolean() ? null : randomAlphaOfLength(10),
randomBoolean() ? null : createRandomProgress(),
randomBoolean() ? null : MemoryUsageTests.createRandom(),
randomBoolean() ? null : NodeAttributesTests.createRandom(),
randomBoolean() ? null : randomAlphaOfLengthBetween(1, 20));
}
Expand All @@ -70,6 +71,9 @@ public static void toXContent(DataFrameAnalyticsStats stats, XContentBuilder bui
if (stats.getProgress() != null) {
builder.field(DataFrameAnalyticsStats.PROGRESS.getPreferredName(), stats.getProgress());
}
if (stats.getMemoryUsage() != null) {
builder.field(DataFrameAnalyticsStats.MEMORY_USAGE.getPreferredName(), stats.getMemoryUsage());
}
if (stats.getNode() != null) {
builder.field(DataFrameAnalyticsStats.NODE.getPreferredName(), stats.getNode());
}
Expand Down
Loading

0 comments on commit f921838

Please sign in to comment.