Skip to content

Commit

Permalink
Merge branch 'master' into ccr
Browse files Browse the repository at this point in the history
* master:
  Painless: Add Bindings (#33042)
  Update version after client credentials backport
  Fix forbidden apis on FIPS (#33202)
  Remote 6.x transport BWC Layer for `_shrink` (#33236)
  Test fix - Graph HLRC tests needed another field adding to randomisation exception list
  HLRC: Add ML Get Records API (#33085)
  [ML] Fix character set finder bug with unencodable charsets (#33234)
  TESTS: Fix overly long lines (#33240)
  Test fix - Graph HLRC test was missing field name to be excluded from randomisation logic
  Remove unsupported group_shard_failures parameter (#33208)
  Update BucketUtils#suggestShardSideQueueSize signature (#33210)
  Parse PEM Key files leniantly (#33173)
  INGEST: Add Pipeline Processor (#32473)
  Core: Add java time xcontent serializers (#33120)
  Consider multi release jars when running third party audit (#33206)
  Update MSI documentation (#31950)
  HLRC: create base timed request class (#33216)
  [DOCS] Fixes command page titles
  HLRC: Move ML protocol classes into client ml package (#33203)
  Scroll queries asking for rescore are considered invalid (#32918)
  Painless: Fix Semicolon Regression (#33212)
  ingest: minor - update test to include dissect (#33211)
  Switch remaining LLREST usage to new style Requests (#33171)
  HLREST: add reindex API (#32679)
  • Loading branch information
dnhatn committed Aug 29, 2018
2 parents 41c7fc8 + b52818e commit 5632e31
Showing 222 changed files with 4,880 additions and 1,432 deletions.
Original file line number Diff line number Diff line change
@@ -87,6 +87,7 @@ class PrecommitTasks {
dependsOn(buildResources)
signatureFile = buildResources.copy("forbidden/third-party-audit.txt")
javaHome = project.runtimeJavaHome
targetCompatibility = project.runtimeJavaVersion
}
return thirdPartyAuditTask
}
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import org.elasticsearch.test.NamingConventionsCheck;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.JavaVersion;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.Input;
@@ -66,6 +67,17 @@ public class ThirdPartyAuditTask extends DefaultTask {

private String javaHome;

private JavaVersion targetCompatibility;

@Input
public JavaVersion getTargetCompatibility() {
return targetCompatibility;
}

public void setTargetCompatibility(JavaVersion targetCompatibility) {
this.targetCompatibility = targetCompatibility;
}

@InputFiles
public Configuration getForbiddenAPIsConfiguration() {
return getProject().getConfigurations().getByName("forbiddenApisCliJar");
@@ -157,10 +169,19 @@ public void runThirdPartyAudit() throws IOException {

private void extractJars(FileCollection jars) {
File jarExpandDir = getJarExpandDir();
// We need to clean up to make sure old dependencies don't linger
getProject().delete(jarExpandDir);
jars.forEach(jar ->
getProject().copy(spec -> {
spec.from(getProject().zipTree(jar));
spec.into(jarExpandDir);
// Exclude classes for multi release jars above target
for (int i = Integer.parseInt(targetCompatibility.getMajorVersion()) + 1;
i <= Integer.parseInt(JavaVersion.VERSION_HIGHER.getMajorVersion());
i++
) {
spec.exclude("META-INF/versions/" + i + "/**");
}
})
);
}
Original file line number Diff line number Diff line change
@@ -24,13 +24,14 @@
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.elasticsearch.client.RequestConverters.EndpointBuilder;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetJobRequest;
import org.elasticsearch.client.ml.GetRecordsRequest;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.protocol.xpack.ml.CloseJobRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.GetJobRequest;
import org.elasticsearch.protocol.xpack.ml.GetBucketsRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;

import java.io.IOException;

@@ -124,4 +125,18 @@ static Request getBuckets(GetBucketsRequest getBucketsRequest) throws IOExceptio
request.setEntity(createEntity(getBucketsRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}

static Request getRecords(GetRecordsRequest getRecordsRequest) throws IOException {
String endpoint = new EndpointBuilder()
.addPathPartAsIs("_xpack")
.addPathPartAsIs("ml")
.addPathPartAsIs("anomaly_detectors")
.addPathPart(getRecordsRequest.getJobId())
.addPathPartAsIs("results")
.addPathPartAsIs("records")
.build();
Request request = new Request(HttpGet.METHOD_NAME, endpoint);
request.setEntity(createEntity(getRecordsRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}
}
Original file line number Diff line number Diff line change
@@ -19,18 +19,20 @@
package org.elasticsearch.client;

import org.elasticsearch.action.ActionListener;
import org.elasticsearch.protocol.xpack.ml.CloseJobRequest;
import org.elasticsearch.protocol.xpack.ml.CloseJobResponse;
import org.elasticsearch.protocol.xpack.ml.DeleteJobRequest;
import org.elasticsearch.protocol.xpack.ml.DeleteJobResponse;
import org.elasticsearch.protocol.xpack.ml.GetBucketsRequest;
import org.elasticsearch.protocol.xpack.ml.GetBucketsResponse;
import org.elasticsearch.protocol.xpack.ml.GetJobRequest;
import org.elasticsearch.protocol.xpack.ml.GetJobResponse;
import org.elasticsearch.protocol.xpack.ml.OpenJobRequest;
import org.elasticsearch.protocol.xpack.ml.OpenJobResponse;
import org.elasticsearch.protocol.xpack.ml.PutJobRequest;
import org.elasticsearch.protocol.xpack.ml.PutJobResponse;
import org.elasticsearch.client.ml.CloseJobRequest;
import org.elasticsearch.client.ml.CloseJobResponse;
import org.elasticsearch.client.ml.DeleteJobRequest;
import org.elasticsearch.client.ml.DeleteJobResponse;
import org.elasticsearch.client.ml.GetBucketsRequest;
import org.elasticsearch.client.ml.GetBucketsResponse;
import org.elasticsearch.client.ml.GetJobRequest;
import org.elasticsearch.client.ml.GetJobResponse;
import org.elasticsearch.client.ml.GetRecordsRequest;
import org.elasticsearch.client.ml.GetRecordsResponse;
import org.elasticsearch.client.ml.OpenJobRequest;
import org.elasticsearch.client.ml.OpenJobResponse;
import org.elasticsearch.client.ml.PutJobRequest;
import org.elasticsearch.client.ml.PutJobResponse;

import java.io.IOException;
import java.util.Collections;
@@ -56,9 +58,9 @@ public final class MachineLearningClient {
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
*
* @param request The PutJobRequest containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
* @param request The PutJobRequest containing the {@link org.elasticsearch.client.ml.job.config.Job} settings
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return PutJobResponse with enclosed {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} object
* @return PutJobResponse with enclosed {@link org.elasticsearch.client.ml.job.config.Job} object
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public PutJobResponse putJob(PutJobRequest request, RequestOptions options) throws IOException {
@@ -75,7 +77,7 @@ public PutJobResponse putJob(PutJobRequest request, RequestOptions options) thro
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html">ML PUT job documentation</a>
*
* @param request The request containing the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} settings
* @param request The request containing the {@link org.elasticsearch.client.ml.job.config.Job} settings
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
@@ -98,7 +100,7 @@ public void putJobAsync(PutJobRequest request, RequestOptions options, ActionLis
* @param request {@link GetJobRequest} Request containing a list of jobId(s) and additional options
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return {@link GetJobResponse} response object containing
* the {@link org.elasticsearch.protocol.xpack.ml.job.config.Job} objects and the number of jobs found
* the {@link org.elasticsearch.client.ml.job.config.Job} objects and the number of jobs found
* @throws IOException when there is a serialization issue sending the request or receiving the response
*/
public GetJobResponse getJob(GetJobRequest request, RequestOptions options) throws IOException {
@@ -285,4 +287,40 @@ public void getBucketsAsync(GetBucketsRequest request, RequestOptions options, A
listener,
Collections.emptySet());
}

/**
* Gets the records for a Machine Learning Job.
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-record.html">ML GET records documentation</a>
*
* @param request the request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
*/
public GetRecordsResponse getRecords(GetRecordsRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(request,
MLRequestConverters::getRecords,
options,
GetRecordsResponse::fromXContent,
Collections.emptySet());
}

/**
* Gets the records for a Machine Learning Job, notifies listener once the requested records are retrieved.
* <p>
* For additional info
* see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-record.html">ML GET records documentation</a>
*
* @param request the request
* @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener Listener to be notified upon request completion
*/
public void getRecordsAsync(GetRecordsRequest request, RequestOptions options, ActionListener<GetRecordsResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(request,
MLRequestConverters::getRecords,
options,
GetRecordsResponse::fromXContent,
listener,
Collections.emptySet());
}
}
Original file line number Diff line number Diff line change
@@ -106,6 +106,7 @@
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.rankeval.RankEvalRequest;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.elasticsearch.protocol.xpack.XPackInfoRequest;
import org.elasticsearch.protocol.xpack.XPackUsageRequest;
import org.elasticsearch.protocol.xpack.license.DeleteLicenseRequest;
@@ -820,6 +821,21 @@ static Request clusterHealth(ClusterHealthRequest healthRequest) {
return request;
}

static Request reindex(ReindexRequest reindexRequest) throws IOException {
String endpoint = new EndpointBuilder().addPathPart("_reindex").build();
Request request = new Request(HttpPost.METHOD_NAME, endpoint);
Params params = new Params(request)
.withRefresh(reindexRequest.isRefresh())
.withTimeout(reindexRequest.getTimeout())
.withWaitForActiveShards(reindexRequest.getWaitForActiveShards());

if (reindexRequest.getScrollTime() != null) {
params.putParam("scroll", reindexRequest.getScrollTime());
}
request.setEntity(createEntity(reindexRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}

static Request rollover(RolloverRequest rolloverRequest) throws IOException {
String endpoint = new EndpointBuilder().addPathPart(rolloverRequest.getAlias()).addPathPartAsIs("_rollover")
.addPathPart(rolloverRequest.getNewIndexName()).build();
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.rankeval.RankEvalRequest;
import org.elasticsearch.index.rankeval.RankEvalResponse;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.ReindexRequest;
import org.elasticsearch.plugins.spi.NamedXContentProvider;
import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestStatus;
@@ -395,6 +397,33 @@ public final void bulkAsync(BulkRequest bulkRequest, RequestOptions options, Act
performRequestAsyncAndParseEntity(bulkRequest, RequestConverters::bulk, options, BulkResponse::fromXContent, listener, emptySet());
}

/**
* Executes a reindex request.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html">Reindex API on elastic.co</a>
* @param reindexRequest the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public final BulkByScrollResponse reindex(ReindexRequest reindexRequest, RequestOptions options) throws IOException {
return performRequestAndParseEntity(
reindexRequest, RequestConverters::reindex, options, BulkByScrollResponse::fromXContent, emptySet()
);
}

/**
* Asynchronously executes a reindex request.
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html">Reindex API on elastic.co</a>
* @param reindexRequest the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @param listener the listener to be notified upon request completion
*/
public final void reindexAsync(ReindexRequest reindexRequest, RequestOptions options, ActionListener<BulkByScrollResponse> listener) {
performRequestAsyncAndParseEntity(
reindexRequest, RequestConverters::reindex, options, BulkByScrollResponse::fromXContent, listener, emptySet()
);
}

/**
* Pings the remote Elasticsearch cluster and returns true if the ping succeeded, false otherwise
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client;

import org.elasticsearch.common.unit.TimeValue;

/**
* A base request for any requests that supply timeouts.
*
* Please note, any requests that use a ackTimeout should set timeout as they
* represent the same backing field on the server.
*/
public class TimedRequest implements Validatable {

private TimeValue timeout;
private TimeValue masterTimeout;

public void setTimeout(TimeValue timeout) {
this.timeout = timeout;

}

public void setMasterTimeout(TimeValue masterTimeout) {
this.masterTimeout = masterTimeout;
}

/**
* Returns the request timeout
*/
public TimeValue timeout() {
return timeout;
}

/**
* Returns the timeout for the request to be completed on the master node
*/
public TimeValue masterNodeTimeout() {
return masterTimeout;
}
}
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.ParseField;
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.ParseField;
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.common.xcontent.XContentParser;
Original file line number Diff line number Diff line change
@@ -16,17 +16,17 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.protocol.xpack.ml;
package org.elasticsearch.client.ml;

import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.client.ml.job.config.Job;
import org.elasticsearch.client.ml.job.results.Result;
import org.elasticsearch.client.ml.job.util.PageParams;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.protocol.xpack.ml.job.config.Job;
import org.elasticsearch.protocol.xpack.ml.job.results.Result;
import org.elasticsearch.protocol.xpack.ml.job.util.PageParams;

import java.io.IOException;
import java.util.Objects;
Loading

0 comments on commit 5632e31

Please sign in to comment.