Merge remote-tracking branch 'elastic/master' into state-transfer-only

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -75,6 +75,8 @@ public ElasticsearchCluster(String path, String clusterName, Project project, Fi
                 services, artifactsExtractDir, workingDirBase
             )
         );
+        // configure the cluster name eagerly so nodes know about it
+        this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
         addWaitForClusterHealth();
     }
@@ -217,7 +219,6 @@ public void setJavaHome(File javaHome) {
     public void start() {
         String nodeNames = nodes.stream().map(ElasticsearchNode::getName).collect(Collectors.joining(","));
         for (ElasticsearchNode node : nodes) {
-            node.defaultConfig.put("cluster.name", safeName(clusterName));
             if (Version.fromString(node.getVersion()).getMajor() >= 7) {
                 node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
                 node.defaultConfig.put("discovery.seed_providers", "file");
@@ -328,7 +329,8 @@ private void addWaitForClusterHealth() {
                     nodes.size()
                 );
                 if (httpSslEnabled) {
-                    wait.setCertificateAuthorities(getFirstNode().getHttpCertificateAuthoritiesFile());
+
+                    getFirstNode().configureHttpWait(wait);
                 }
                 List<Map<String, String>> credentials = getFirstNode().getCredentials();
                 if (getFirstNode().getCredentials().isEmpty() == false) {

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.gradle.FileSupplier;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.http.WaitForHttpResource;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 
@@ -581,7 +582,11 @@ public List<String> getAllTransportPortURI() {
     }
 
     public File getServerLog() {
-        return confPathLogs.resolve(safeName(getName()).replaceAll("-[0-9]+$", "") + "_server.json").toFile();
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_server.json").toFile();
+    }
+
+    public File getAuditLog() {
+        return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_audit.json").toFile();
     }
 
     @Override
@@ -880,12 +885,32 @@ public boolean isHttpSslEnabled() {
         );
     }
 
-    public File getHttpCertificateAuthoritiesFile() {
-        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities") == false) {
-            throw new TestClustersException("Can't get certificates authority file, not configured for " + this);
+    void configureHttpWait(WaitForHttpResource wait) {
+        if (settings.containsKey("xpack.security.http.ssl.certificate_authorities")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate_authorities").get().toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.certificate")) {
+            wait.setCertificateAuthorities(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.certificate").get().toString())
+                    .toFile()
+            );
+        }
+        if (settings.containsKey("xpack.security.http.ssl.keystore.path")) {
+            wait.setTrustStoreFile(
+                getConfigDir()
+                    .resolve(settings.get("xpack.security.http.ssl.keystore.path").get().toString())
+                    .toFile()
+            );
+        }
+        if (keystoreSettings.containsKey("xpack.security.http.ssl.keystore.secure_password")) {
+            wait.setTrustStorePassword(
+                keystoreSettings.get("xpack.security.http.ssl.keystore.secure_password").get().toString()
+            );
         }
-        return getConfigDir()
-            .resolve(settings.get("xpack.security.http.ssl.certificate_authorities").get().toString())
-            .toFile();
     }
 }

client/rest-high-level/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'nebula.maven-base-publish'
@@ -115,14 +116,15 @@ if (isEclipse) {
 File nodeCert = file("./testnode.crt")
 File nodeTrustStore = file("./testnode.jks")
 
-integTestRunner {
+integTest.runner {
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
   systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')
 }
 
-integTestCluster {
+testClusters.integTest {
+  distribution = "DEFAULT"
   systemProperty 'es.scripting.update.ctx_in_params', 'false'
-  setting 'reindex.remote.whitelist', ['"[::1]:*"', '"127.0.0.1:*"']
+  setting 'reindex.remote.whitelist', '[ "[::1]:*", "127.0.0.1:*" ]'
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.authc.token.enabled', 'true'
@@ -131,22 +133,10 @@ integTestCluster {
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
   setting 'indices.lifecycle.poll_interval', '1000ms'
-  keystoreSetting 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
-  setupCommand 'setupDummyUser',
-          'bin/elasticsearch-users',
-          'useradd', System.getProperty('tests.rest.cluster.username', 'test_user'),
-          '-p', System.getProperty('tests.rest.cluster.password', 'test-password'),
-          '-r', 'superuser'
+  keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
+  user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
+          password:  System.getProperty('tests.rest.cluster.password', 'test-password')
+
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore
-  waitCondition = { node, ant ->
-    File tmpFile = new File(node.cwd, 'wait.success')
-    ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
-            dest: tmpFile.toString(),
-            username: System.getProperty('tests.rest.cluster.username', 'test_user'),
-            password: System.getProperty('tests.rest.cluster.password', 'test-password'),
-            ignoreerrors: true,
-            retries: 10)
-    return tmpFile.exists()
-  }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DestConfig.java

@@ -28,36 +28,48 @@
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Configuration containing the destination index for the {@link DataFrameTransformConfig}
  */
 public class DestConfig implements ToXContentObject {
 
     public static final ParseField INDEX = new ParseField("index");
+    public static final ParseField PIPELINE = new ParseField("pipeline");
 
     public static final ConstructingObjectParser<DestConfig, Void> PARSER = new ConstructingObjectParser<>("data_frame_config_dest",
         true,
-        args -> new DestConfig((String)args[0]));
+        args -> new DestConfig((String)args[0], (String)args[1]));
 
     static {
         PARSER.declareString(constructorArg(), INDEX);
+        PARSER.declareString(optionalConstructorArg(), PIPELINE);
     }
 
     private final String index;
+    private final String pipeline;
 
-    public DestConfig(String index) {
+    DestConfig(String index, String pipeline) {
         this.index = Objects.requireNonNull(index, INDEX.getPreferredName());
+        this.pipeline = pipeline;
     }
 
     public String getIndex() {
         return index;
     }
 
+    public String getPipeline() {
+        return pipeline;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(INDEX.getPreferredName(), index);
+        if (pipeline != null) {
+            builder.field(PIPELINE.getPreferredName(), pipeline);
+        }
         builder.endObject();
         return builder;
     }
@@ -72,11 +84,45 @@ public boolean equals(Object other) {
         }
 
         DestConfig that = (DestConfig) other;
-        return Objects.equals(index, that.index);
+        return Objects.equals(index, that.index) &&
+            Objects.equals(pipeline, that.pipeline);
     }
 
     @Override
     public int hashCode(){
-        return Objects.hash(index);
+        return Objects.hash(index, pipeline);
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String index;
+        private String pipeline;
+
+        /**
+         * Sets which index to which to write the data
+         * @param index where to write the data
+         * @return The {@link Builder} with index set
+         */
+        public Builder setIndex(String index) {
+            this.index = Objects.requireNonNull(index, INDEX.getPreferredName());
+            return this;
+        }
+
+        /**
+         * Sets the pipeline through which the indexed documents should be processed
+         * @param pipeline The pipeline ID
+         * @return The {@link Builder} with pipeline set
+         */
+        public Builder setPipeline(String pipeline) {
+            this.pipeline = pipeline;
+            return this;
+        }
+
+        public DestConfig build() {
+            return new DestConfig(index, pipeline);
+        }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

@@ -307,7 +307,7 @@ private DataFrameTransformConfig validDataFrameTransformConfig(String id, String
         aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars"));
         PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build();
 
-        DestConfig destConfig = (destination != null) ? new DestConfig(destination) : null;
+        DestConfig destConfig = (destination != null) ? DestConfig.builder().setIndex(destination).build() : null;
 
         return DataFrameTransformConfig.builder()
             .setId(id)
@@ -318,6 +318,7 @@ private DataFrameTransformConfig validDataFrameTransformConfig(String id, String
             .build();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43324")
     public void testGetStats() throws Exception {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);
@@ -333,7 +334,7 @@ public void testGetStats() throws Exception {
         DataFrameTransformConfig transform = DataFrameTransformConfig.builder()
             .setId(id)
             .setSource(SourceConfig.builder().setIndex(sourceIndex).setQuery(new MatchAllQueryBuilder()).build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .setDescription("transform for testing stats")
             .build();

client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java

@@ -1286,7 +1286,7 @@ public void testGetFilters() throws Exception {
             GetFiltersResponse getFiltersResponse = execute(getFiltersRequest,
                 machineLearningClient::getFilter,
                 machineLearningClient::getFilterAsync);
-            assertThat(getFiltersResponse.count(), equalTo(2L));
+            assertThat(getFiltersResponse.count(), equalTo(3L));
             assertThat(getFiltersResponse.filters().size(), equalTo(2));
             assertThat(getFiltersResponse.filters().stream().map(MlFilter::getId).collect(Collectors.toList()),
                 containsInAnyOrder("get-filter-test-2", "get-filter-test-3"));

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DestConfigTests.java

@@ -27,7 +27,8 @@
 public class DestConfigTests extends AbstractXContentTestCase<DestConfig> {
 
     public static DestConfig randomDestConfig() {
-        return new DestConfig(randomAlphaOfLength(10));
+        return new DestConfig(randomAlphaOfLength(10),
+            randomBoolean() ? null : randomAlphaOfLength(10));
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

@@ -125,6 +125,11 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             .setIndex("source-index")
             .setQueryConfig(queryConfig).build();
         // end::put-data-frame-transform-source-config
+        // tag::put-data-frame-transform-dest-config
+        DestConfig destConfig = DestConfig.builder()
+            .setIndex("pivot-destination")
+            .setPipeline("my-pipeline").build();
+        // end::put-data-frame-transform-dest-config
         // tag::put-data-frame-transform-group-config
         GroupConfig groupConfig = GroupConfig.builder()
             .groupBy("reviewer", // <1>
@@ -149,7 +154,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             .builder()
             .setId("reviewer-avg-rating") // <1>
             .setSource(sourceConfig) // <2>
-            .setDest(new DestConfig("pivot-destination")) // <3>
+            .setDest(destConfig) // <3>
             .setPivotConfig(pivotConfig) // <4>
             .setDescription("This is my test transform") // <5>
             .build();
@@ -222,7 +227,7 @@ public void testStartStop() throws IOException, InterruptedException {
         DataFrameTransformConfig transformConfig = DataFrameTransformConfig.builder()
             .setId("mega-transform")
             .setSource(SourceConfig.builder().setIndex("source-data").setQueryConfig(queryConfig).build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
 
@@ -344,7 +349,7 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
         DataFrameTransformConfig transformConfig2 = DataFrameTransformConfig.builder()
@@ -353,7 +358,7 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest2"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest2").build())
             .setPivotConfig(pivotConfig)
             .build();
 
@@ -488,7 +493,7 @@ public void testGetStats() throws IOException, InterruptedException {
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
@@ -574,7 +579,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
                 .setIndex("source-data")
                 .setQuery(new MatchAllQueryBuilder())
                 .build())
-            .setDest(new DestConfig("pivot-dest"))
+            .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
             .build();
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java

@@ -452,7 +452,7 @@ public void testSnapshotVerifyRepository() throws IOException {
         List<VerifyRepositoryResponse.NodeView> repositoryMetaDataResponse = response.getNodes();
         // end::verify-repository-response
         assertThat(1, equalTo(repositoryMetaDataResponse.size()));
-        assertThat("node-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
+        assertThat("integTest-0", equalTo(repositoryMetaDataResponse.get(0).getName()));
     }
 
     public void testSnapshotVerifyRepositoryAsync() throws InterruptedException {

docs/java-rest/high-level/dataframe/put_data_frame.asciidoc

@@ -33,7 +33,7 @@ include-tagged::{doc-tests-file}[{api}-config]
 --------------------------------------------------
 <1> The {dataframe-transform} ID
 <2> The source indices and query from which to gather data
-<3> The destination index
+<3> The destination index and optional pipeline
 <4> The PivotConfig
 <5> Optional free text description of the transform
 
@@ -49,6 +49,16 @@ If query is not set, a `match_all` query is used by default.
 include-tagged::{doc-tests-file}[{api}-source-config]
 --------------------------------------------------
 
+==== DestConfig
+
+The index where to write the data and the optional pipeline
+through which the docs should be indexed
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-dest-config]
+--------------------------------------------------
+
 ===== QueryConfig
 
 The query with which to select data from the source.