From ace1d40e21b56ae67fabd5eac44f203538ee4648 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 13 Jun 2019 19:32:35 +0300 Subject: [PATCH 01/11] Refactor Version class to make version bumps easier (#42668) With this change we only have to add one line to add a new version. The intent is to make it less error prone and easier to write a script to automate the process. --- .../main/java/org/elasticsearch/Version.java | 76 +++++++++++-------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 7c48b38a8cf90..1396d3ffd2c82 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; @@ -46,28 +47,52 @@ public class Version implements Comparable, ToXContentFragment { */ public static final int V_EMPTY_ID = 0; public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST); - public static final int V_7_0_0_ID = 7000099; - public static final Version V_7_0_0 = new Version(V_7_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_0_1_ID = 7000199; - public static final Version V_7_0_1 = new Version(V_7_0_1_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_1_0_ID = 7010099; - public static final Version V_7_1_0 = new Version(V_7_1_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_1_1_ID = 7010199; - public static final Version V_7_1_1 = new Version(V_7_1_1_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_1_2_ID = 7010299; - public static final Version V_7_1_2 = new Version(V_7_1_2_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_2_0_ID = 7020099; - public static final Version V_7_2_0 = new Version(V_7_2_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0); - public static final int V_7_3_0_ID = 7030099; - public static final Version V_7_3_0 = new Version(V_7_3_0_ID, org.apache.lucene.util.Version.LUCENE_8_1_0); - public static final int V_8_0_0_ID = 8000099; - public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_1_0); + public static final Version V_7_0_0 = new Version(7000099, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_0_1 = new Version(7000199, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_1_0 = new Version(7010099, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_1_1 = new Version(7010199, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_1_2 = new Version(7010299, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_2_0 = new Version(7020099, org.apache.lucene.util.Version.LUCENE_8_0_0); + public static final Version V_7_3_0 = new Version(7030099, org.apache.lucene.util.Version.LUCENE_8_1_0); + public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_1_0); public static final Version CURRENT = V_8_0_0; + private static final ImmutableOpenIntMap idToVersion; static { + final ImmutableOpenIntMap.Builder builder = ImmutableOpenIntMap.builder(); + + for (final Field declaredField : Version.class.getFields()) { + if (declaredField.getType().equals(Version.class)) { + final String fieldName = declaredField.getName(); + if (fieldName.equals("CURRENT") || fieldName.equals("V_EMPTY")) { + continue; + } + assert fieldName.matches("V_\\d+_\\d+_\\d+") + : "expected Version field [" + fieldName + "] to match V_\\d+_\\d+_\\d+"; + try { + final Version version = (Version) declaredField.get(null); + if (Assertions.ENABLED) { + final String[] fields = fieldName.split("_"); + final int major = Integer.valueOf(fields[1]) * 1000000; + final int minor = Integer.valueOf(fields[2]) * 10000; + final int revision = Integer.valueOf(fields[3]) * 100; + final int expectedId = major + minor + revision + 99; + assert version.id == expectedId : + "expected version [" + fieldName + "] to have id [" + expectedId + "] but was [" + version.id + "]"; + } + final Version maybePrevious = builder.put(version.id, version); + assert maybePrevious == null : + "expected [" + version.id + "] to be uniquely mapped but saw [" + maybePrevious + "] and [" + version + "]"; + } catch (final IllegalAccessException e) { + assert false : "Version field [" + fieldName + "] should be public"; + } + } + } assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to [" + org.apache.lucene.util.Version.LATEST + "] is still set to [" + CURRENT.luceneVersion + "]"; + + idToVersion = builder.build(); } public static Version readVersion(StreamInput in) throws IOException { @@ -75,23 +100,10 @@ public static Version readVersion(StreamInput in) throws IOException { } public static Version fromId(int id) { + if (idToVersion.containsKey(id)) { + return idToVersion.get(id); + } switch (id) { - case V_8_0_0_ID: - return V_8_0_0; - case V_7_3_0_ID: - return V_7_3_0; - case V_7_2_0_ID: - return V_7_2_0; - case V_7_1_2_ID: - return V_7_1_2; - case V_7_1_1_ID: - return V_7_1_1; - case V_7_1_0_ID: - return V_7_1_0; - case V_7_0_1_ID: - return V_7_0_1; - case V_7_0_0_ID: - return V_7_0_0; case V_EMPTY_ID: return V_EMPTY; default: From 0140d512f9492d7cf88b4d2cec77c5fea99c09e3 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Thu, 13 Jun 2019 09:46:55 -0700 Subject: [PATCH 02/11] [DOCS] Update node descriptions for default distribution (#42812) --- docs/reference/modules/ml-node.asciidoc | 116 ------------------ docs/reference/modules/node.asciidoc | 149 +++++++++++++++++++----- 2 files changed, 119 insertions(+), 146 deletions(-) delete mode 100644 docs/reference/modules/ml-node.asciidoc diff --git a/docs/reference/modules/ml-node.asciidoc b/docs/reference/modules/ml-node.asciidoc deleted file mode 100644 index 365a08dfe7bb1..0000000000000 --- a/docs/reference/modules/ml-node.asciidoc +++ /dev/null @@ -1,116 +0,0 @@ -[float] -[[modules-node-xpack]] -== [xpack]#X-Pack node settings# - -//This content is referenced from the elastic/elasticsearch/docs/reference/modules/node.asciidoc - -If {xpack} is installed, there is an additional node type: - -<>:: - -A node that has `xpack.ml.enabled` and `node.ml` set to `true`, which is the -default behavior when {xpack} is installed. If you want to use {ml-features}, there must be at least one {ml} node in your cluster. For more -information about {ml-features}, -see {stack-ov}/xpack-ml.html[Machine learning in the {stack}]. - -IMPORTANT: Do not set use the `node.ml` setting unless {xpack} is installed. -Otherwise, the node fails to start. - -If {xpack} is installed, nodes are master-eligible, data, ingest, and {ml} -nodes by default. As the cluster grows and in particular if you have large -{ml} jobs, consider separating dedicated master-eligible nodes from dedicated -data nodes and dedicated {ml} nodes. - -To create a dedicated master-eligible node when {xpack} is installed, set: - -[source,yaml] -------------------- -node.master: true <1> -node.data: false <2> -node.ingest: false <3> -node.ml: false <4> -xpack.ml.enabled: true <5> -------------------- -<1> The `node.master` role is enabled by default. -<2> Disable the `node.data` role (enabled by default). -<3> Disable the `node.ingest` role (enabled by default). -<4> Disable the `node.ml` role (enabled by default in {xpack}). -<5> The `xpack.ml.enabled` setting is enabled by default in {xpack}. - -To create a dedicated data node when {xpack} is installed, set: - -[source,yaml] -------------------- -node.master: false <1> -node.data: true <2> -node.ingest: false <3> -node.ml: false <4> -------------------- -<1> Disable the `node.master` role (enabled by default). -<2> The `node.data` role is enabled by default. -<3> Disable the `node.ingest` role (enabled by default). -<4> Disable the `node.ml` role (enabled by default in {xpack}). - -To create a dedicated ingest node when {xpack} is installed, set: - -[source,yaml] -------------------- -node.master: false <1> -node.data: false <2> -node.ingest: true <3> -cluster.remote.connect: false <4> -node.ml: false <5> -------------------- -<1> Disable the `node.master` role (enabled by default). -<2> Disable the `node.data` role (enabled by default). -<3> The `node.ingest` role is enabled by default. -<4> Disable {ccs} (enabled by default). -<5> Disable the `node.ml` role (enabled by default in {xpack}). - -To create a dedicated coordinating node when {xpack} is installed, set: - -[source,yaml] -------------------- -node.master: false <1> -node.data: false <2> -node.ingest: false <3> -cluster.remote.connect: false <4> -node.ml: false <5> -------------------- -<1> Disable the `node.master` role (enabled by default). -<2> Disable the `node.data` role (enabled by default). -<3> Disable the `node.ingest` role (enabled by default). -<4> Disable {ccs} (enabled by default). -<5> Disable the `node.ml` role (enabled by default in {xpack}). - -[float] -[[ml-node]] -=== [xpack]#Machine learning node# - -The {ml-features} provide {ml} nodes, which run jobs and handle {ml} API -requests. If `xpack.ml.enabled` is set to true and `node.ml` is set to `false`, -the node can service API requests but it cannot run jobs. - -If you want to use {ml-features} in your cluster, you must enable {ml} -(set `xpack.ml.enabled` to `true`) on all master-eligible nodes. Do not use -these settings if you do not have {xpack} installed. - -For more information about these settings, see <>. - -To create a dedicated {ml} node, set: - -[source,yaml] -------------------- -node.master: false <1> -node.data: false <2> -node.ingest: false <3> -cluster.remote.connect: false <4> -node.ml: true <5> -xpack.ml.enabled: true <6> -------------------- -<1> Disable the `node.master` role (enabled by default). -<2> Disable the `node.data` role (enabled by default). -<3> Disable the `node.ingest` role (enabled by default). -<4> Disable {ccs} (enabled by default). -<5> The `node.ml` role is enabled by default in {xpack}. -<6> The `xpack.ml.enabled` setting is enabled by default in {xpack}. diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index 49d05e289a3f6..6a097bffd7031 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -2,7 +2,7 @@ == Node Any time that you start an instance of Elasticsearch, you are starting a -_node_. A collection of connected nodes is called a +_node_. A collection of connected nodes is called a <>. If you are running a single node of Elasticsearch, then you have a cluster of one node. @@ -12,8 +12,14 @@ is used exclusively for communication between nodes; the HTTP layer is used by REST clients. All nodes know about all the other nodes in the cluster and can forward client -requests to the appropriate node. Besides that, each node serves one or more -purpose: +requests to the appropriate node. + +By default, a node is all of the following types: master-eligible, data, ingest, +and machine learning (if available). + +TIP: As the cluster grows and in particular if you have large {ml} jobs, +consider separating dedicated master-eligible nodes from dedicated data nodes +and dedicated {ml} nodes. <>:: @@ -34,6 +40,17 @@ and enrich the document before indexing. With a heavy ingest load, it makes sense to use dedicated ingest nodes and to mark the master and data nodes as `node.ingest: false`. +<>:: + +A node that has `xpack.ml.enabled` and `node.ml` set to `true`, which is the +default behavior in the {es} {default-dist}. If you want to use {ml-features}, +there must be at least one {ml} node in your cluster. For more information about +{ml-features}, see +{stack-ov}/xpack-ml.html[Machine learning in the {stack}]. ++ +IMPORTANT: If you use the {oss-dist}, do not set `node.ml`. Otherwise, the node +fails to start. + [NOTE] [[coordinating-node]] .Coordinating node @@ -85,7 +102,25 @@ better _not_ to use dedicated master nodes for this purpose. It is important for the stability of the cluster that master-eligible nodes do as little work as possible. -To create a dedicated master-eligible node, set: +To create a dedicated master-eligible node in the {default-dist}, set: + +[source,yaml] +------------------- +node.master: true <1> +node.data: false <2> +node.ingest: false <3> +node.ml: false <4> +xpack.ml.enabled: true <5> +cluster.remote.connect: false <6> +------------------- +<1> The `node.master` role is enabled by default. +<2> Disable the `node.data` role (enabled by default). +<3> Disable the `node.ingest` role (enabled by default). +<4> Disable the `node.ml` role (enabled by default). +<5> The `xpack.ml.enabled` setting is enabled by default. +<6> Disable {ccs} (enabled by default). + +To create a dedicated master-eligible node in the {oss-dist}, set: [source,yaml] ------------------- @@ -99,11 +134,6 @@ cluster.remote.connect: false <4> <3> Disable the `node.ingest` role (enabled by default). <4> Disable {ccs} (enabled by default). -ifdef::include-xpack[] -NOTE: These settings apply only when {xpack} is not installed. To create a -dedicated master-eligible node when {xpack} is installed, see <>. -endif::include-xpack[] - [float] [[data-node]] === Data Node @@ -116,8 +146,22 @@ monitor these resources and to add more data nodes if they are overloaded. The main benefit of having dedicated data nodes is the separation of the master and data roles. -To create a dedicated data node, set: +To create a dedicated data node in the {default-dist}, set: +[source,yaml] +------------------- +node.master: false <1> +node.data: true <2> +node.ingest: false <3> +node.ml: false <4> +cluster.remote.connect: false <5> +------------------- +<1> Disable the `node.master` role (enabled by default). +<2> The `node.data` role is enabled by default. +<3> Disable the `node.ingest` role (enabled by default). +<4> Disable the `node.ml` role (enabled by default). +<5> Disable {ccs} (enabled by default). +To create a dedicated data node in the {oss-dist}, set: [source,yaml] ------------------- node.master: false <1> @@ -130,11 +174,6 @@ cluster.remote.connect: false <4> <3> Disable the `node.ingest` role (enabled by default). <4> Disable {ccs} (enabled by default). -ifdef::include-xpack[] -NOTE: These settings apply only when {xpack} is not installed. To create a -dedicated data node when {xpack} is installed, see <>. -endif::include-xpack[] - [float] [[node-ingest-node]] === Ingest Node @@ -144,7 +183,23 @@ ingest processors. Depending on the type of operations performed by the ingest processors and the required resources, it may make sense to have dedicated ingest nodes, that will only perform this specific task. -To create a dedicated ingest node, set: +To create a dedicated ingest node in the {default-dist}, set: + +[source,yaml] +------------------- +node.master: false <1> +node.data: false <2> +node.ingest: true <3> +node.ml: false <4> +cluster.remote.connect: false <5> +------------------- +<1> Disable the `node.master` role (enabled by default). +<2> Disable the `node.data` role (enabled by default). +<3> The `node.ingest` role is enabled by default. +<4> Disable the `node.ml` role (enabled by default). +<5> Disable {ccs} (enabled by default). + +To create a dedicated ingest node in the {oss-dist}, set: [source,yaml] ------------------- @@ -158,11 +213,6 @@ cluster.remote.connect: false <4> <3> The `node.ingest` role is enabled by default. <4> Disable {ccs} (enabled by default). -ifdef::include-xpack[] -NOTE: These settings apply only when {xpack} is not installed. To create a -dedicated ingest node when {xpack} is installed, see <>. -endif::include-xpack[] - [float] [[coordinating-only-node]] === Coordinating only node @@ -184,7 +234,23 @@ acknowledgement of cluster state updates from every node! The benefit of coordinating only nodes should not be overstated -- data nodes can happily serve the same purpose. -To create a dedicated coordinating node, set: +To create a dedicated coordinating node in the {default-dist}, set: + +[source,yaml] +------------------- +node.master: false <1> +node.data: false <2> +node.ingest: false <3> +node.ml: false <4> +cluster.remote.connect: false <5> +------------------- +<1> Disable the `node.master` role (enabled by default). +<2> Disable the `node.data` role (enabled by default). +<3> Disable the `node.ingest` role (enabled by default). +<4> Disable the `node.ml` role (enabled by default). +<5> Disable {ccs} (enabled by default). + +To create a dedicated coordinating node in the {oss-dist}, set: [source,yaml] ------------------- @@ -198,10 +264,37 @@ cluster.remote.connect: false <4> <3> Disable the `node.ingest` role (enabled by default). <4> Disable {ccs} (enabled by default). -ifdef::include-xpack[] -NOTE: These settings apply only when {xpack} is not installed. To create a -dedicated coordinating node when {xpack} is installed, see <>. -endif::include-xpack[] +[float] +[[ml-node]] +=== [xpack]#Machine learning node# + +The {ml-features} provide {ml} nodes, which run jobs and handle {ml} API +requests. If `xpack.ml.enabled` is set to true and `node.ml` is set to `false`, +the node can service API requests but it cannot run jobs. + +If you want to use {ml-features} in your cluster, you must enable {ml} +(set `xpack.ml.enabled` to `true`) on all master-eligible nodes. If you have the +{oss-dist}, do not use these settings. + +For more information about these settings, see <>. + +To create a dedicated {ml} node in the {default-dist}, set: + +[source,yaml] +------------------- +node.master: false <1> +node.data: false <2> +node.ingest: false <3> +node.ml: true <4> +xpack.ml.enabled: true <5> +cluster.remote.connect: false <6> +------------------- +<1> Disable the `node.master` role (enabled by default). +<2> Disable the `node.data` role (enabled by default). +<3> Disable the `node.ingest` role (enabled by default). +<4> The `node.ml` role is enabled by default. +<5> The `xpack.ml.enabled` setting is enabled by default. +<6> Disable {ccs} (enabled by default). [float] [[change-node-role]] @@ -282,7 +375,3 @@ your data! The RPM and Debian distributions do this for you already. More node settings can be found in <>. Of particular note are the <>, the <> and the <>. - -ifdef::include-xpack[] -include::ml-node.asciidoc[] -endif::include-xpack[] From ec3ac9b62452f04ce44dea0a904a6e2b31dd8076 Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Thu, 13 Jun 2019 10:24:28 -0700 Subject: [PATCH 03/11] Compile subset of build-tools classes with Java 8 target compatibility (#43177) --- buildSrc/build.gradle | 35 +++++++++++++++---- .../elasticsearch/gradle/JdkJarHellCheck.java | 0 .../gradle/LazyFileOutputStream.java | 0 .../org/elasticsearch/gradle/LoggedExec.java | 9 +++-- .../org/elasticsearch/gradle/Version.java | 0 .../gradle/VersionProperties.java | 0 6 files changed, 36 insertions(+), 8 deletions(-) rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/JdkJarHellCheck.java (100%) rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/LazyFileOutputStream.java (100%) rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/LoggedExec.java (94%) rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/Version.java (100%) rename buildSrc/src/{main => minimumRuntime}/java/org/elasticsearch/gradle/VersionProperties.java (100%) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 339f1a75875be..a3ca1c7ffa348 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -67,8 +67,26 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) { } // Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11 -targetCompatibility = '8' -sourceCompatibility = '8' +targetCompatibility = '11' +sourceCompatibility = '11' + +sourceSets { + // We have a few classes that need to be compiled for older java versions + minimumRuntime { } +} + +compileMinimumRuntimeJava { + targetCompatibility = 8 + sourceCompatibility = 8 +} + +jar { + from sourceSets.minimumRuntime.output +} + +javadoc { + source sourceSets.minimumRuntime.allSource +} /***************************************************************************** * Dependencies used by the entire build * @@ -79,6 +97,11 @@ repositories { } dependencies { + if (project.ext.has("isEclipse") == false || project.ext.isEclipse == false) { + // eclipse is confused if this is set explicitly + compile sourceSets.minimumRuntime.output + } + compile localGroovy() compile 'commons-codec:commons-codec:1.12' @@ -96,6 +119,9 @@ dependencies { testCompile "junit:junit:${props.getProperty('junit')}" testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}" testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2' + minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}" + minimumRuntimeCompile localGroovy() + minimumRuntimeCompile gradleApi() } /***************************************************************************** @@ -123,10 +149,6 @@ if (project != rootProject) { apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' - // we need to apply these again to override the build plugin - targetCompatibility = "10" - sourceCompatibility = "10" - // groovydoc succeeds, but has some weird internal exception... groovydoc.enabled = false @@ -134,6 +156,7 @@ if (project != rootProject) { dependencyLicenses.enabled = false dependenciesInfo.enabled = false forbiddenApisMain.enabled = false + forbiddenApisMinimumRuntime.enabled = false forbiddenApisTest.enabled = false jarHell.enabled = false thirdPartyAudit.enabled = false diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/JdkJarHellCheck.java similarity index 100% rename from buildSrc/src/main/java/org/elasticsearch/gradle/JdkJarHellCheck.java rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/JdkJarHellCheck.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LazyFileOutputStream.java similarity index 100% rename from buildSrc/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LazyFileOutputStream.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java similarity index 94% rename from buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java index 0921c611895fa..89031916160b2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggedExec.java +++ b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/LoggedExec.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.function.Consumer; import java.util.function.Function; @@ -79,7 +78,13 @@ public void setSpoolOutput(boolean spoolOutput) { }; } else { out = new ByteArrayOutputStream(); - outputLogger = logger -> logger.error(((ByteArrayOutputStream) out).toString(StandardCharsets.UTF_8)); + outputLogger = logger -> { + try { + logger.error(((ByteArrayOutputStream) out).toString("UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + }; } setStandardOutput(out); setErrorOutput(out); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Version.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java similarity index 100% rename from buildSrc/src/main/java/org/elasticsearch/gradle/Version.java rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/Version.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java b/buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/VersionProperties.java similarity index 100% rename from buildSrc/src/main/java/org/elasticsearch/gradle/VersionProperties.java rename to buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/VersionProperties.java From aa4ff855a6694c9270497413e57dee50a4f0a099 Mon Sep 17 00:00:00 2001 From: lcawl Date: Thu, 13 Jun 2019 11:15:24 -0700 Subject: [PATCH 04/11] [DOCS] Fix link to ML node description --- docs/reference/ml/configuring.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ml/configuring.asciidoc b/docs/reference/ml/configuring.asciidoc index 9304a93d360c7..a27818e0cf1f5 100644 --- a/docs/reference/ml/configuring.asciidoc +++ b/docs/reference/ml/configuring.asciidoc @@ -5,7 +5,7 @@ If you want to use {ml-features}, there must be at least one {ml} node in your cluster and all master-eligible nodes must have {ml} enabled. By default, all nodes are {ml} nodes. For more information about these settings, see -{ref}/modules-node.html#modules-node-xpack[{ml} nodes]. +{ref}/modules-node.html#ml-node[{ml} nodes]. To use the {ml-features} to analyze your data, you must create a job and send your data to that job. From d9dbf79c71b37584746a213141833446307dcc0a Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 13 Jun 2019 20:19:39 +0200 Subject: [PATCH 05/11] Disable x-pack:qa:kerberos-tests due to failures (#43210) relates #40678 --- .../xpack/security/authc/kerberos/KerberosAuthenticationIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java index 0281f38933b03..a2aa559295a8c 100644 --- a/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java +++ b/x-pack/qa/kerberos-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosAuthenticationIT.java @@ -8,6 +8,7 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpHost; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -47,6 +48,7 @@ * Demonstrates login by keytab and login by password for given user principal * name using rest client. */ +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/40678") public class KerberosAuthenticationIT extends ESRestTestCase { private static final String ENABLE_KERBEROS_DEBUG_LOGS_KEY = "test.krb.debug"; private static final String TEST_USER_WITH_KEYTAB_KEY = "test.userkt"; From a3f2f4079cb07f45afa86974e986112f35986459 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 13 Jun 2019 11:38:29 -0700 Subject: [PATCH 06/11] Add native code info to ML info api (#43172) The machine learning feature of xpack has native binaries with a different commit id than the rest of code. It is currently exposed in the xpack info api. This commit adds that commit information to the ML info api, so that it may be removed from the info api. --- docs/reference/ml/apis/get-ml-info.asciidoc | 6 ++++ .../ml/action/TransportMlInfoAction.java | 28 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/reference/ml/apis/get-ml-info.asciidoc b/docs/reference/ml/apis/get-ml-info.asciidoc index 00c57c8cf7f5b..41b680e1327c0 100644 --- a/docs/reference/ml/apis/get-ml-info.asciidoc +++ b/docs/reference/ml/apis/get-ml-info.asciidoc @@ -57,7 +57,13 @@ This is a possible response: } }, "upgrade_mode": false, + "native_code" : { + "version": "7.0.0", + "build_hash": "99a07c016d5a73" + }, "limits" : { } } ---- // TESTRESPONSE[s/"upgrade_mode": false/"upgrade_mode": $body.upgrade_mode/] +// TESTRESPONSE[s/"version": "7.0.0",/"version": "$body.native_code.version",/] +// TESTRESPONSE[s/"build_hash": "99a07c016d5a73"/"build_hash": "$body.native_code.build_hash"/] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java index 48fbf9566fd1e..d423428b3b7f6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportMlInfoAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.env.Environment; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.ml.MachineLearningField; @@ -20,19 +21,43 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits; import org.elasticsearch.xpack.core.ml.job.config.Job; +import org.elasticsearch.xpack.ml.process.NativeController; +import org.elasticsearch.xpack.ml.process.NativeControllerHolder; +import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; public class TransportMlInfoAction extends HandledTransportAction { private final ClusterService clusterService; + private final Map nativeCodeInfo; @Inject - public TransportMlInfoAction(TransportService transportService, ActionFilters actionFilters, ClusterService clusterService) { + public TransportMlInfoAction(TransportService transportService, ActionFilters actionFilters, + ClusterService clusterService, Environment env) { super(MlInfoAction.NAME, transportService, actionFilters, (Supplier) MlInfoAction.Request::new); this.clusterService = clusterService; + + try { + NativeController nativeController = NativeControllerHolder.getNativeController(clusterService.getNodeName(), env); + // TODO: this leniency is only for tests. it can be removed when NativeController is created as a component and + // becomes a ctor arg to this action + if (nativeController != null) { + nativeCodeInfo = nativeController.getNativeCodeInfo(); + } else { + nativeCodeInfo = Collections.emptyMap(); + } + } catch (IOException e) { + // this should not be possible since this action is only registered when ML is enabled, + // and the MachineLearning plugin would have failed to create components + throw new IllegalStateException("native controller failed to load", e); + } catch (TimeoutException e) { + throw new RuntimeException("Could not get native code info from native controller", e); + } } @Override @@ -40,6 +65,7 @@ protected void doExecute(Task task, MlInfoAction.Request request, ActionListener Map info = new HashMap<>(); info.put("defaults", defaults()); info.put("limits", limits()); + info.put("native_code", nativeCodeInfo); info.put(MlMetadata.UPGRADE_MODE.getPreferredName(), upgradeMode()); listener.onResponse(new MlInfoAction.Response(info)); } From aa12af8a3c4022c1b7c1eeab1a400645e603a07d Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Jun 2019 14:43:14 -0400 Subject: [PATCH 07/11] Enable node roles to be pluggable (#43175) This commit introduces the possibility for a plugin to introduce additional node roles. --- .../routing/allocation/Allocators.java | 5 +- docs/reference/cat/nodes.asciidoc | 2 +- docs/reference/getting-started.asciidoc | 2 +- .../cluster/node/info/NodesInfoResponse.java | 6 +- .../admin/cluster/node/stats/NodeStats.java | 5 +- .../node/tasks/list/ListTasksResponse.java | 5 +- .../cluster/stats/ClusterStatsNodes.java | 27 ++-- .../cluster/node/DiscoveryNode.java | 150 ++++++++++++------ .../cluster/node/DiscoveryNodeRole.java | 137 ++++++++++++++++ .../cluster/node/DiscoveryNodes.java | 6 +- .../java/org/elasticsearch/node/Node.java | 10 ++ .../org/elasticsearch/plugins/Plugin.java | 6 + .../rest/action/cat/RestNodesAction.java | 3 +- .../transport/RemoteClusterAware.java | 4 +- ...AddVotingConfigExclusionsRequestTests.java | 58 +++++-- ...tAddVotingConfigExclusionsActionTests.java | 13 +- .../admin/cluster/stats/ClusterStatsIT.java | 14 +- .../shrink/TransportResizeActionTests.java | 3 +- .../TransportMultiSearchActionTests.java | 7 +- .../TransportMasterNodeActionTests.java | 7 +- .../nodes/TransportNodesActionTests.java | 6 +- .../ClusterStateCreationUtils.java | 4 +- ...ReplicationAllPermitsAcquisitionTests.java | 4 +- .../cluster/ClusterChangedEventTests.java | 15 +- .../cluster/NodeConnectionsServiceTests.java | 4 +- .../ClusterBootstrapServiceTests.java | 33 +++- .../ClusterFormationFailureHelperTests.java | 5 +- .../coordination/CoordinationStateTests.java | 7 +- .../coordination/CoordinatorTests.java | 7 +- .../coordination/FollowersCheckerTests.java | 6 +- .../cluster/coordination/NodeJoinTests.java | 10 +- .../coordination/PublicationTests.java | 5 +- .../metadata/AutoExpandReplicasTests.java | 16 +- .../MetaDataCreateIndexServiceTests.java | 3 +- .../MetaDataIndexStateServiceTests.java | 5 +- .../metadata/TemplateUpgradeServiceTests.java | 6 +- .../cluster/node/DiscoveryNodeRoleIT.java | 99 ++++++++++++ .../cluster/node/DiscoveryNodeRoleTests.java | 78 +++++++++ .../cluster/node/DiscoveryNodesTests.java | 16 +- .../DelayedAllocationServiceTests.java | 4 +- .../allocation/AllocationCommandsTests.java | 8 +- .../allocation/FailedNodeRoutingTests.java | 17 +- .../decider/DiskThresholdDeciderTests.java | 7 +- .../DiskThresholdDeciderUnitTests.java | 14 +- ...storeInProgressAllocationDeciderTests.java | 3 +- .../gateway/AsyncShardFetchTests.java | 5 +- .../gateway/ClusterStateUpdatersTests.java | 5 +- .../GatewayMetaStatePersistedStateTests.java | 3 +- .../gateway/GatewayMetaStateTests.java | 4 +- .../FakeThreadPoolMasterServiceTests.java | 4 +- ...ClusterStateServiceRandomUpdatesTests.java | 15 +- .../PersistentTasksClusterServiceTests.java | 3 +- .../snapshots/SnapshotResiliencyTests.java | 7 +- .../transport/RemoteClusterServiceTests.java | 26 +-- .../cluster/ESAllocationTestCase.java | 8 +- .../ESIndexLevelReplicationTestCase.java | 3 +- .../index/shard/IndexShardTestCase.java | 4 +- .../test/ClusterServiceUtils.java | 6 +- .../org/elasticsearch/test/ESTestCase.java | 12 ++ .../test/InternalTestCluster.java | 8 +- .../test/test/InternalTestClusterTests.java | 49 +++--- .../GetDatafeedStatsActionResponseTests.java | 6 +- .../ml/MachineLearningFeatureSetTests.java | 17 +- .../ClusterStatsMonitoringDocTests.java | 13 +- .../IndexRecoveryMonitoringDocTests.java | 5 +- .../ml/JobStatsMonitoringDocTests.java | 3 +- .../security/authc/TokenServiceTests.java | 4 +- ...stractSimpleSecurityTransportTestCase.java | 7 +- .../watcher/WatcherIndexingListenerTests.java | 9 +- .../watcher/WatcherLifeCycleServiceTests.java | 12 +- .../xpack/watcher/WatcherServiceTests.java | 5 +- .../execution/ExecutionServiceTests.java | 5 +- 72 files changed, 783 insertions(+), 317 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java index 39a2bdfca0953..90f03c26dcb27 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedShard; @@ -92,7 +93,7 @@ public static AllocationDeciders defaultAllocationDeciders(Settings settings, Cl public static DiscoveryNode newNode(String nodeId, Map attributes) { return new DiscoveryNode("", nodeId, new TransportAddress(TransportAddress.META_ADDRESS, - portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER, - DiscoveryNode.Role.DATA), Version.CURRENT); + portGenerator.incrementAndGet()), attributes, Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE, + DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); } } diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 06ea5697c785c..a882f5f9fa398 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -14,7 +14,7 @@ Might look like: [source,txt] -------------------------------------------------- ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name -127.0.0.1 65 99 42 3.07 mdi * mJw06l1 +127.0.0.1 65 99 42 3.07 dim * mJw06l1 -------------------------------------------------- // TESTRESPONSE[s/3.07/(\\d+\\.\\d+( \\d+\\.\\d+ (\\d+\\.\\d+)?)?)?/] // TESTRESPONSE[s/65 99 42/\\d+ \\d+ \\d+/] diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index c602515ed5250..eef66544d1b65 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -330,7 +330,7 @@ And the response: [source,txt] -------------------------------------------------- ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name -127.0.0.1 10 5 5 4.46 mdi * PB2SGZY +127.0.0.1 10 5 5 4.46 dim * PB2SGZY -------------------------------------------------- // TESTRESPONSE[s/10 5 5 4.46/\\d+ \\d+ \\d+ (\\d+\\.\\d+)? (\\d+\\.\\d+)? (\\d+\.\\d+)?/] // TESTRESPONSE[s/[*]/[*]/ s/PB2SGZY/.+/ non_json] diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java index 3bee722a472ec..4d70b97ca2cf6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java @@ -22,7 +22,7 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodesResponse; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -74,8 +74,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.startArray("roles"); - for (DiscoveryNode.Role role : nodeInfo.getNode().getRoles()) { - builder.value(role.getRoleName()); + for (DiscoveryNodeRole role : nodeInfo.getNode().getRoles()) { + builder.value(role.roleName()); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 7a69bc0e506bd..1d32fbef2c557 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -269,8 +270,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("ip", getNode().getAddress()); builder.startArray("roles"); - for (DiscoveryNode.Role role : getNode().getRoles()) { - builder.value(role.getRoleName()); + for (DiscoveryNodeRole role : getNode().getRoles()) { + builder.value(role.roleName()); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java index 557338ed0f77e..d6fc541ba687a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.TaskOperationFailure; import org.elasticsearch.action.support.tasks.BaseTasksResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -175,8 +176,8 @@ public XContentBuilder toXContentGroupedByNode(XContentBuilder builder, Params p builder.field("ip", node.getAddress()); builder.startArray("roles"); - for (DiscoveryNode.Role role : node.getRoles()) { - builder.value(role.getRoleName()); + for (DiscoveryNodeRole role : node.getRoles()) { + builder.value(role.roleName()); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java index 49d82e574fb6b..bcadbc8e3292a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java @@ -21,11 +21,11 @@ import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; - import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.network.NetworkModule; @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicInteger; public class ClusterStatsNodes implements ToXContentFragment { @@ -186,27 +187,27 @@ public static class Counts implements ToXContentFragment { private final int total; private final Map roles; - private Counts(List nodeInfos) { - this.roles = new HashMap<>(); - for (DiscoveryNode.Role role : DiscoveryNode.Role.values()) { - this.roles.put(role.getRoleName(), 0); + private Counts(final List nodeInfos) { + // TODO: do we need to report zeros? + final Map roles = new HashMap<>(DiscoveryNode.getPossibleRoleNames().size()); + roles.put(COORDINATING_ONLY, 0); + for (final String possibleRoleName : DiscoveryNode.getPossibleRoleNames()) { + roles.put(possibleRoleName, 0); } - this.roles.put(COORDINATING_ONLY, 0); int total = 0; - for (NodeInfo nodeInfo : nodeInfos) { + for (final NodeInfo nodeInfo : nodeInfos) { total++; if (nodeInfo.getNode().getRoles().isEmpty()) { - Integer count = roles.get(COORDINATING_ONLY); - roles.put(COORDINATING_ONLY, ++count); + roles.merge(COORDINATING_ONLY, 1, Integer::sum); } else { - for (DiscoveryNode.Role role : nodeInfo.getNode().getRoles()) { - Integer count = roles.get(role.getRoleName()); - roles.put(role.getRoleName(), ++count); + for (DiscoveryNodeRole role : nodeInfo.getNode().getRoles()) { + roles.merge(role.roleName(), 1, Integer::sum); } } } this.total = total; + this.roles = Map.copyOf(roles); } public int getTotal() { @@ -225,7 +226,7 @@ static final class Fields { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.TOTAL, total); - for (Map.Entry entry : roles.entrySet()) { + for (Map.Entry entry : new TreeMap<>(roles).entrySet()) { builder.field(entry.getKey(), entry.getValue()); } return builder; diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index afab4af320416..893a496f134dc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -32,11 +32,14 @@ import java.io.IOException; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; /** @@ -78,8 +81,7 @@ public static boolean isIngestNode(Settings settings) { private final TransportAddress address; private final Map attributes; private final Version version; - private final Set roles; - + private final Set roles; /** * Creates a new {@link DiscoveryNode} @@ -95,7 +97,7 @@ public static boolean isIngestNode(Settings settings) { * @param version the version of the node */ public DiscoveryNode(final String id, TransportAddress address, Version version) { - this(id, address, Collections.emptyMap(), EnumSet.allOf(Role.class), version); + this(id, address, Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, version); } /** @@ -113,7 +115,7 @@ public DiscoveryNode(final String id, TransportAddress address, Version version) * @param roles node roles * @param version the version of the node */ - public DiscoveryNode(String id, TransportAddress address, Map attributes, Set roles, + public DiscoveryNode(String id, TransportAddress address, Map attributes, Set roles, Version version) { this("", id, address, attributes, roles, version); } @@ -135,7 +137,7 @@ public DiscoveryNode(String id, TransportAddress address, Map at * @param version the version of the node */ public DiscoveryNode(String nodeName, String nodeId, TransportAddress address, - Map attributes, Set roles, Version version) { + Map attributes, Set roles, Version version) { this(nodeName, nodeId, UUIDs.randomBase64UUID(), address.address().getHostString(), address.getAddress(), address, attributes, roles, version); } @@ -159,7 +161,7 @@ public DiscoveryNode(String nodeName, String nodeId, TransportAddress address, * @param version the version of the node */ public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String hostName, String hostAddress, - TransportAddress address, Map attributes, Set roles, Version version) { + TransportAddress address, Map attributes, Set roles, Version version) { if (nodeName != null) { this.nodeName = nodeName.intern(); } else { @@ -178,37 +180,27 @@ public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String this.attributes = Collections.unmodifiableMap(attributes); //verify that no node roles are being provided as attributes Predicate> predicate = (attrs) -> { - for (Role role : Role.values()) { - assert attrs.containsKey(role.getRoleName()) == false; + boolean success = true; + for (final DiscoveryNodeRole role : DiscoveryNode.roleNameToPossibleRoles.values()) { + success &= attrs.containsKey(role.roleName()) == false; + assert success : role.roleName(); } - return true; + return success; }; - assert predicate.test(attributes); - Set rolesSet = EnumSet.noneOf(Role.class); - rolesSet.addAll(roles); - this.roles = Collections.unmodifiableSet(rolesSet); + assert predicate.test(attributes) : attributes; + this.roles = Set.copyOf(roles); } /** Creates a DiscoveryNode representing the local node. */ public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeId) { Map attributes = Node.NODE_ATTRIBUTES.getAsMap(settings); - Set roles = getRolesFromSettings(settings); + Set roles = getRolesFromSettings(settings); return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes, roles, Version.CURRENT); } /** extract node roles from the given settings */ - public static Set getRolesFromSettings(Settings settings) { - Set roles = EnumSet.noneOf(Role.class); - if (Node.NODE_INGEST_SETTING.get(settings)) { - roles.add(Role.INGEST); - } - if (Node.NODE_MASTER_SETTING.get(settings)) { - roles.add(Role.MASTER); - } - if (Node.NODE_DATA_SETTING.get(settings)) { - roles.add(Role.DATA); - } - return roles; + public static Set getRolesFromSettings(final Settings settings) { + return roleNameToPossibleRoles.values().stream().filter(s -> s.roleSetting().get(settings)).collect(Collectors.toUnmodifiableSet()); } /** @@ -229,10 +221,41 @@ public DiscoveryNode(StreamInput in) throws IOException { this.attributes.put(in.readString(), in.readString()); } int rolesSize = in.readVInt(); - this.roles = EnumSet.noneOf(Role.class); - for (int i = 0; i < rolesSize; i++) { - this.roles.add(in.readEnum(Role.class)); + final Set roles = new HashSet<>(rolesSize); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + for (int i = 0; i < rolesSize; i++) { + final String roleName = in.readString(); + final String roleNameAbbreviation = in.readString(); + final DiscoveryNodeRole role = roleNameToPossibleRoles.get(roleName); + if (role == null) { + roles.add(new DiscoveryNodeRole.UnknownRole(roleName, roleNameAbbreviation)); + } else { + assert roleName.equals(role.roleName()) : "role name [" + roleName + "] does not match role [" + role.roleName() + "]"; + assert roleNameAbbreviation.equals(role.roleNameAbbreviation()) + : "role name abbreviation [" + roleName + "] does not match role [" + role.roleNameAbbreviation() + "]"; + roles.add(role); + } + } + } else { + // an old node will only send us legacy roles since pluggable roles is a new concept + for (int i = 0; i < rolesSize; i++) { + final LegacyRole legacyRole = in.readEnum(LegacyRole.class); + switch (legacyRole) { + case MASTER: + roles.add(DiscoveryNodeRole.MASTER_ROLE); + break; + case DATA: + roles.add(DiscoveryNodeRole.DATA_ROLE); + break; + case INGEST: + roles.add(DiscoveryNodeRole.INGEST_ROLE); + break; + default: + throw new AssertionError(legacyRole.roleName()); + } + } } + this.roles = Set.copyOf(roles); this.version = Version.readVersion(in); } @@ -249,9 +272,26 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } - out.writeVInt(roles.size()); - for (Role role : roles) { - out.writeEnum(role); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeVInt(roles.size()); + for (final DiscoveryNodeRole role : roles) { + out.writeString(role.roleName()); + out.writeString(role.roleNameAbbreviation()); + } + } else { + // an old node will only understand legacy roles since pluggable roles is a new concept + final List rolesToWrite = + roles.stream().filter(DiscoveryNodeRole.BUILT_IN_ROLES::contains).collect(Collectors.toUnmodifiableList()); + out.writeVInt(rolesToWrite.size()); + for (final DiscoveryNodeRole role : rolesToWrite) { + if (role == DiscoveryNodeRole.MASTER_ROLE) { + out.writeEnum(LegacyRole.MASTER); + } else if (role == DiscoveryNodeRole.DATA_ROLE) { + out.writeEnum(LegacyRole.DATA); + } else if (role == DiscoveryNodeRole.INGEST_ROLE) { + out.writeEnum(LegacyRole.INGEST); + } + } } Version.writeVersion(version, out); } @@ -299,28 +339,28 @@ public Map getAttributes() { * Should this node hold data (shards) or not. */ public boolean isDataNode() { - return roles.contains(Role.DATA); + return roles.contains(DiscoveryNodeRole.DATA_ROLE); } /** * Can this node become master or not. */ public boolean isMasterNode() { - return roles.contains(Role.MASTER); + return roles.contains(DiscoveryNodeRole.MASTER_ROLE); } /** * Returns a boolean that tells whether this an ingest node or not */ public boolean isIngestNode() { - return roles.contains(Role.INGEST); + return roles.contains(DiscoveryNodeRole.INGEST_ROLE); } /** * Returns a set of all the roles that the node fulfills. * If the node doesn't have any specific role, the set is returned empty, which means that the node is a coordinating only node. */ - public Set getRoles() { + public Set getRoles() { return roles; } @@ -391,29 +431,43 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + private static Map roleNameToPossibleRoles; + + public static void setPossibleRoles(final Set possibleRoles) { + final Map roleNameToPossibleRoles = + possibleRoles.stream().collect(Collectors.toUnmodifiableMap(DiscoveryNodeRole::roleName, Function.identity())); + // collect the abbreviation names into a map to ensure that there are not any duplicate abbreviations + final Map roleNameAbbreviationToPossibleRoles = roleNameToPossibleRoles.values() + .stream() + .collect(Collectors.toUnmodifiableMap(DiscoveryNodeRole::roleNameAbbreviation, Function.identity())); + assert roleNameToPossibleRoles.size() == roleNameAbbreviationToPossibleRoles.size() : + "roles by name [" + roleNameToPossibleRoles + "], roles by name abbreviation [" + roleNameAbbreviationToPossibleRoles + "]"; + DiscoveryNode.roleNameToPossibleRoles = roleNameToPossibleRoles; + } + + public static Set getPossibleRoleNames() { + return roleNameToPossibleRoles.keySet(); + } + /** * Enum that holds all the possible roles that that a node can fulfill in a cluster. * Each role has its name and a corresponding abbreviation used by cat apis. */ - public enum Role { - MASTER("master", "m"), - DATA("data", "d"), - INGEST("ingest", "i"); + private enum LegacyRole { + MASTER("master"), + DATA("data"), + INGEST("ingest"); private final String roleName; - private final String abbreviation; - Role(String roleName, String abbreviation) { + LegacyRole(final String roleName) { this.roleName = roleName; - this.abbreviation = abbreviation; } - public String getRoleName() { + public String roleName() { return roleName; } - public String getAbbreviation() { - return abbreviation; - } } + } diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java new file mode 100644 index 0000000000000..ba63335d10046 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeRole.java @@ -0,0 +1,137 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.node; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.node.Node; + +import java.util.Objects; +import java.util.Set; + +/** + * Represents a node role. + */ +public abstract class DiscoveryNodeRole { + + private final String roleName; + + /** + * The name of the role. + * + * @return the role name + */ + public final String roleName() { + return roleName; + } + + private final String roleNameAbbreviation; + + /** + * The abbreviation of the name of the role. This is used in the cat nodes API to display an abbreviated version of the name of the + * role. + * + * @return the role name abbreviation + */ + public final String roleNameAbbreviation() { + return roleNameAbbreviation; + } + + protected DiscoveryNodeRole(final String roleName, final String roleNameAbbreviation) { + this.roleName = Objects.requireNonNull(roleName); + this.roleNameAbbreviation = Objects.requireNonNull(roleNameAbbreviation); + } + + protected abstract Setting roleSetting(); + + @Override + public String toString() { + return "DiscoveryNodeRole{" + + "roleName='" + roleName + '\'' + + ", roleNameAbbreviation='" + roleNameAbbreviation + '\'' + + '}'; + } + + /** + * Represents the role for a data node. + */ + public static final DiscoveryNodeRole DATA_ROLE = new DiscoveryNodeRole("data", "d") { + + @Override + protected Setting roleSetting() { + return Node.NODE_DATA_SETTING; + } + + }; + + /** + * Represents the role for an ingest node. + */ + public static final DiscoveryNodeRole INGEST_ROLE = new DiscoveryNodeRole("ingest", "i") { + + @Override + protected Setting roleSetting() { + return Node.NODE_INGEST_SETTING; + } + + }; + + /** + * Represents the role for a master-eligible node. + */ + public static final DiscoveryNodeRole MASTER_ROLE = new DiscoveryNodeRole("master", "m") { + + @Override + protected Setting roleSetting() { + return Node.NODE_MASTER_SETTING; + } + + }; + + /** + * The built-in node roles. + */ + public static Set BUILT_IN_ROLES = Set.of(DATA_ROLE, INGEST_ROLE, MASTER_ROLE); + + /** + * Represents an unknown role. This can occur if a newer version adds a role that an older version does not know about, or a newer + * version removes a role that an older version knows about. + */ + static class UnknownRole extends DiscoveryNodeRole { + + /** + * Construct an unknown role with the specified role name and role name abbreviation. + * + * @param roleName the role name + * @param roleNameAbbreviation the role name abbreviation + */ + UnknownRole(final String roleName, final String roleNameAbbreviation) { + super(roleName, roleNameAbbreviation); + } + + @Override + protected Setting roleSetting() { + // since this setting is not registered, it will always return false when testing if the local node has the role + assert false; + return Setting.boolSetting("node. " + roleName(), false, Setting.Property.NodeScope); + } + + } + +} diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index 02f4d5d93bf4e..57eb2ac7541dc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -353,19 +353,19 @@ public String[] resolveNodes(String... nodes) { if (index != -1) { String matchAttrName = nodeId.substring(0, index); String matchAttrValue = nodeId.substring(index + 1); - if (DiscoveryNode.Role.DATA.getRoleName().equals(matchAttrName)) { + if (DiscoveryNodeRole.DATA_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(dataNodes.keys()); } else { resolvedNodesIds.removeAll(dataNodes.keys()); } - } else if (DiscoveryNode.Role.MASTER.getRoleName().equals(matchAttrName)) { + } else if (DiscoveryNodeRole.MASTER_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(masterNodes.keys()); } else { resolvedNodesIds.removeAll(masterNodes.keys()); } - } else if (DiscoveryNode.Role.INGEST.getRoleName().equals(matchAttrName)) { + } else if (DiscoveryNodeRole.INGEST_ROLE.roleName().equals(matchAttrName)) { if (Booleans.parseBoolean(matchAttrValue, true)) { resolvedNodesIds.addAll(ingestNodes.keys()); } else { diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 82d274098b582..e959904b8fee7 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -55,6 +55,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.metadata.TemplateUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.RoutingService; import org.elasticsearch.cluster.routing.allocation.DiskThresholdMonitor; import org.elasticsearch.cluster.service.ClusterService; @@ -155,6 +156,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import javax.net.ssl.SNIHostName; + import java.io.BufferedWriter; import java.io.Closeable; import java.io.IOException; @@ -311,6 +313,14 @@ protected Node( this.pluginsService = new PluginsService(tmpSettings, environment.configFile(), environment.modulesFile(), environment.pluginsFile(), classpathPlugins); final Settings settings = pluginsService.updatedSettings(); + final Set possibleRoles = Stream.concat( + DiscoveryNodeRole.BUILT_IN_ROLES.stream(), + pluginsService.filterPlugins(Plugin.class) + .stream() + .map(Plugin::getRoles) + .flatMap(Set::stream)) + .collect(Collectors.toSet()); + DiscoveryNode.setPossibleRoles(possibleRoles); localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId()); // create the environment based on the finalized (processed) view of the settings diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index dcba40b58c21a..08a359bab39fc 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.component.LifecycleComponent; import org.elasticsearch.common.inject.Module; @@ -50,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.UnaryOperator; /** @@ -236,6 +238,10 @@ public List> getExecutorBuilders(Settings settings) { */ public List getBootstrapChecks() { return Collections.emptyList(); } + public Set getRoles() { + return Set.of(); + } + /** * Close the resources opened by this plugin. * diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 74a103dcb540c..f623408597382 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; @@ -324,7 +325,7 @@ Table buildTable(boolean fullId, RestRequest req, ClusterStateResponse state, No if (node.getRoles().isEmpty()) { roles = "-"; } else { - roles = node.getRoles().stream().map(DiscoveryNode.Role::getAbbreviation).collect(Collectors.joining()); + roles = node.getRoles().stream().map(DiscoveryNodeRole::roleNameAbbreviation).sorted().collect(Collectors.joining()); } table.addCell(roles); table.addCell(masterId == null ? "x" : masterId.equals(node.getId()) ? "*" : "-"); diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 316fcd275a5a0..3126963f36930 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -22,6 +22,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.ClusterNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.collect.Tuple; @@ -37,7 +38,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -136,7 +136,7 @@ static DiscoveryNode buildSeedNode(String clusterName, String address, boolean p TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 0); String hostName = address.substring(0, indexOfPortSeparator(address)); return new DiscoveryNode("", clusterName + "#" + address, UUIDs.randomBase64UUID(), hostName, address, - transportAddress, Collections.singletonMap("server_name", hostName), EnumSet.allOf(DiscoveryNode.Role.class), + transportAddress, Collections.singletonMap("server_name", hostName), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT.minimumCompatibilityVersion()); } else { TransportAddress transportAddress = new TransportAddress(RemoteClusterAware.parseSeedAddress(address)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java index 52a5e7d3ce3dd..44880d5bdde59 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/AddVotingConfigExclusionsRequestTests.java @@ -25,16 +25,16 @@ import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Set; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -55,14 +55,29 @@ public void testSerialization() throws IOException { } public void testResolve() { - final DiscoveryNode localNode - = new DiscoveryNode("local", "local", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode localNode = new DiscoveryNode( + "local", + "local", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion localNodeExclusion = new VotingConfigExclusion(localNode); - final DiscoveryNode otherNode1 - = new DiscoveryNode("other1", "other1", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode otherNode1 = new DiscoveryNode( + "other1", + "other1", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion otherNode1Exclusion = new VotingConfigExclusion(otherNode1); - final DiscoveryNode otherNode2 - = new DiscoveryNode("other2", "other2", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode otherNode2 = new DiscoveryNode( + "other2", + "other2", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion otherNode2Exclusion = new VotingConfigExclusion(otherNode2); final DiscoveryNode otherDataNode = new DiscoveryNode("data", "data", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -85,14 +100,29 @@ public void testResolve() { } public void testResolveAndCheckMaximum() { - final DiscoveryNode localNode - = new DiscoveryNode("local", "local", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode localNode = new DiscoveryNode( + "local", + "local", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion localNodeExclusion = new VotingConfigExclusion(localNode); - final DiscoveryNode otherNode1 - = new DiscoveryNode("other1", "other1", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode otherNode1 = new DiscoveryNode( + "other1", + "other1", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion otherNode1Exclusion = new VotingConfigExclusion(otherNode1); - final DiscoveryNode otherNode2 - = new DiscoveryNode("other2", "other2", buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + final DiscoveryNode otherNode2 = new DiscoveryNode( + "other2", + "other2", + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); final VotingConfigExclusion otherNode2Exclusion = new VotingConfigExclusion(otherNode2); final ClusterState.Builder builder = ClusterState.builder(new ClusterName("cluster")).nodes(new Builder() diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java index 06d21fa4ba15b..d4dda501f5395 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsActionTests.java @@ -28,12 +28,12 @@ import org.elasticsearch.cluster.ClusterStateObserver.Listener; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.coordination.CoordinationMetaData; -import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfigExclusion; +import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes.Builder; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; @@ -60,7 +60,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; import static org.elasticsearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction.MAXIMUM_VOTING_CONFIG_EXCLUSIONS_SETTING; import static org.elasticsearch.cluster.ClusterState.builder; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; @@ -96,7 +95,13 @@ public static void createThreadPoolAndClusterService() { } private static DiscoveryNode makeDiscoveryNode(String name) { - return new DiscoveryNode(name, name, buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), Version.CURRENT); + return new DiscoveryNode( + name, + name, + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); } @AfterClass diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java index 427aaa45cd678..ca3888aa7f807 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java @@ -25,7 +25,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -63,9 +63,9 @@ public void testNodeCounts() { int total = 1; internalCluster().startNode(); Map expectedCounts = new HashMap<>(); - expectedCounts.put(DiscoveryNode.Role.DATA.getRoleName(), 1); - expectedCounts.put(DiscoveryNode.Role.MASTER.getRoleName(), 1); - expectedCounts.put(DiscoveryNode.Role.INGEST.getRoleName(), 1); + expectedCounts.put(DiscoveryNodeRole.DATA_ROLE.roleName(), 1); + expectedCounts.put(DiscoveryNodeRole.MASTER_ROLE.roleName(), 1); + expectedCounts.put(DiscoveryNodeRole.INGEST_ROLE.roleName(), 1); expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0); int numNodes = randomIntBetween(1, 5); @@ -84,13 +84,13 @@ public void testNodeCounts() { waitForNodes(total); if (isDataNode) { - incrementCountForRole(DiscoveryNode.Role.DATA.getRoleName(), expectedCounts); + incrementCountForRole(DiscoveryNodeRole.DATA_ROLE.roleName(), expectedCounts); } if (isMasterNode) { - incrementCountForRole(DiscoveryNode.Role.MASTER.getRoleName(), expectedCounts); + incrementCountForRole(DiscoveryNodeRole.MASTER_ROLE.roleName(), expectedCounts); } if (isIngestNode) { - incrementCountForRole(DiscoveryNode.Role.INGEST.getRoleName(), expectedCounts); + incrementCountForRole(DiscoveryNodeRole.INGEST_ROLE.roleName(), expectedCounts); } if (!isDataNode && !isMasterNode && !isIngestNode) { incrementCountForRole(ClusterStatsNodes.Counts.COORDINATING_ONLY, expectedCounts); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 580ea76429388..eaea5babe3b7b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -219,7 +220,7 @@ private DiscoveryNode newNode(String nodeId) { nodeId, buildNewFakeTransportAddress(), emptyMap(), - Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA), + Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java index 7ecc172924b1c..b405b0790debc 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; @@ -154,12 +155,12 @@ public void testDefaultMaxConcurrentSearches() { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); for (int i = 0; i < numDataNodes; i++) { builder.add(new DiscoveryNode("_id" + i, buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT)); + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT)); } builder.add(new DiscoveryNode("master", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT)); + Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT)); builder.add(new DiscoveryNode("ingest", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.INGEST), Version.CURRENT)); + Collections.singleton(DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT)); ClusterState state = ClusterState.builder(new ClusterName("_name")).nodes(builder).build(); int result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state); diff --git a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java index 3967123ad5065..a42791e69f658 100644 --- a/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java @@ -34,14 +34,15 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException; import org.elasticsearch.discovery.MasterNotDiscoveredException; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -95,9 +96,9 @@ public void setUp() throws Exception { transportService.start(); transportService.acceptIncomingRequests(); localNode = new DiscoveryNode("local_node", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT); + Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); remoteNode = new DiscoveryNode("remote_node", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT); + Collections.singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); allNodes = new DiscoveryNode[]{localNode, remoteNode}; } diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index 0a2b495e2b820..46c5b74a94365 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; @@ -43,7 +44,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -188,7 +188,7 @@ public void setUp() throws Exception { List discoveryNodes = new ArrayList<>(); for (int i = 0; i < numNodes; i++) { Map attributes = new HashMap<>(); - Set roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))); + Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); if (frequently()) { attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5)); } @@ -235,7 +235,7 @@ public DataNodesOnlyTransportNodesAction getDataNodesOnlyTransportNodesAction(Tr ); } - private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { + private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { String node = "node_" + nodeId; return new DiscoveryNode(node, node, buildNewFakeTransportAddress(), attributes, roles, Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 6b628d88c59d4..a427cd7297bf3 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -41,7 +42,6 @@ import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -407,7 +407,7 @@ public static ClusterState state(DiscoveryNode localNode, DiscoveryNode masterNo private static DiscoveryNode newNode(int nodeId) { return new DiscoveryNode("node_" + nodeId, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); } private static String selectAndRemove(Set strings) { diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java index 28373347b19ca..d5eb6a81bf24c 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationAllPermitsAcquisitionTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -64,7 +65,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -128,7 +128,7 @@ public void setUp() throws Exception { clusterService = createClusterService(threadPool); final ClusterState.Builder state = ClusterState.builder(clusterService.state()); - Set roles = new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())); + Set roles = new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES); DiscoveryNode node1 = new DiscoveryNode("_name1", "_node1", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("_name2", "_node2", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); state.nodes(DiscoveryNodes.builder() diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java index f79fef74e917f..6de7da8c70991 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.UUIDs; @@ -422,24 +423,24 @@ private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boo final int localNodeIndex = isLocalMaster ? 0 : randomIntBetween(1, numNodes - 1); // randomly assign the local node if not master for (int i = 0; i < numNodes; i++) { final String nodeId = NODE_ID_PREFIX + i; - Set roles = new HashSet<>(); + Set roles = new HashSet<>(); if (i == 0) { // the master node builder.masterNodeId(nodeId); - roles.add(DiscoveryNode.Role.MASTER); + roles.add(DiscoveryNodeRole.MASTER_ROLE); } else if (i == 1) { // the alternate master node - roles.add(DiscoveryNode.Role.MASTER); + roles.add(DiscoveryNodeRole.MASTER_ROLE); } else if (i == 2) { // we need at least one data node - roles.add(DiscoveryNode.Role.DATA); + roles.add(DiscoveryNodeRole.DATA_ROLE); } else { // remaining nodes can be anything (except for master) if (randomBoolean()) { - roles.add(DiscoveryNode.Role.MASTER); + roles.add(DiscoveryNodeRole.MASTER_ROLE); } if (randomBoolean()) { - roles.add(DiscoveryNode.Role.DATA); + roles.add(DiscoveryNodeRole.DATA_ROLE); } } final DiscoveryNode node = newNode(nodeId, roles); @@ -452,7 +453,7 @@ private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boo } // Create a new DiscoveryNode - private static DiscoveryNode newNode(final String nodeId, Set roles) { + private static DiscoveryNode newNode(final String nodeId, Set roles) { return new DiscoveryNode(nodeId, nodeId, nodeId, "host", "host_address", buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java index a5679153aeb21..151f4d9268523 100644 --- a/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.coordination.DeterministicTaskQueue; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.UUIDs; @@ -51,7 +52,6 @@ import org.junit.Before; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -80,7 +80,7 @@ public class NodeConnectionsServiceTests extends ESTestCase { private List generateNodes() { List nodes = new ArrayList<>(); for (int i = randomIntBetween(20, 50); i > 0; i--) { - Set roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))); + Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); nodes.add(new DiscoveryNode("node_" + i, "" + i, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT)); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java index 4dd3e3e33c39f..6a7562ab8038a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterBootstrapServiceTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.node.Node; @@ -33,6 +33,7 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -43,7 +44,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.BOOTSTRAP_PLACEHOLDER_PREFIX; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; @@ -88,8 +88,13 @@ protected void onSendRequest(long requestId, String action, TransportRequest req } private DiscoveryNode newDiscoveryNode(String nodeName) { - return new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(), singleton(Role.MASTER), - Version.CURRENT); + return new DiscoveryNode( + nodeName, + randomAlphaOfLength(10), + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT); } public void testBootstrapsAutomaticallyWithDefaultConfiguration() { @@ -390,10 +395,22 @@ public void testCancelsBootstrapIfNodeMatchesMultipleRequirements() { clusterBootstrapService.onFoundPeersUpdated(); deterministicTaskQueue.runAllTasks(); - discoveredNodes.set(Stream.of(new DiscoveryNode(otherNode1.getName(), randomAlphaOfLength(10), buildNewFakeTransportAddress(), - emptyMap(), singleton(Role.MASTER), Version.CURRENT), - new DiscoveryNode("yet-another-node", randomAlphaOfLength(10), otherNode1.getAddress(), emptyMap(), singleton(Role.MASTER), - Version.CURRENT)).collect(Collectors.toList())); + discoveredNodes.set(Stream.of( + new DiscoveryNode( + otherNode1.getName(), + randomAlphaOfLength(10), + buildNewFakeTransportAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT), + new DiscoveryNode( + "yet-another-node", + randomAlphaOfLength(10), + otherNode1.getAddress(), + emptyMap(), + Set.of(DiscoveryNodeRole.MASTER_ROLE), + Version.CURRENT)) + .collect(Collectors.toList())); clusterBootstrapService.onFoundPeersUpdated(); deterministicTaskQueue.runAllTasks(); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java index 2f36d315a24e9..f16c7a7878ea9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -353,8 +354,8 @@ public void testDescriptionAfterBootstrapping() { final DiscoveryNode otherMasterNode = new DiscoveryNode("other-master", buildNewFakeTransportAddress(), Version.CURRENT); final DiscoveryNode otherNonMasterNode = new DiscoveryNode("other-non-master", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(randomSubsetOf(Arrays.stream(DiscoveryNode.Role.values()) - .filter(r -> r != DiscoveryNode.Role.MASTER).collect(Collectors.toList()))), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES).stream() + .filter(r -> r != DiscoveryNodeRole.MASTER_ROLE).collect(Collectors.toList())), Version.CURRENT); String[] configNodeIds = new String[]{"n1", "n2"}; diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java index afe2e065a875e..5bbd3dfd24993 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinationStateTests.java @@ -23,13 +23,13 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; +import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.node.Node; @@ -39,7 +39,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -89,7 +88,7 @@ public static DiscoveryNode createNode(String id) { return new DiscoveryNode("", id, UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(), - EnumSet.allOf(Role.class), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); } public void testSetInitialState() { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index f01be6afbeeee..c23d793b57ee9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -44,7 +44,7 @@ import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; @@ -91,7 +91,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -1959,7 +1958,7 @@ ClusterNode restartedNode(Function adaptGlobalMetaData, Func final DiscoveryNode newLocalNode = new DiscoveryNode(localNode.getName(), localNode.getId(), UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(), - localNode.isMasterNode() ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT); + localNode.isMasterNode() ? DiscoveryNodeRole.BUILT_IN_ROLES : emptySet(), Version.CURRENT); return new ClusterNode(nodeIndex, newLocalNode, node -> new MockPersistedState(newLocalNode, persistedState, adaptGlobalMetaData, adaptCurrentTerm), nodeSettings); } @@ -2322,7 +2321,7 @@ private static DiscoveryNode createDiscoveryNode(int nodeIndex, boolean masterEl return new DiscoveryNode("", "node" + nodeIndex, UUIDs.randomBase64UUID(random()), // generated deterministically for repeatable tests address.address().getHostString(), address.getAddress(), address, Collections.emptyMap(), - masterEligible ? EnumSet.allOf(Role.class) : emptySet(), Version.CURRENT); + masterEligible ? DiscoveryNodeRole.BUILT_IN_ROLES : emptySet(), Version.CURRENT); } /** diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java index a57810b88e500..dc763f403d257 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.coordination.Coordinator.Mode; import org.elasticsearch.cluster.coordination.FollowersChecker.FollowerCheckRequest; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; @@ -43,7 +44,6 @@ import org.elasticsearch.transport.TransportService; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -577,13 +577,13 @@ private static List randomNodes(final int numNodes) { attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5)); } final DiscoveryNode node = newNode(i, attributes, - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())))); + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES))); nodesList.add(node); } return nodesList; } - private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { + private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles, Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java index 1000b5d4eba73..5bbdd020433b0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinTests.java @@ -22,11 +22,12 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.cluster.service.MasterServiceTests; @@ -55,7 +56,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Random; @@ -189,9 +189,11 @@ protected DiscoveryNode newNode(int i) { } protected DiscoveryNode newNode(int i, boolean master) { - Set roles = new HashSet<>(); + final Set roles; if (master) { - roles.add(DiscoveryNode.Role.MASTER); + roles = Set.of(DiscoveryNodeRole.MASTER_ROLE); + } else { + roles = Set.of(); } final String prefix = master ? "master_" : "data_"; return new DiscoveryNode(prefix + i, i + "", buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java index d4cab5110eef2..5fb56cc4aa3a3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/PublicationTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.coordination.CoordinationMetaData.VotingConfiguration; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Tuple; @@ -464,13 +465,13 @@ private static List randomNodes(final int numNodes) { attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5)); } final DiscoveryNode node = newNode(i, attributes, - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())))); + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES))); nodesList.add(node); } return nodesList; } - private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { + private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles, Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java index b96d5eacb15d1..f78104201c9fb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/AutoExpandReplicasTests.java @@ -25,11 +25,11 @@ import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.indices.cluster.ClusterStateChanges; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -104,11 +104,9 @@ public void testInvalidValues() { private static final AtomicInteger nodeIdGenerator = new AtomicInteger(); - protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) { - Set roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values()))); - for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) { - roles.add(mustHaveRole); - } + protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) { + Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); + Collections.addAll(roles, mustHaveRoles); final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet()); return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT); @@ -126,12 +124,12 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE try { List allNodes = new ArrayList<>(); - DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master + DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master allNodes.add(localNode); int numDataNodes = randomIntBetween(3, 5); List dataNodes = new ArrayList<>(numDataNodes); for (int i = 0; i < numDataNodes; i++) { - dataNodes.add(createNode(DiscoveryNode.Role.DATA)); + dataNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE)); } allNodes.addAll(dataNodes); ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0])); @@ -179,7 +177,7 @@ public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedE .collect(Collectors.toList()); if (randomBoolean()) { - nodesToAdd.add(createNode(DiscoveryNode.Role.DATA)); + nodesToAdd.add(createNode(DiscoveryNodeRole.DATA_ROLE)); } state = cluster.joinNodesAndBecomeMaster(state, nodesToAdd); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index cda92d3953ea3..3eef8ff686eef 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRoutingState; @@ -397,7 +398,7 @@ private DiscoveryNode newNode(String nodeId) { nodeId, buildNewFakeTransportAddress(), emptyMap(), - Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA), Version.CURRENT); + Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); } public void testValidateIndexName() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java index d9a8973c1183b..96ce35dda77df 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -151,9 +152,9 @@ public void testCloseRoutingTableRemovesRoutingTable() { state = ClusterState.builder(state) .nodes(DiscoveryNodes.builder(state.nodes()) .add(new DiscoveryNode("old_node", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_7_0_0)) + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.V_7_0_0)) .add(new DiscoveryNode("new_node", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.V_7_2_0))) + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.V_7_2_0))) .build(); state = MetaDataIndexStateService.closeRoutingTable(state, blockedIndices, results).v1(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java index 4bff7f8dc6198..b6b812b5219ee 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/TemplateUpgradeServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; @@ -46,7 +47,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -247,8 +247,8 @@ public void testUpdateTemplates() { assertThat(service.upgradesInProgress.get(), equalTo(2)); } - private static final Set MASTER_DATA_ROLES = - Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA)); + private static final Set MASTER_DATA_ROLES = + Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE); @SuppressWarnings("unchecked") public void testClusterStateUpdate() throws InterruptedException { diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java new file mode 100644 index 0000000000000..06c4b14cae417 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleIT.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.node; + +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.hamcrest.Matcher; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.not; + +public class DiscoveryNodeRoleIT extends ESIntegTestCase { + + public static class AdditionalRolePlugin extends Plugin { + + public AdditionalRolePlugin() { + + } + + static final Setting NODE_ADDITIONAL_SETTING = + Setting.boolSetting("node.additional", true, Setting.Property.NodeScope); + + static DiscoveryNodeRole ADDITIONAL_ROLE = new DiscoveryNodeRole("additional", "a") { + + @Override + protected Setting roleSetting() { + return NODE_ADDITIONAL_SETTING; + } + + }; + + @Override + public Set getRoles() { + return Set.of(ADDITIONAL_ROLE); + } + + @Override + public List> getSettings() { + return List.of(NODE_ADDITIONAL_SETTING); + } + + } + + @Override + protected Collection> nodePlugins() { + return List.of(AdditionalRolePlugin.class); + } + + public void testDefaultHasAdditionalRole() { + runTestNodeHasAdditionalRole(Settings.EMPTY); + } + + public void testExplicitlyHasAdditionalRole() { + runTestNodeHasAdditionalRole(Settings.builder().put(AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.getKey(), true).build()); + } + + public void testDoesNotHaveAdditionalRole() { + runTestNodeHasAdditionalRole(Settings.builder().put(AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.getKey(), false).build()); + } + + private void runTestNodeHasAdditionalRole(final Settings settings) { + final String name = internalCluster().startNode(settings); + final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo(name).get(); + assertThat(response.getNodes(), hasSize(1)); + final Matcher> matcher; + if (AdditionalRolePlugin.NODE_ADDITIONAL_SETTING.get(settings)) { + matcher = hasItem(AdditionalRolePlugin.ADDITIONAL_ROLE); + } else { + matcher = not(hasItem(AdditionalRolePlugin.ADDITIONAL_ROLE)); + } + assertThat(response.getNodes().get(0).getNode().getRoles(), matcher); + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java new file mode 100644 index 0000000000000..9ef11bf6a8c48 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.node; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.test.ESTestCase; + +import java.util.Set; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasToString; + +public class DiscoveryNodeRoleTests extends ESTestCase { + + public void testDiscoveryNodeSetPossibleRolesRejectsDuplicateRoleNames() { + final IllegalStateException e = expectThrows( + IllegalStateException.class, + () -> DiscoveryNode.setPossibleRoles(Set.of( + new DiscoveryNodeRole("foo", "f") { + + @Override + protected Setting roleSetting() { + return null; + } + + }, + new DiscoveryNodeRole("foo", "f") { + + @Override + protected Setting roleSetting() { + return null; + } + + }))); + assertThat(e, hasToString(containsString("Duplicate key foo "))); + } + + public void testDiscoveryNodeSetPossibleRolesRejectsDuplicateRoleNameAbbreviations() { + final IllegalStateException e = expectThrows( + IllegalStateException.class, + () -> DiscoveryNode.setPossibleRoles(Set.of( + new DiscoveryNodeRole("foo_1", "f") { + + @Override + protected Setting roleSetting() { + return null; + } + + }, + new DiscoveryNodeRole("foo_2", "f") { + + @Override + protected Setting roleSetting() { + return null; + } + + }))); + assertThat(e, hasToString(containsString("Duplicate key f "))); + } + +} diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java index 5c92029520e8d..9a5186618b333 100644 --- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java @@ -241,7 +241,7 @@ private static List randomNodes(final int numNodes) { attributes.put("custom", randomBoolean() ? "match" : randomAlphaOfLengthBetween(3, 5)); } final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes, - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())))); + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES))); nodesList.add(node); } return nodesList; @@ -259,7 +259,7 @@ private static DiscoveryNodes buildDiscoveryNodes() { return discoBuilder.build(); } - private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { + private static DiscoveryNode newNode(int nodeId, Map attributes, Set roles) { return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, buildNewFakeTransportAddress(), attributes, roles, Version.CURRENT); } @@ -275,21 +275,21 @@ Set matchingNodeIds(DiscoveryNodes nodes) { Set matchingNodeIds(DiscoveryNodes nodes) { return Collections.singleton(nodes.getMasterNodeId()); } - }, MASTER_ELIGIBLE(DiscoveryNode.Role.MASTER.getRoleName() + ":true") { + }, MASTER_ELIGIBLE(DiscoveryNodeRole.MASTER_ROLE.roleName() + ":true") { @Override Set matchingNodeIds(DiscoveryNodes nodes) { Set ids = new HashSet<>(); nodes.getMasterNodes().keysIt().forEachRemaining(ids::add); return ids; } - }, DATA(DiscoveryNode.Role.DATA.getRoleName() + ":true") { + }, DATA(DiscoveryNodeRole.DATA_ROLE.roleName() + ":true") { @Override Set matchingNodeIds(DiscoveryNodes nodes) { Set ids = new HashSet<>(); nodes.getDataNodes().keysIt().forEachRemaining(ids::add); return ids; } - }, INGEST(DiscoveryNode.Role.INGEST.getRoleName() + ":true") { + }, INGEST(DiscoveryNodeRole.INGEST_ROLE.roleName() + ":true") { @Override Set matchingNodeIds(DiscoveryNodes nodes) { Set ids = new HashSet<>(); @@ -328,13 +328,13 @@ Set matchingNodeIds(DiscoveryNodes nodes) { public void testMaxMinNodeVersion() { DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); discoBuilder.add(new DiscoveryNode("name_" + 1, "node_" + 1, buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), Version.fromString("5.1.0"))); discoBuilder.add(new DiscoveryNode("name_" + 2, "node_" + 2, buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), Version.fromString("6.3.0"))); discoBuilder.add(new DiscoveryNode("name_" + 3, "node_" + 3, buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), Version.fromString("1.1.0"))); discoBuilder.localNodeId("name_1"); discoBuilder.masterNodeId("name_2"); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java index 6897380ee9f01..098803e3cfd53 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.cluster.ESAllocationTestCase; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; @@ -223,7 +223,7 @@ public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exc ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData) .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build()) .nodes(DiscoveryNodes.builder() - .add(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0") + .add(newNode("node0", singleton(DiscoveryNodeRole.MASTER_ROLE))).localNodeId("node0").masterNodeId("node0") .add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).add(newNode("node4"))).build(); // allocate shards clusterState = allocationService.reroute(clusterState, "reroute"); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java index 1405be54fd51e..f3dbdd0eb2552 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; @@ -60,7 +61,6 @@ import org.elasticsearch.index.shard.ShardNotFoundException; import java.util.Collections; -import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -160,7 +160,7 @@ public void testAllocateCommand() { .add(newNode("node1")) .add(newNode("node2")) .add(newNode("node3")) - .add(newNode("node4", singleton(DiscoveryNode.Role.MASTER))) + .add(newNode("node4", singleton(DiscoveryNodeRole.MASTER_ROLE))) ).build(); clusterState = allocation.reroute(clusterState, "reroute"); assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); @@ -620,7 +620,7 @@ public void testMoveShardToNonDataNode() { DiscoveryNode node1 = new DiscoveryNode("node1", "node1", "node1", "test1", "test1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("node2", "node2", "node2", "test2", "test2", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(randomSubsetOf(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST))), Version.CURRENT); + new HashSet<>(randomSubsetOf(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE))), Version.CURRENT); clusterState = ClusterState.builder(clusterState).nodes( DiscoveryNodes.builder() @@ -659,7 +659,7 @@ public void testMoveShardFromNonDataNode() { DiscoveryNode node1 = new DiscoveryNode("node1", "node1", "node1", "test1", "test1", buildNewFakeTransportAddress(), emptyMap(), MASTER_DATA_ROLES, Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("node2", "node2", "node2", "test2", "test2", buildNewFakeTransportAddress(), emptyMap(), - new HashSet<>(randomSubsetOf(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST))), Version.CURRENT); + new HashSet<>(randomSubsetOf(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE))), Version.CURRENT); clusterState = ClusterState.builder(clusterState).nodes( DiscoveryNodes.builder() diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java index 2e56ae6297b89..6f8f75d48fe37 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.cluster.routing.allocation; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; @@ -33,6 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; @@ -40,7 +40,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.indices.cluster.ClusterStateChanges; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.TestThreadPool; @@ -210,11 +209,11 @@ private static Version getNodeVersion(ShardRouting shardRouting, ClusterState st public ClusterState randomInitialClusterState() { List allNodes = new ArrayList<>(); - DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master + DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master allNodes.add(localNode); // at least two nodes that have the data role so that we can allocate shards - allNodes.add(createNode(DiscoveryNode.Role.DATA)); - allNodes.add(createNode(DiscoveryNode.Role.DATA)); + allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE)); + allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE)); for (int i = 0; i < randomIntBetween(2, 5); i++) { allNodes.add(createNode()); } @@ -223,11 +222,9 @@ public ClusterState randomInitialClusterState() { } - protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) { - Set roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values()))); - for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) { - roles.add(mustHaveRole); - } + protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) { + Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); + Collections.addAll(roles, mustHaveRoles); final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet()); return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, VersionUtils.randomIndexCompatibleVersion(random())); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 24838b22d47e2..cf8abf6ccd6e5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -912,9 +913,9 @@ public void testForSingleDataNode() { logger.info("--> adding one master node, one data node"); DiscoveryNode discoveryNode1 = new DiscoveryNode("", "node1", buildNewFakeTransportAddress(), emptyMap(), - singleton(DiscoveryNode.Role.MASTER), Version.CURRENT); + singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); DiscoveryNode discoveryNode2 = new DiscoveryNode("", "node2", buildNewFakeTransportAddress(), emptyMap(), - singleton(DiscoveryNode.Role.DATA), Version.CURRENT); + singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(discoveryNode1).add(discoveryNode2).build(); ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) @@ -977,7 +978,7 @@ Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLU // Add another datanode, it should relocate. logger.info("--> adding node3"); DiscoveryNode discoveryNode3 = new DiscoveryNode("", "node3", buildNewFakeTransportAddress(), emptyMap(), - singleton(DiscoveryNode.Role.DATA), Version.CURRENT); + singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); ClusterState updateClusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()) .add(discoveryNode3)).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index 77967788adffb..47c4cb60331c4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource; @@ -47,7 +48,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -73,9 +73,9 @@ public void testCanAllocateUsesMaxAvailableSpace() { ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); RoutingTable routingTable = RoutingTable.builder() .addAsNew(metaData.index("test")) @@ -129,9 +129,9 @@ public void testCannotAllocateDueToLackOfDiskResources() { ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); RoutingTable routingTable = RoutingTable.builder() .addAsNew(metaData.index("test")) @@ -178,9 +178,9 @@ public void testCanRemainUsesLeastAvailableSpace() { ImmutableOpenMap.Builder shardRoutingMap = ImmutableOpenMap.builder(); DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java index 60e3e2438c19f..cfb0979a9ee40 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/RestoreInProgressAllocationDeciderTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -171,7 +172,7 @@ private ClusterState createInitialClusterState() { .build(); DiscoveryNodes discoveryNodes = DiscoveryNodes.builder() - .add(newNode("master", Collections.singleton(DiscoveryNode.Role.MASTER))) + .add(newNode("master", Collections.singleton(DiscoveryNodeRole.MASTER_ROLE))) .localNodeId("master") .masterNodeId("master") .build(); diff --git a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java index 2a45947706298..2a38942eb5a22 100644 --- a/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -43,11 +44,11 @@ public class AsyncShardFetchTests extends ESTestCase { private final DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); private final Response response1 = new Response(node1); private final Throwable failure1 = new Throwable("simulated failure 1"); private final DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); private final Response response2 = new Response(node2); private final Throwable failure2 = new Throwable("simulate failure 2"); diff --git a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java index cf4067694c2d7..fd985132bebbd 100644 --- a/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/ClusterStateUpdatersTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; @@ -279,7 +280,7 @@ public void testSetLocalNode() { .metaData(metaData) .build(); final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), - Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT); + Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); final ClusterState updatedState = setLocalNode(initialState, localNode); @@ -322,7 +323,7 @@ public void testHideStateIfNotRecovered() { .blocks(ClusterBlocks.builder().addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)) .build(); final DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), - Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT); + Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); final ClusterState updatedState = Function.identity() .andThen(state -> setLocalNode(state, localNode)) .andThen(ClusterStateUpdaters::recoverClusterBlocks) diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java index 8ccfa5e406ae2..13348cef75fd4 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStatePersistedStateTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; @@ -50,7 +51,7 @@ public class GatewayMetaStatePersistedStateTests extends ESTestCase { public void setUp() throws Exception { nodeEnvironment = newNodeEnvironment(); localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), - Sets.newHashSet(DiscoveryNode.Role.MASTER), Version.CURRENT); + Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); clusterName = new ClusterName(randomAlphaOfLength(10)); settings = Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName.value()).build(); super.setUp(); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index 22259b919ec6f..2f9c290717a72 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -29,7 +29,7 @@ import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; -import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; @@ -144,7 +144,7 @@ private ClusterState clusterStateWithJustOpenedIndex(IndexMetaData indexMetaData } private DiscoveryNodes.Builder generateDiscoveryNodes(boolean masterEligible) { - Set dataOnlyRoles = Collections.singleton(DiscoveryNode.Role.DATA); + Set dataOnlyRoles = Set.of(DiscoveryNodeRole.DATA_ROLE); return DiscoveryNodes.builder().add(newNode("node1", masterEligible ? MASTER_DATA_ROLES : dataOnlyRoles)) .add(newNode("master_node", MASTER_DATA_ROLES)).localNodeId("node1").masterNodeId(masterEligible ? "node1" : "master_node"); } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java index 978c5b739e0f2..64d10831c8e9e 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/FakeThreadPoolMasterServiceTests.java @@ -26,10 +26,10 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -44,7 +44,7 @@ public void testFakeMasterService() { List runnableTasks = new ArrayList<>(); AtomicReference lastClusterStateRef = new AtomicReference<>(); DiscoveryNode discoveryNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); + new HashSet<>(DiscoveryNodeRole.BUILT_IN_ROLES), Version.CURRENT); lastClusterStateRef.set(ClusterStateCreationUtils.state(discoveryNode, discoveryNode)); long firstClusterStateVersion = lastClusterStateRef.get().version(); AtomicReference> publishingCallback = new AtomicReference<>(); diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 9fd7f24db024b..ea07fb7c149a2 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -257,11 +258,11 @@ public void testInitializingPrimaryRemovesInitializingReplicaWithSameAID() { public ClusterState randomInitialClusterState(Map clusterStateServiceMap, Supplier indicesServiceSupplier) { List allNodes = new ArrayList<>(); - DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master + DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE); // local node is the master allNodes.add(localNode); // at least two nodes that have the data role so that we can allocate shards - allNodes.add(createNode(DiscoveryNode.Role.DATA)); - allNodes.add(createNode(DiscoveryNode.Role.DATA)); + allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE)); + allNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE)); for (int i = 0; i < randomIntBetween(2, 5); i++) { allNodes.add(createNode()); } @@ -436,11 +437,9 @@ public ClusterState randomlyUpdateClusterState(ClusterState state, private static final AtomicInteger nodeIdGenerator = new AtomicInteger(); - protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) { - Set roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values()))); - for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) { - roles.add(mustHaveRole); - } + protected DiscoveryNode createNode(DiscoveryNodeRole... mustHaveRoles) { + Set roles = new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)); + Collections.addAll(roles, mustHaveRoles); final String id = String.format(Locale.ROOT, "node_%03d", nodeIdGenerator.incrementAndGet()); return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT); diff --git a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java index f29953633a1bb..1aa3e607519f1 100644 --- a/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/persistent/PersistentTasksClusterServiceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.service.ClusterService; @@ -793,7 +794,7 @@ private DiscoveryNode newNode(String nodeId) { nodeId, buildNewFakeTransportAddress(), emptyMap(), - Set.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA), + Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 72290b02e958b..6c6b136ba2751 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -97,6 +97,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.metadata.MetaDataMappingService; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingService; import org.elasticsearch.cluster.routing.ShardRouting; @@ -804,14 +805,14 @@ public TestClusterNode nodeById(final String nodeId) { } private TestClusterNode newMasterNode(String nodeName) throws IOException { - return newNode(nodeName, DiscoveryNode.Role.MASTER); + return newNode(nodeName, DiscoveryNodeRole.MASTER_ROLE); } private TestClusterNode newDataNode(String nodeName) throws IOException { - return newNode(nodeName, DiscoveryNode.Role.DATA); + return newNode(nodeName, DiscoveryNodeRole.DATA_ROLE); } - private TestClusterNode newNode(String nodeName, DiscoveryNode.Role role) throws IOException { + private TestClusterNode newNode(String nodeName, DiscoveryNodeRole role) throws IOException { return new TestClusterNode( new DiscoveryNode(nodeName, randomAlphaOfLength(10), buildNewFakeTransportAddress(), emptyMap(), Collections.singleton(role), Version.CURRENT), this::getDisruption); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java index 1105fe137e322..9b0f687fc0738 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.AbstractScopedSettings; @@ -42,7 +43,6 @@ import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -739,49 +739,49 @@ public void testGetNodePredicateNodeRoles() { Predicate nodePredicate = RemoteClusterService.getNodePredicate(Settings.EMPTY); { DiscoveryNode all = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class)), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); assertTrue(nodePredicate.test(all)); } { DiscoveryNode dataMaster = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA, DiscoveryNode.Role.MASTER)), Version.CURRENT); + Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); assertTrue(nodePredicate.test(dataMaster)); } { DiscoveryNode dedicatedMaster = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.MASTER)), Version.CURRENT); + Set.of(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); assertFalse(nodePredicate.test(dedicatedMaster)); } { DiscoveryNode dedicatedIngest = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.INGEST)), Version.CURRENT); + Set.of(DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT); assertTrue(nodePredicate.test(dedicatedIngest)); } { DiscoveryNode masterIngest = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER)), Version.CURRENT); + Set.of(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); assertTrue(nodePredicate.test(masterIngest)); } { DiscoveryNode dedicatedData = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA)), Version.CURRENT); + Set.of(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); assertTrue(nodePredicate.test(dedicatedData)); } { DiscoveryNode ingestData = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.of(DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST)), Version.CURRENT); + Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE), Version.CURRENT); assertTrue(nodePredicate.test(ingestData)); } { DiscoveryNode coordOnly = new DiscoveryNode("id", address, Collections.emptyMap(), - new HashSet<>(EnumSet.noneOf(DiscoveryNode.Role.class)), Version.CURRENT); + Set.of(), Version.CURRENT); assertTrue(nodePredicate.test(coordOnly)); } } public void testGetNodePredicateNodeVersion() { TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0); - Set roles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class)); + Set roles = DiscoveryNodeRole.BUILT_IN_ROLES; Predicate nodePredicate = RemoteClusterService.getNodePredicate(Settings.EMPTY); Version version = VersionUtils.randomVersion(random()); DiscoveryNode node = new DiscoveryNode("id", address, Collections.emptyMap(), roles, version); @@ -790,7 +790,7 @@ public void testGetNodePredicateNodeVersion() { public void testGetNodePredicateNodeAttrs() { TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0); - Set roles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class)); + Set roles = DiscoveryNodeRole.BUILT_IN_ROLES; Settings settings = Settings.builder().put("cluster.remote.node.attr", "gateway").build(); Predicate nodePredicate = RemoteClusterService.getNodePredicate(settings); { @@ -816,8 +816,8 @@ public void testGetNodePredicatesCombination() { TransportAddress address = new TransportAddress(TransportAddress.META_ADDRESS, 0); Settings settings = Settings.builder().put("cluster.remote.node.attr", "gateway").build(); Predicate nodePredicate = RemoteClusterService.getNodePredicate(settings); - Set allRoles = new HashSet<>(EnumSet.allOf(DiscoveryNode.Role.class)); - Set dedicatedMasterRoles = new HashSet<>(EnumSet.of(DiscoveryNode.Role.MASTER)); + Set allRoles = DiscoveryNodeRole.BUILT_IN_ROLES; + Set dedicatedMasterRoles = Set.of(DiscoveryNodeRole.MASTER_ROLE); { DiscoveryNode node = new DiscoveryNode("id", address, Collections.singletonMap("gateway", "true"), dedicatedMasterRoles, Version.CURRENT); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index b3681247b4e9c..9be598bb384bd 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; @@ -43,7 +44,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Random; @@ -94,8 +94,8 @@ public static AllocationDeciders randomAllocationDeciders(Settings settings, Clu return new AllocationDeciders(deciders); } - protected static Set MASTER_DATA_ROLES = - Collections.unmodifiableSet(EnumSet.of(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA)); + protected static Set MASTER_DATA_ROLES = + Collections.unmodifiableSet(Set.of(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE)); protected static DiscoveryNode newNode(String nodeId) { return newNode(nodeId, Version.CURRENT); @@ -109,7 +109,7 @@ protected static DiscoveryNode newNode(String nodeId, Map attrib return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), attributes, MASTER_DATA_ROLES, Version.CURRENT); } - protected static DiscoveryNode newNode(String nodeId, Set roles) { + protected static DiscoveryNode newNode(String nodeId, Set roles) { return new DiscoveryNode(nodeId, buildNewFakeTransportAddress(), emptyMap(), roles, Version.CURRENT); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index b11a0f84fb84a..081318fd8132c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -49,6 +49,7 @@ import org.elasticsearch.action.support.replication.TransportWriteActionTestHelper; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; @@ -158,7 +159,7 @@ IndexRequest copyIndexRequest(IndexRequest inRequest) throws IOException { protected DiscoveryNode getDiscoveryNode(String id) { return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), - Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT); + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT); } protected class ReplicationGroup implements AutoCloseable, Iterable { diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 2a2176f1c100d..a14fbd1583f89 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; @@ -86,7 +87,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; -import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -562,7 +562,7 @@ protected void recoveryEmptyReplica(IndexShard replica, boolean startReplica) th } protected DiscoveryNode getFakeDiscoNode(String id) { - return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), EnumSet.allOf(DiscoveryNode.Role.class), + return new DiscoveryNode(id, id, buildNewFakeTransportAddress(), Collections.emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java index 5477d292cc2c6..4a34337cdc384 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java @@ -26,7 +26,9 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.coordination.ClusterStatePublisher; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterApplier; import org.elasticsearch.cluster.service.ClusterApplier.ClusterApplyListener; @@ -35,11 +37,9 @@ import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.cluster.coordination.ClusterStatePublisher; import org.elasticsearch.threadpool.ThreadPool; import java.util.Collections; -import java.util.EnumSet; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -123,7 +123,7 @@ public void onFailure(String source, Exception e) { public static ClusterService createClusterService(ThreadPool threadPool) { DiscoveryNode discoveryNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); return createClusterService(threadPool, discoveryNode); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 3eb9ecc4024a9..66a27d10780d4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -52,6 +52,8 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.SuppressForbidden; @@ -347,6 +349,16 @@ public final void before() { } } + @BeforeClass + public static void setPossibleRoles() { + DiscoveryNode.setPossibleRoles(DiscoveryNodeRole.BUILT_IN_ROLES); + } + + @AfterClass + public static void clearPossibleRoles() { + DiscoveryNode.setPossibleRoles(Set.of()); + } + /** * Whether or not we check after each test whether it has left warnings behind. That happens if any deprecated feature or syntax * was used by the test and the test didn't assert on it using {@link #assertWarnings(String...)}. diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index f1d8a0068a35b..881bc8242a612 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -44,7 +44,7 @@ import org.elasticsearch.cluster.coordination.ClusterBootstrapService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNode.Role; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -697,10 +697,10 @@ private String buildNodeName(int id, Settings settings) { private static String getRoleSuffix(Settings settings) { String suffix = ""; if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings)) { - suffix = suffix + Role.MASTER.getAbbreviation(); + suffix = suffix + DiscoveryNodeRole.MASTER_ROLE.roleNameAbbreviation(); } if (Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings)) { - suffix = suffix + Role.DATA.getAbbreviation(); + suffix = suffix + DiscoveryNodeRole.DATA_ROLE.roleNameAbbreviation(); } if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings) == false && Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings) == false @@ -1726,7 +1726,7 @@ private void removeExclusions(Set excludedNodeIds) { public synchronized void fullRestart(RestartCallback callback) throws Exception { int numNodesRestarted = 0; final Settings[] newNodeSettings = new Settings[nextNodeId.get()]; - Map, List> nodesByRoles = new HashMap<>(); + Map, List> nodesByRoles = new HashMap<>(); Set[] rolesOrderedByOriginalStartupOrder = new Set[nextNodeId.get()]; for (NodeAndClient nodeAndClient : nodes.values()) { callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient()); diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 923a79951ce51..f7e0aa526290a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -21,6 +21,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; @@ -53,9 +54,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA; -import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST; -import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER; import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists; @@ -316,33 +314,30 @@ public Path nodeConfigPath(int nodeOrdinal) { } }, 0, "", mockPlugins(), Function.identity()); cluster.beforeTest(random()); - List roles = new ArrayList<>(); + List roles = new ArrayList<>(); for (int i = 0; i < numNodes; i++) { - final DiscoveryNode.Role role = i == numNodes - 1 && roles.contains(MASTER) == false ? - MASTER : // last node and still no master - randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST); + final DiscoveryNodeRole role = i == numNodes - 1 && roles.contains(DiscoveryNodeRole.MASTER_ROLE) == false ? + DiscoveryNodeRole.MASTER_ROLE : // last node and still no master + randomFrom(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.INGEST_ROLE); roles.add(role); } - cluster.setBootstrapMasterNodeIndex(randomIntBetween(0, (int) roles.stream().filter(role -> role.equals(MASTER)).count() - 1)); + cluster.setBootstrapMasterNodeIndex( + randomIntBetween(0, (int) roles.stream().filter(role -> role.equals(DiscoveryNodeRole.MASTER_ROLE)).count() - 1)); try { - Map> pathsPerRole = new HashMap<>(); + Map> pathsPerRole = new HashMap<>(); for (int i = 0; i < numNodes; i++) { - final DiscoveryNode.Role role = roles.get(i); + final DiscoveryNodeRole role = roles.get(i); final String node; - switch (role) { - case MASTER: - node = cluster.startMasterOnlyNode(); - break; - case DATA: - node = cluster.startDataOnlyNode(); - break; - case INGEST: - node = cluster.startCoordinatingOnlyNode(Settings.EMPTY); - break; - default: - throw new IllegalStateException("get your story straight"); + if (role == DiscoveryNodeRole.MASTER_ROLE) { + node = cluster.startMasterOnlyNode(); + } else if (role == DiscoveryNodeRole.DATA_ROLE) { + node = cluster.startDataOnlyNode(); + } else if (role == DiscoveryNodeRole.INGEST_ROLE) { + node = cluster.startCoordinatingOnlyNode(Settings.EMPTY); + } else { + throw new IllegalStateException("get your story straight"); } Set rolePaths = pathsPerRole.computeIfAbsent(role, k -> new HashSet<>()); for (Path path : getNodePaths(cluster, node)) { @@ -352,21 +347,21 @@ public Path nodeConfigPath(int nodeOrdinal) { cluster.validateClusterFormed(); cluster.fullRestart(); - Map> result = new HashMap<>(); + Map> result = new HashMap<>(); for (String name : cluster.getNodeNames()) { DiscoveryNode node = cluster.getInstance(ClusterService.class, name).localNode(); List paths = Arrays.stream(getNodePaths(cluster, name)).map(Path::toString).collect(Collectors.toList()); if (node.isMasterNode()) { - result.computeIfAbsent(MASTER, k -> new HashSet<>()).addAll(paths); + result.computeIfAbsent(DiscoveryNodeRole.MASTER_ROLE, k -> new HashSet<>()).addAll(paths); } else if (node.isDataNode()) { - result.computeIfAbsent(DATA, k -> new HashSet<>()).addAll(paths); + result.computeIfAbsent(DiscoveryNodeRole.DATA_ROLE, k -> new HashSet<>()).addAll(paths); } else { - result.computeIfAbsent(INGEST, k -> new HashSet<>()).addAll(paths); + result.computeIfAbsent(DiscoveryNodeRole.INGEST_ROLE, k -> new HashSet<>()).addAll(paths); } } assertThat(result.size(), equalTo(pathsPerRole.size())); - for (DiscoveryNode.Role role : result.keySet()) { + for (DiscoveryNodeRole role : result.keySet()) { assertThat("path are not the same for " + role, result.get(role), equalTo(pathsPerRole.get(role))); } } finally { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java index eb40d6894f1e0..a44e1e53435c4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/GetDatafeedStatsActionResponseTests.java @@ -14,18 +14,18 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractStreamableTestCase; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response; import org.elasticsearch.xpack.core.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction.Response; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; @@ -75,7 +75,7 @@ public void testDatafeedStatsToXContent() throws IOException { TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 9000); DiscoveryNode node = new DiscoveryNode("df-node-name", "df-node-id", transportAddress, attributes, - EnumSet.noneOf(DiscoveryNode.Role.class), + Set.of(), Version.CURRENT); Response.DatafeedStats stats = new Response.DatafeedStats("df-id", DatafeedState.STARTED, node, null); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java index 9c9263cd78e52..b90ee2c4595a9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -370,10 +371,10 @@ private ClusterState givenNodeCount(int nodeCount) { for (int i = 0; i < nodeCount; i++) { Map attrs = new HashMap<>(); attrs.put(MachineLearning.MAX_OPEN_JOBS_NODE_ATTR, Integer.toString(20)); - Set roles = new HashSet<>(); - roles.add(DiscoveryNode.Role.DATA); - roles.add(DiscoveryNode.Role.MASTER); - roles.add(DiscoveryNode.Role.INGEST); + Set roles = new HashSet<>(); + roles.add(DiscoveryNodeRole.DATA_ROLE); + roles.add(DiscoveryNodeRole.MASTER_ROLE); + roles.add(DiscoveryNodeRole.INGEST_ROLE); nodesBuilder.add(new DiscoveryNode("ml-feature-set-given-ml-node-" + i, new TransportAddress(TransportAddress.META_ADDRESS, 9100 + i), attrs, @@ -382,10 +383,10 @@ private ClusterState givenNodeCount(int nodeCount) { } for (int i = 0; i < randomIntBetween(1, 3); i++) { Map attrs = new HashMap<>(); - Set roles = new HashSet<>(); - roles.add(DiscoveryNode.Role.DATA); - roles.add(DiscoveryNode.Role.MASTER); - roles.add(DiscoveryNode.Role.INGEST); + Set roles = new HashSet<>(); + roles.add(DiscoveryNodeRole.DATA_ROLE); + roles.add(DiscoveryNodeRole.MASTER_ROLE); + roles.add(DiscoveryNodeRole.INGEST_ROLE); nodesBuilder.add(new DiscoveryNode("ml-feature-set-given-non-ml-node-" + i, new TransportAddress(TransportAddress.META_ADDRESS, 9300 + i), attrs, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 94b205b3d85d1..a4e76d60f041c 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; @@ -175,7 +176,7 @@ public void testNodesHash() { randomAlphaOfLength(5), new TransportAddress(TransportAddress.META_ADDRESS, 9301 + i), randomBoolean() ? singletonMap("attr", randomAlphaOfLength(3)) : emptyMap, - singleton(randomFrom(DiscoveryNode.Role.values())), + singleton(randomFrom(DiscoveryNodeRole.BUILT_IN_ROLES)), Version.CURRENT)); } @@ -201,7 +202,7 @@ public void testToXContent() throws IOException { "_host_address", transportAddress, singletonMap("attr", "value"), - singleton(DiscoveryNode.Role.MASTER), + singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); final ClusterState clusterState = ClusterState.builder(clusterName) @@ -441,10 +442,10 @@ public void testToXContent() throws IOException { + "\"nodes\":{" + "\"count\":{" + "\"total\":1," - + "\"data\":0," + "\"coordinating_only\":0," - + "\"master\":1," - + "\"ingest\":0" + + "\"data\":0," + + "\"ingest\":0," + + "\"master\":1" + "}," + "\"versions\":[" + "\"" + mockNodeVersion + "\"" @@ -586,7 +587,7 @@ private DiscoveryNode masterNode() { "_host_address", new TransportAddress(TransportAddress.META_ADDRESS, 9300), singletonMap("attr", "value"), - singleton(DiscoveryNode.Role.MASTER), + singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java index 08e04725c2e0b..f018828c3a46d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -78,7 +79,7 @@ public void testToXContent() throws IOException { "_host_address_0", new TransportAddress(TransportAddress.META_ADDRESS, 9300), singletonMap("attr", "value_0"), - singleton(DiscoveryNode.Role.MASTER), + singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); final DiscoveryNode discoveryNodeOne = new DiscoveryNode("_node_1", @@ -88,7 +89,7 @@ public void testToXContent() throws IOException { "_host_address_1", new TransportAddress(TransportAddress.META_ADDRESS, 9301), singletonMap("attr", "value_1"), - singleton(DiscoveryNode.Role.DATA), + singleton(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT.minimumIndexCompatibilityVersion()); final ShardId shardId = new ShardId("_index_a", "_uuid_a", 0); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java index 2bbe7f6d2523b..2f71af8d0cd2d 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; @@ -86,7 +87,7 @@ public void testToXContent() throws IOException { "_host_address", new TransportAddress(TransportAddress.META_ADDRESS, 9300), singletonMap("attr", "value"), - singleton(DiscoveryNode.Role.MASTER), + singleton(DiscoveryNodeRole.MASTER_ROLE), Version.CURRENT); final ModelSizeStats modelStats = new ModelSizeStats.Builder("_model") diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index 47e5b734c8042..2a4cfb4882966 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -61,6 +62,7 @@ import org.junit.BeforeClass; import javax.crypto.SecretKey; + import java.io.IOException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; @@ -812,7 +814,7 @@ private DiscoveryNode addAnotherDataNodeWithVersion(ClusterService clusterServic final ClusterState currentState = clusterService.state(); final DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(currentState.getNodes()); final DiscoveryNode anotherDataNode = new DiscoveryNode("another_data_node#" + version, buildNewFakeTransportAddress(), - Collections.emptyMap(), Collections.singleton(DiscoveryNode.Role.DATA), version); + Collections.emptyMap(), Collections.singleton(DiscoveryNodeRole.DATA_ROLE), version); discoBuilder.add(anotherDataNode); final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState); newStateBuilder.nodes(discoBuilder); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java index 1b7a7f8d870f2..80802db88f942 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/AbstractSimpleSecurityTransportTestCase.java @@ -8,6 +8,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; import org.elasticsearch.common.settings.MockSecureSettings; @@ -45,6 +46,7 @@ import javax.net.ssl.SSLServerSocket; import javax.net.ssl.SSLServerSocketFactory; import javax.net.ssl.SSLSocket; + import java.io.IOException; import java.io.UncheckedIOException; import java.net.InetAddress; @@ -53,7 +55,6 @@ import java.net.UnknownHostException; import java.nio.file.Path; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; @@ -259,7 +260,7 @@ public boolean matches(SNIServerName sniServerName) { HashMap attributes = new HashMap<>(); attributes.put("server_name", sniIp); DiscoveryNode node = new DiscoveryNode("server_node_id", new TransportAddress(serverAddress), attributes, - EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); new Thread(() -> { try { @@ -306,7 +307,7 @@ public void testInvalidSNIServerName() throws Exception { HashMap attributes = new HashMap<>(); attributes.put("server_name", sniIp); DiscoveryNode node = new DiscoveryNode("server_node_id", new TransportAddress(serverAddress), attributes, - EnumSet.allOf(DiscoveryNode.Role.class), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); ConnectTransportException connectException = expectThrows(ConnectTransportException.class, () -> serviceC.connectToNode(node, TestProfiles.LIGHT_PROFILE)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java index 6cea483995ac9..ef3275c3f9ab8 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherIndexingListenerTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingNode; @@ -432,14 +433,14 @@ public void testOnNonDataNodes() { DiscoveryNode node1 = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Collections.singletonList( - randomFrom(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER))), + randomFrom(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE))), Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("node_2", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Collections.singletonList(DiscoveryNode.Role.DATA)), Version.CURRENT); + new HashSet<>(Collections.singletonList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT); DiscoveryNode node3 = new DiscoveryNode("node_3", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(Collections.singletonList(DiscoveryNode.Role.DATA)), Version.CURRENT); + new HashSet<>(Collections.singletonList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT); IndexMetaData.Builder indexMetaDataBuilder = createIndexBuilder(Watch.INDEX, 1 ,0); @@ -698,6 +699,6 @@ private IndexMetaData.Builder createIndexBuilder(String name, int numberOfShards private static DiscoveryNode newNode(String nodeId) { return new DiscoveryNode(nodeId, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java index 19e2a13487123..548583ac14b7d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.RoutingTable; @@ -327,13 +328,13 @@ public void testNonDataNode() { IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(index).addShard(shardRouting); DiscoveryNode node1 = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(randomFrom(DiscoveryNode.Role.INGEST, DiscoveryNode.Role.MASTER))), Version.CURRENT); + new HashSet<>(asList(randomFrom(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.MASTER_ROLE))), Version.CURRENT); DiscoveryNode node2 = new DiscoveryNode("node_2", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.DATA)), Version.CURRENT); + new HashSet<>(asList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT); DiscoveryNode node3 = new DiscoveryNode("node_3", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.DATA)), Version.CURRENT); + new HashSet<>(asList(DiscoveryNodeRole.DATA_ROLE)), Version.CURRENT); IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(Watch.INDEX) .settings(Settings.builder() @@ -465,7 +466,8 @@ public void testWatcherStopsOnClusterLevelBlock() { } public void testMasterOnlyNodeCanStart() { - List roles = Collections.singletonList(randomFrom(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.INGEST)); + List roles = + Collections.singletonList(randomFrom(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.INGEST_ROLE)); ClusterState state = ClusterState.builder(new ClusterName("my-cluster")) .nodes(new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1") .add(new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), @@ -596,6 +598,6 @@ private static DiscoveryNode newNode(String nodeName) { private static DiscoveryNode newNode(String nodeName, Version version) { return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.values())), version); + DiscoveryNodeRole.BUILT_IN_ROLES, version); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index 3d1fe78e27a00..f4ee831266b38 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; @@ -62,11 +63,9 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.concurrent.ExecutorService; -import static java.util.Arrays.asList; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.mockito.Matchers.any; @@ -277,6 +276,6 @@ void stopExecutor() { private static DiscoveryNode newNode() { return new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java index fd06045204710..cfc55deb131b4 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/ExecutionServiceTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; @@ -87,13 +88,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; -import static java.util.Arrays.asList; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; @@ -156,7 +155,7 @@ public void init() throws Exception { parser = mock(WatchParser.class); DiscoveryNode discoveryNode = new DiscoveryNode("node_1", ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(), - new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT); + DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT); ClusterService clusterService = mock(ClusterService.class); when(clusterService.localNode()).thenReturn(discoveryNode); From a00da6e95398df533d67a0ec5faaa3011c2ffc4b Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Thu, 13 Jun 2019 12:07:40 -0700 Subject: [PATCH 08/11] Allow big integers and decimals to be mapped dynamically. (#42827) This PR proposes to model big integers as longs (and big decimals as doubles) in the context of dynamic mappings. Previously, the dynamic mapping logic did not recognize big integers or decimals, and would an error of the form "No matching token for number_type [BIG_INTEGER]" when a dynamic big integer was encountered. It now accepts these numeric types and interprets them as 'long' and 'double' respectively. This allows `dynamic_templates` to accept and and remap them as another type such as `keyword` or `scaled_float`. Addresses #37846. --- .../common/xcontent/XContentParser.java | 2 +- .../xcontent/json/JsonXContentParser.java | 4 ++ .../index/mapper/DocumentParser.java | 8 ++- .../index/mapper/DocumentParserTests.java | 60 +++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index 93321048d86c4..82a663bd9dc5d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -117,7 +117,7 @@ public boolean isValue() { } enum NumberType { - INT, LONG, FLOAT, DOUBLE + INT, BIG_INTEGER, LONG, FLOAT, DOUBLE, BIG_DECIMAL } XContentType contentType(); diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index be0d01763b893..7489222df2e76 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -199,12 +199,16 @@ private NumberType convertNumberType(JsonParser.NumberType numberType) { switch (numberType) { case INT: return NumberType.INT; + case BIG_INTEGER: + return NumberType.BIG_INTEGER; case LONG: return NumberType.LONG; case FLOAT: return NumberType.FLOAT; case DOUBLE: return NumberType.DOUBLE; + case BIG_DECIMAL: + return NumberType.BIG_DECIMAL; } throw new IllegalStateException("No matching token for number_type [" + numberType + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 9f5293e7da97a..07231759113e9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -764,13 +764,17 @@ private static Mapper.Builder createBuilderFromDynamicValue(final ParseCont return builder; } else if (token == XContentParser.Token.VALUE_NUMBER) { XContentParser.NumberType numberType = context.parser().numberType(); - if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG) { + if (numberType == XContentParser.NumberType.INT + || numberType == XContentParser.NumberType.LONG + || numberType == XContentParser.NumberType.BIG_INTEGER) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG); if (builder == null) { builder = newLongBuilder(currentFieldName); } return builder; - } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE) { + } else if (numberType == XContentParser.NumberType.FLOAT + || numberType == XContentParser.NumberType.DOUBLE + || numberType == XContentParser.NumberType.BIG_DECIMAL) { Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE); if (builder == null) { // no templates are defined, we use float by default instead of double diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index f7f3432aa6ad2..9373a5ceeae20 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; @@ -37,6 +39,8 @@ import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -706,6 +710,62 @@ public void testMappedNullValue() throws Exception { assertEquals(0, doc.rootDoc().getFields("foo").length); } + public void testDynamicBigInteger() throws Exception { + DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startArray("dynamic_templates").startObject() + .startObject("big-integer-to-keyword") + .field("match", "big-*") + .field("match_mapping_type", "long") + .startObject("mapping").field("type", "keyword").endObject() + .endObject() + .endObject().endArray() + .endObject() + .endObject()); + + DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping)); + BigInteger value = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE); + BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .field("big-integer", value) + .endObject()); + ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON)); + + IndexableField[] fields = doc.rootDoc().getFields("big-integer"); + assertEquals(2, fields.length); + assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType); + assertEquals(new BytesRef(value.toString()), fields[0].binaryValue()); + } + + public void testDynamicBigDecimal() throws Exception { + DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startArray("dynamic_templates").startObject() + .startObject("big-decimal-to-scaled-float") + .field("match", "big-*") + .field("match_mapping_type", "double") + .startObject("mapping") + .field("type", "keyword") + .endObject() + .endObject() + .endObject().endArray() + .endObject() + .endObject()); + + BigDecimal value = BigDecimal.valueOf(Double.MAX_VALUE).add(BigDecimal.valueOf(10.1)); + DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping)); + BytesReference bytes = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .field("big-decimal", value) + .endObject()); + ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", bytes, XContentType.JSON)); + + IndexableField[] fields = doc.rootDoc().getFields("big-decimal"); + assertEquals(2, fields.length); + assertTrue(fields[0].fieldType() instanceof KeywordFieldMapper.KeywordFieldType); + assertEquals(new BytesRef(value.toString()), fields[0].binaryValue()); + } + public void testDynamicDottedFieldNameLongArray() throws Exception { DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") From cd3404f104464ce8bd59d416ca1edc8884d82883 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Jun 2019 14:58:19 -0400 Subject: [PATCH 09/11] Adjust BWC version for discovery node roles Now that pluggable node roles are backported to 7.x (versioned at 7.3.0), this commit adjusts the BWC version in master accordingly. --- .../java/org/elasticsearch/cluster/node/DiscoveryNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index 893a496f134dc..f175f5d6f4bf2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -222,7 +222,7 @@ public DiscoveryNode(StreamInput in) throws IOException { } int rolesSize = in.readVInt(); final Set roles = new HashSet<>(rolesSize); - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { for (int i = 0; i < rolesSize; i++) { final String roleName = in.readString(); final String roleNameAbbreviation = in.readString(); @@ -272,7 +272,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { out.writeVInt(roles.size()); for (final DiscoveryNodeRole role : roles) { out.writeString(role.roleName()); From 19909edcfdf5792b38c1363b07379783ebd0e6c4 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Thu, 13 Jun 2019 21:21:32 +0200 Subject: [PATCH 10/11] SQL: Fix wrong results when sorting on aggregate (#43154) - Previously, when shorting on an aggregate function the bucket processing ended early when the explicit (LIMIT XXX) or the impliciti limit of 512 was reached. As a consequence, only a set of grouping buckets was processed and the results returned didn't reflect the global ordering. - Previously, the priority queue shorting method had an inverse comparison check and the final response from the priority queue was also returned in the inversed order because of the calls to the `pop()` method. Fixes: #42851 --- .../sql/qa/single_node/JdbcCsvSpecIT.java | 2 +- .../xpack/sql/qa/ErrorsTestCase.java | 1 + .../sql/qa/cli/CliIntegrationTestCase.java | 8 +- .../xpack/sql/qa/cli/ErrorsTestCase.java | 9 +- .../xpack/sql/qa/cli/FetchSizeTestCase.java | 28 ++++ .../xpack/sql/qa/jdbc/DebugCsvSpec.java | 64 -------- .../xpack/sql/qa/jdbc/ErrorsTestCase.java | 10 ++ .../xpack/sql/qa/jdbc/SqlSpecTestCase.java | 7 + .../xpack/sql/qa/rest/RestSqlTestCase.java | 10 +- .../src/main/resources/agg-ordering.sql-spec | 6 +- .../search/CompositeAggregationCursor.java | 33 ++-- .../execution/search/CompositeAggsRowSet.java | 20 ++- .../xpack/sql/execution/search/Querier.java | 149 +++++++++--------- .../search/SchemaCompositeAggsRowSet.java | 2 +- .../sql/execution/search/SourceGenerator.java | 10 +- .../CompositeAggregationCursorTests.java | 5 +- .../sql/execution/search/QuerierTests.java | 109 +++++++++++++ 17 files changed, 300 insertions(+), 173 deletions(-) delete mode 100644 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java index 428bc1c21ef8f..685a36f483ba6 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java @@ -32,7 +32,7 @@ public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer @Override protected int fetchSize() { // using a smaller fetchSize for nested documents' tests to uncover bugs - // similar with https://github.com/elastic/elasticsearch/issues/35176 quicker + // similar to https://github.com/elastic/elasticsearch/issues/35176 quicker return fileName.startsWith("nested") && randomBoolean() ? randomIntBetween(1,5) : super.fetchSize(); } } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java index 5e3b034d75708..c5ae7f63ad06d 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/ErrorsTestCase.java @@ -21,4 +21,5 @@ public interface ErrorsTestCase { void testSelectGroupByScore() throws Exception; void testSelectScoreSubField() throws Exception; void testSelectScoreInScalar() throws Exception; + void testHardLimitForSortOnAggregate() throws Exception; } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java index e7a73cd12d524..cf221bbc14012 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/CliIntegrationTestCase.java @@ -56,8 +56,8 @@ protected SecurityConfig securityConfig() { return null; } - protected void index(String index, CheckedConsumer body) throws IOException { - Request request = new Request("PUT", "/" + index + "/_doc/1"); + protected void index(String index, int docId, CheckedConsumer body) throws IOException { + Request request = new Request("PUT", "/" + index + "/_doc/" + docId); request.addParameter("refresh", "true"); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); body.accept(builder); @@ -66,6 +66,10 @@ protected void index(String index, CheckedConsumer client().performRequest(request); } + protected void index(String index, CheckedConsumer body) throws IOException { + index(index, 1, body); + } + public String command(String command) throws IOException { return cli.command(command); } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java index ca251a31844c0..a3ad325d0acec 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ErrorsTestCase.java @@ -97,8 +97,15 @@ public void testSelectScoreInScalar() throws Exception { assertEquals("line 1:12: [SCORE()] cannot be an argument to a function" + END, readLine()); } + @Override + public void testHardLimitForSortOnAggregate() throws Exception { + index("test", body -> body.field("a", 1).field("b", 2)); + String commandResult = command("SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 10000"); + assertEquals(START + "Bad request [[3;33;22mThe maximum LIMIT for aggregate sorting is [512], received [10000]" + END, + commandResult); + } + public static void assertFoundOneProblem(String commandResult) { assertEquals(START + "Bad request [[3;33;22mFound 1 problem(s)", commandResult); } - } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java index 84f74bcbac137..02de2dff4f7d6 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/FetchSizeTestCase.java @@ -49,4 +49,32 @@ public void testInvalidFetchSize() throws IOException { assertEquals(ErrorsTestCase.START + "Invalid fetch size [[3;33;22m" + Long.MAX_VALUE + ErrorsTestCase.END, command("fetch size = " + Long.MAX_VALUE)); } + + // Test for issue: https://github.com/elastic/elasticsearch/issues/42851 + // Even though fetch size and limit are smaller than the noRows, all buckets + // should be processed to achieve the global ordering of the aggregate function. + public void testOrderingOnAggregate() throws IOException { + Request request = new Request("PUT", "/test/_bulk"); + request.addParameter("refresh", "true"); + StringBuilder bulk = new StringBuilder(); + for (int i = 1; i <= 100; i++) { + bulk.append("{\"index\":{}}\n"); + bulk.append("{\"a\":").append(i).append(", \"b\" : ").append(i).append("}\n"); + } + request.setJsonEntity(bulk.toString()); + client().performRequest(request); + + assertEquals("[?1l>[?1000l[?2004lfetch size set to [90m4[0m", command("fetch size = 4")); + assertEquals("[?1l>[?1000l[?2004lfetch separator set to \"[90m -- fetch sep -- [0m\"", + command("fetch separator = \" -- fetch sep -- \"")); + assertThat(command("SELECT max(b) FROM test GROUP BY a ORDER BY max(b) DESC LIMIT 20"), containsString("max(b)")); + assertThat(readLine(), containsString("----------")); + for (int i = 100; i > 80; i--) { + if (i < 100 && i % 4 == 0) { + assertThat(readLine(), containsString(" -- fetch sep -- ")); + } + assertThat(readLine(), containsString(Integer.toString(i))); + } + assertEquals("", readLine()); + } } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java deleted file mode 100644 index d5a633e5ea388..0000000000000 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DebugCsvSpec.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.sql.qa.jdbc; - -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.logging.log4j.Logger; -import org.elasticsearch.test.junit.annotations.TestLogging; -import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.csvConnection; -import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.executeCsvQuery; -import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser; - -@TestLogging("org.elasticsearch.xpack.sql:TRACE") -public abstract class DebugCsvSpec extends SpecBaseIntegrationTestCase { - private final CsvTestCase testCase; - - @ParametersFactory(shuffle = false, argumentFormatting = SqlSpecTestCase.PARAM_FORMATTING) - public static List readScriptSpec() throws Exception { - Parser parser = specParser(); - return readScriptSpec("/debug.csv-spec", parser); - } - - public DebugCsvSpec(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) { - super(fileName, groupName, testName, lineNumber); - this.testCase = testCase; - } - - @Override - protected void assertResults(ResultSet expected, ResultSet elastic) throws SQLException { - Logger log = logEsResultSet() ? logger : null; - - // - // uncomment this to printout the result set and create new CSV tests - // - JdbcTestUtils.logResultSetMetadata(elastic, log); - JdbcTestUtils.logResultSetData(elastic, log); - //JdbcAssert.assertResultSets(expected, elastic, log); - } - - @Override - protected boolean logEsResultSet() { - return true; - } - - @Override - protected final void doTest() throws Throwable { - try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) { - // pass the testName as table for debugging purposes (in case the underlying reader is missing) - ResultSet expected = executeCsvQuery(csv, testName); - ResultSet elasticResults = executeJdbcQuery(es, testCase.query); - assertResults(expected, elasticResults); - } - } -} \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java index be3ba3d096ae2..6f12963634fdb 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ErrorsTestCase.java @@ -116,4 +116,14 @@ public void testSelectScoreInScalar() throws Exception { assertThat(e.getMessage(), startsWith("Found 1 problem(s)\nline 1:12: [SCORE()] cannot be an argument to a function")); } } + + @Override + public void testHardLimitForSortOnAggregate() throws Exception { + index("test", body -> body.field("a", 1).field("b", 2)); + try (Connection c = esJdbc()) { + SQLException e = expectThrows(SQLException.class, () -> + c.prepareStatement("SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 10000").executeQuery()); + assertEquals("The maximum LIMIT for aggregate sorting is [512], received [10000]", e.getMessage()); + } + } } diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java index ef01dc1fca11e..cfbec77a3e636 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SqlSpecTestCase.java @@ -65,6 +65,13 @@ public SqlSpecTestCase(String fileName, String groupName, String testName, Integ this.query = query; } + @Override + protected int fetchSize() { + // using a smaller fetchSize for nested documents' tests to uncover bugs + // similar to https://github.com/elastic/elasticsearch/issues/42581 + return randomIntBetween(1, 20); + } + @Override protected final void doTest() throws Throwable { // we skip the tests in case of these locales because ES-SQL is Locale-insensitive for now diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index c88f31bb2fd71..5a16261bfbbd7 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.qa.rest; import com.fasterxml.jackson.core.io.JsonStringEncoder; - import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; @@ -314,7 +313,14 @@ public void testSelectScoreInScalar() throws Exception { expectBadRequest(() -> runSql(randomMode(), "SELECT SIN(SCORE()) FROM test"), containsString("line 1:12: [SCORE()] cannot be an argument to a function")); } - + + @Override + public void testHardLimitForSortOnAggregate() throws Exception { + index("{\"a\": 1, \"b\": 2}"); + expectBadRequest(() -> runSql(randomMode(), "SELECT max(a) max FROM test GROUP BY b ORDER BY max LIMIT 10000"), + containsString("The maximum LIMIT for aggregate sorting is [512], received [10000]")); + } + public void testUseColumnarForUnsupportedFormats() throws Exception { String format = randomFrom("txt", "csv", "tsv"); index("{\"foo\":1}"); diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec index 79d58c48e4469..9a193d76b3166 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg-ordering.sql-spec @@ -99,13 +99,13 @@ aggNotSpecifiedWithHavingOnLargeGroupBy SELECT MAX(salary) AS max FROM test_emp GROUP BY emp_no HAVING AVG(salary) > 1000 ORDER BY MIN(salary); aggWithTieBreakerDescAsc -SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no ASC; +SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no ASC LIMIT 50; aggWithTieBreakerDescDesc -SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no DESC; +SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MIN(languages) DESC NULLS FIRST, emp_no DESC LIMIT 50; aggWithTieBreakerAscDesc -SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MAX(languages) ASC NULLS FIRST, emp_no DESC; +SELECT emp_no, MIN(languages) AS min FROM test_emp GROUP BY emp_no ORDER BY MAX(languages) ASC NULLS FIRST, emp_no DESC LIMIT 50; aggWithMixOfOrdinals SELECT gender AS g, MAX(salary) AS m FROM test_emp GROUP BY gender ORDER BY 2 DESC LIMIT 3; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java index 34fab72ca1385..26c1a690a4000 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursor.java @@ -132,7 +132,7 @@ public void nextPage(Configuration cfg, Client client, NamedWriteableRegistry re SearchRequest search = Querier.prepareRequest(client, query, cfg.pageTimeout(), includeFrozen, indices); - client.search(search, new ActionListener() { + client.search(search, new ActionListener<>() { @Override public void onResponse(SearchResponse r) { try { @@ -143,9 +143,9 @@ public void onResponse(SearchResponse r) { return; } - updateCompositeAfterKey(r, query); - CompositeAggsRowSet rowSet = new CompositeAggsRowSet(extractors, mask, r, limit, serializeQuery(query), includeFrozen, - indices); + boolean hasAfterKey = updateCompositeAfterKey(r, query); + CompositeAggsRowSet rowSet = new CompositeAggsRowSet(extractors, mask, r, limit, + hasAfterKey ? serializeQuery(query) : null, includeFrozen, indices); listener.onResponse(rowSet); } catch (Exception ex) { listener.onFailure(ex); @@ -178,7 +178,7 @@ static CompositeAggregation getComposite(SearchResponse response) { throw new SqlIllegalArgumentException("Unrecognized root group found; {}", agg.getClass()); } - static void updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next) { + static boolean updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next) { CompositeAggregation composite = getComposite(r); if (composite == null) { @@ -187,22 +187,25 @@ static void updateCompositeAfterKey(SearchResponse r, SearchSourceBuilder next) Map afterKey = composite.afterKey(); // a null after-key means done - if (afterKey != null) { - AggregationBuilder aggBuilder = next.aggregations().getAggregatorFactories().iterator().next(); - // update after-key with the new value - if (aggBuilder instanceof CompositeAggregationBuilder) { - CompositeAggregationBuilder comp = (CompositeAggregationBuilder) aggBuilder; - comp.aggregateAfter(afterKey); - } else { - throw new SqlIllegalArgumentException("Invalid client request; expected a group-by but instead got {}", aggBuilder); - } + if (afterKey == null) { + return false; + } + + AggregationBuilder aggBuilder = next.aggregations().getAggregatorFactories().iterator().next(); + // update after-key with the new value + if (aggBuilder instanceof CompositeAggregationBuilder) { + CompositeAggregationBuilder comp = (CompositeAggregationBuilder) aggBuilder; + comp.aggregateAfter(afterKey); + return true; + } else { + throw new SqlIllegalArgumentException("Invalid client request; expected a group-by but instead got {}", aggBuilder); } } /** * Deserializes the search source from a byte array. */ - static SearchSourceBuilder deserializeQuery(NamedWriteableRegistry registry, byte[] source) throws IOException { + private static SearchSourceBuilder deserializeQuery(NamedWriteableRegistry registry, byte[] source) throws IOException { try (NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(source), registry)) { return new SearchSourceBuilder(in); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java index f93e00eac5ac9..88b93359d0f77 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggsRowSet.java @@ -28,8 +28,8 @@ class CompositeAggsRowSet extends ResultRowSet { private final int size; private int row = 0; - CompositeAggsRowSet(List exts, BitSet mask, SearchResponse response, int limit, byte[] next, - boolean includeFrozen, String... indices) { + CompositeAggsRowSet(List exts, BitSet mask, SearchResponse response, + int limit, byte[] next, boolean includeFrozen, String... indices) { super(exts, mask); CompositeAggregation composite = CompositeAggregationCursor.getComposite(response); @@ -40,18 +40,22 @@ class CompositeAggsRowSet extends ResultRowSet { } // page size - size = limit < 0 ? buckets.size() : Math.min(buckets.size(), limit); + size = limit == -1 ? buckets.size() : Math.min(buckets.size(), limit); if (next == null) { cursor = Cursor.EMPTY; } else { - // compute remaining limit - int remainingLimit = limit - size; + // Compute remaining limit + + // If the limit is -1 then we have a local sorting (sort on aggregate function) that requires all the buckets + // to be processed so we stop only when all data is exhausted. + int remainingLimit = (limit == -1) ? limit : ((limit - size) >= 0 ? (limit - size) : 0); + // if the computed limit is zero, or the size is zero it means either there's nothing left or the limit has been reached // note that a composite agg might be valid but return zero groups (since these can be filtered with HAVING/bucket selector) // however the Querier takes care of that and keeps making requests until either the query is invalid or at least one response - // is returned - if (next == null || size == 0 || remainingLimit == 0) { + // is returned. + if (size == 0 || remainingLimit == 0) { cursor = Cursor.EMPTY; } else { cursor = new CompositeAggregationCursor(next, exts, mask, remainingLimit, includeFrozen, indices); @@ -92,4 +96,4 @@ public int size() { public Cursor nextPageCursor() { return cursor; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index 17e5a79fa4a71..2a3c045206a39 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -117,7 +117,6 @@ public void query(List output, QueryContainer query, String index, Ac listener = sortingColumns.isEmpty() ? listener : new LocalAggregationSorterListener(listener, sortingColumns, query.limit()); ActionListener l = null; - if (query.isAggsOnly()) { if (query.aggs().useImplicitGroupBy()) { l = new ImplicitGroupActionListener(listener, client, cfg, output, query, search); @@ -134,7 +133,7 @@ public void query(List output, QueryContainer query, String index, Ac public static SearchRequest prepareRequest(Client client, SearchSourceBuilder source, TimeValue timeout, boolean includeFrozen, String... indices) { - SearchRequest search = client.prepareSearch(indices) + return client.prepareSearch(indices) // always track total hits accurately .setTrackTotalHits(true) .setAllowPartialSearchResults(false) @@ -143,7 +142,6 @@ public static SearchRequest prepareRequest(Client client, SearchSourceBuilder so .setIndicesOptions( includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS) .request(); - return search; } /** @@ -158,7 +156,7 @@ class LocalAggregationSorterListener implements ActionListener { private final ActionListener listener; // keep the top N entries. - private final PriorityQueue, Integer>> data; + private final AggSortingQueue data; private final AtomicInteger counter = new AtomicInteger(); private volatile Schema schema; @@ -174,53 +172,13 @@ class LocalAggregationSorterListener implements ActionListener { } else { noLimit = false; if (limit > MAXIMUM_SIZE) { - throw new PlanningException("The maximum LIMIT for aggregate sorting is [{}], received [{}]", limit, MAXIMUM_SIZE); + throw new PlanningException("The maximum LIMIT for aggregate sorting is [{}], received [{}]", MAXIMUM_SIZE, limit); } else { size = limit; } } - this.data = new PriorityQueue<>(size) { - - // compare row based on the received attribute sort - // if a sort item is not in the list, it is assumed the sorting happened in ES - // and the results are left as is (by using the row ordering), otherwise it is sorted based on the given criteria. - // - // Take for example ORDER BY a, x, b, y - // a, b - are sorted in ES - // x, y - need to be sorted client-side - // sorting on x kicks in, only if the values for a are equal. - - // thanks to @jpountz for the row ordering idea as a way to preserve ordering - @SuppressWarnings("unchecked") - @Override - protected boolean lessThan(Tuple, Integer> l, Tuple, Integer> r) { - for (Tuple tuple : sortingColumns) { - int i = tuple.v1().intValue(); - Comparator comparator = tuple.v2(); - - Object vl = l.v1().get(i); - Object vr = r.v1().get(i); - if (comparator != null) { - int result = comparator.compare(vl, vr); - // if things are equals, move to the next comparator - if (result != 0) { - return result < 0; - } - } - // no comparator means the existing order needs to be preserved - else { - // check the values - if they are equal move to the next comparator - // otherwise return the row order - if (Objects.equals(vl, vr) == false) { - return l.v2().compareTo(r.v2()) < 0; - } - } - } - // everything is equal, fall-back to the row order - return l.v2().compareTo(r.v2()) < 0; - } - }; + this.data = new AggSortingQueue(size, sortingColumns); } @Override @@ -231,9 +189,8 @@ public void onResponse(SchemaRowSet schemaRowSet) { private void doResponse(RowSet rowSet) { // 1. consume all pages received - if (consumeRowSet(rowSet) == false) { - return; - } + consumeRowSet(rowSet); + Cursor cursor = rowSet.nextPageCursor(); // 1a. trigger a next call if there's still data if (cursor != Cursor.EMPTY) { @@ -248,31 +205,21 @@ private void doResponse(RowSet rowSet) { sendResponse(); } - private boolean consumeRowSet(RowSet rowSet) { - // use a synchronized block for visibility purposes (there's no concurrency) + private void consumeRowSet(RowSet rowSet) { ResultRowSet rrs = (ResultRowSet) rowSet; - synchronized (data) { - for (boolean hasRows = rrs.hasCurrentRow(); hasRows; hasRows = rrs.advanceRow()) { - List row = new ArrayList<>(rrs.columnCount()); - rrs.forEachResultColumn(row::add); - // if the queue overflows and no limit was specified, bail out - if (data.insertWithOverflow(new Tuple<>(row, counter.getAndIncrement())) != null && noLimit) { - onFailure(new SqlIllegalArgumentException( - "The default limit [{}] for aggregate sorting has been reached; please specify a LIMIT")); - return false; - } + for (boolean hasRows = rrs.hasCurrentRow(); hasRows; hasRows = rrs.advanceRow()) { + List row = new ArrayList<>(rrs.columnCount()); + rrs.forEachResultColumn(row::add); + // if the queue overflows and no limit was specified, throw an error + if (data.insertWithOverflow(new Tuple<>(row, counter.getAndIncrement())) != null && noLimit) { + onFailure(new SqlIllegalArgumentException( + "The default limit [{}] for aggregate sorting has been reached; please specify a LIMIT", MAXIMUM_SIZE)); } } - return true; } private void sendResponse() { - List> list = new ArrayList<>(data.size()); - Tuple, Integer> pop = null; - while ((pop = data.pop()) != null) { - list.add(pop.v1()); - } - listener.onResponse(new PagingListRowSet(schema, list, schema.size(), cfg.pageSize())); + listener.onResponse(new PagingListRowSet(schema, data.asList(), schema.size(), cfg.pageSize())); } @Override @@ -373,7 +320,7 @@ static class CompositeActionListener extends BaseAggActionListener { @Override protected void handleResponse(SearchResponse response, ActionListener listener) { // there are some results - if (response.getAggregations().asList().size() > 0) { + if (response.getAggregations().asList().isEmpty() == false) { // retry if (CompositeAggregationCursor.shouldRetryDueToEmptyPage(response)) { @@ -383,7 +330,7 @@ protected void handleResponse(SearchResponse response, ActionListener, Integer>> { + + private List> sortingColumns; + + AggSortingQueue(int maxSize, List> sortingColumns) { + super(maxSize); + this.sortingColumns = sortingColumns; + } + + // compare row based on the received attribute sort + // if a sort item is not in the list, it is assumed the sorting happened in ES + // and the results are left as is (by using the row ordering), otherwise it is sorted based on the given criteria. + // + // Take for example ORDER BY a, x, b, y + // a, b - are sorted in ES + // x, y - need to be sorted client-side + // sorting on x kicks in, only if the values for a are equal. + + // thanks to @jpountz for the row ordering idea as a way to preserve ordering + @SuppressWarnings("unchecked") + @Override + protected boolean lessThan(Tuple, Integer> l, Tuple, Integer> r) { + for (Tuple tuple : sortingColumns) { + int i = tuple.v1().intValue(); + Comparator comparator = tuple.v2(); + + Object vl = l.v1().get(i); + Object vr = r.v1().get(i); + if (comparator != null) { + int result = comparator.compare(vl, vr); + // if things are equals, move to the next comparator + if (result != 0) { + return result > 0; + } + } + // no comparator means the existing order needs to be preserved + else { + // check the values - if they are equal move to the next comparator + // otherwise return the row order + if (Objects.equals(vl, vr) == false) { + return l.v2().compareTo(r.v2()) > 0; + } + } + } + // everything is equal, fall-back to the row order + return l.v2().compareTo(r.v2()) > 0; + } + + List> asList() { + List> list = new ArrayList<>(super.size()); + Tuple, Integer> pop; + while ((pop = pop()) != null) { + list.add(0, pop.v1()); + } + return list; + } + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java index 64177aeab6872..3ec4ff6b11427 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SchemaCompositeAggsRowSet.java @@ -34,4 +34,4 @@ class SchemaCompositeAggsRowSet extends CompositeAggsRowSet implements SchemaRow public Schema schema() { return schema; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 8d9e59617aa4d..4e343c1e54f5a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -80,9 +80,13 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB if (source.size() == -1) { source.size(sz); } - // limit the composite aggs only for non-local sorting - if (aggBuilder instanceof CompositeAggregationBuilder && container.sortingColumns().isEmpty()) { - ((CompositeAggregationBuilder) aggBuilder).size(sz); + if (aggBuilder instanceof CompositeAggregationBuilder) { + // limit the composite aggs only for non-local sorting + if (container.sortingColumns().isEmpty()) { + ((CompositeAggregationBuilder) aggBuilder).size(sz); + } else { + ((CompositeAggregationBuilder) aggBuilder).size(size); + } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java index 811a8ff4256f2..0baf470e4b364 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/CompositeAggregationCursorTests.java @@ -46,7 +46,8 @@ protected CompositeAggregationCursor mutateInstance(CompositeAggregationCursor i return new CompositeAggregationCursor(instance.next(), instance.extractors(), randomValueOtherThan(instance.mask(), () -> randomBitSet(instance.extractors().size())), randomValueOtherThan(instance.limit(), () -> randomIntBetween(1, 512)), - randomBoolean(), instance.indices()); + !instance.includeFrozen(), + instance.indices()); } @Override @@ -81,4 +82,4 @@ static BitSet randomBitSet(int size) { } return mask; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java new file mode 100644 index 0000000000000..a6caad899dd89 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/QuerierTests.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.execution.search; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.execution.search.Querier.AggSortingQueue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class QuerierTests extends ESTestCase { + + @SuppressWarnings("rawtypes") + public void testAggSortingAscending() { + Tuple tuple = new Tuple<>(0, Comparator.naturalOrder()); + Querier.AggSortingQueue queue = new AggSortingQueue(10, Collections.singletonList(tuple)); + for (int i = 50; i >= 0; i--) { + queue.insertWithOverflow(new Tuple<>(Collections.singletonList(i), i)); + } + List> results = queue.asList(); + + assertEquals(10, results.size()); + for (int i = 0; i < 10; i ++) { + assertEquals(i, results.get(i).get(0)); + } + } + + @SuppressWarnings("rawtypes") + public void testAggSortingDescending() { + Tuple tuple = new Tuple<>(0, Comparator.reverseOrder()); + Querier.AggSortingQueue queue = new AggSortingQueue(10, Collections.singletonList(tuple)); + for (int i = 0; i <= 50; i++) { + queue.insertWithOverflow(new Tuple<>(Collections.singletonList(i), i)); + } + List> results = queue.asList(); + + assertEquals(10, results.size()); + for (int i = 0; i < 10; i ++) { + assertEquals(50 - i, results.get(i).get(0)); + } + } + + @SuppressWarnings("rawtypes") + public void testAggSorting_TwoFields() { + List> tuples = new ArrayList<>(2); + tuples.add(new Tuple<>(0, Comparator.reverseOrder())); + tuples.add(new Tuple<>(1, Comparator.naturalOrder())); + Querier.AggSortingQueue queue = new AggSortingQueue(10, tuples); + + for (int i = 1; i <= 100; i++) { + queue.insertWithOverflow(new Tuple<>(Arrays.asList(i % 50 + 1, i), i)); + } + List> results = queue.asList(); + + assertEquals(10, results.size()); + for (int i = 0; i < 10; i++) { + assertEquals(50 - (i / 2), results.get(i).get(0)); + assertEquals(49 - (i / 2) + ((i % 2) * 50), results.get(i).get(1)); + } + } + + @SuppressWarnings("rawtypes") + public void testAggSorting_Randomized() { + // Initialize comparators for fields (columns) + int noColumns = randomIntBetween(3, 10); + List> tuples = new ArrayList<>(noColumns); + boolean[] ordering = new boolean[noColumns]; + for (int j = 0; j < noColumns; j++) { + boolean order = randomBoolean(); + ordering[j] = order; + tuples.add(new Tuple<>(j, order ? Comparator.naturalOrder() : Comparator.reverseOrder())); + } + + // Insert random no of documents (rows) with random 0/1 values for each field + int noDocs = randomIntBetween(10, 50); + int queueSize = randomIntBetween(4, noDocs / 2); + List> expected = new ArrayList<>(noDocs); + Querier.AggSortingQueue queue = new AggSortingQueue(queueSize, tuples); + for (int i = 0; i < noDocs; i++) { + List values = new ArrayList<>(noColumns); + for (int j = 0; j < noColumns; j++) { + values.add(randomBoolean() ? 1 : 0); + } + queue.insertWithOverflow(new Tuple<>(values, i)); + expected.add(values); + } + + List> results = queue.asList(); + assertEquals(queueSize, results.size()); + expected.sort((o1, o2) -> { + for (int j = 0; j < noColumns; j++) { + if (o1.get(j) < o2.get(j)) { + return ordering[j] ? -1 : 1; + } else if (o1.get(j) > o2.get(j)) { + return ordering[j] ? 1 : -1; + } + } + return 0; + }); + assertEquals(expected.subList(0, queueSize), results); + } +} From 816f12f800215b1c49e0a6e6f5f621e25963607a Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Jun 2019 15:44:40 -0400 Subject: [PATCH 11/11] Fix JDK-version dependent exception message parsing This commit fixes some JDK-version dependent exception message checking in the discovery node role tests. --- .../elasticsearch/cluster/node/DiscoveryNodeRoleTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java index 9ef11bf6a8c48..f5fdd2d66fac9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeRoleTests.java @@ -49,7 +49,7 @@ protected Setting roleSetting() { } }))); - assertThat(e, hasToString(containsString("Duplicate key foo "))); + assertThat(e, hasToString(containsString("Duplicate key"))); } public void testDiscoveryNodeSetPossibleRolesRejectsDuplicateRoleNameAbbreviations() { @@ -72,7 +72,7 @@ protected Setting roleSetting() { } }))); - assertThat(e, hasToString(containsString("Duplicate key f "))); + assertThat(e, hasToString(containsString("Duplicate key"))); } }