diff --git a/commons/build.gradle b/commons/build.gradle index f3eca858..85b9c5c6 100644 --- a/commons/build.gradle +++ b/commons/build.gradle @@ -1 +1,5 @@ -apply plugin: 'java' \ No newline at end of file +apply plugin: 'java' + +dependencies { + compile "com.beust:jcommander:1.48" +} \ No newline at end of file diff --git a/commons/src/main/java/org/apache/mesos/elasticsearch/common/cli/validators/CLIValidators.java b/commons/src/main/java/org/apache/mesos/elasticsearch/common/cli/validators/CLIValidators.java new file mode 100644 index 00000000..e35b3432 --- /dev/null +++ b/commons/src/main/java/org/apache/mesos/elasticsearch/common/cli/validators/CLIValidators.java @@ -0,0 +1,68 @@ +package org.apache.mesos.elasticsearch.common.cli.validators; + +import com.beust.jcommander.IParameterValidator; +import com.beust.jcommander.IValueValidator; +import com.beust.jcommander.ParameterException; + +/** + * Holds CLI validators + */ +public class CLIValidators { + + /** + * Abstract class to validate a number. + * @param A numeric type + */ + public abstract static class PositiveValue implements IValueValidator { + @Override + public void validate(String name, T value) throws ParameterException { + if (notValid(value)) { + throw new ParameterException("Parameter " + name + " should be greater than zero (found " + value + ")"); + } + } + + public abstract Boolean notValid(T value); + } + + /** + * Validates a positive number. For type Long + */ + public static class PositiveLong extends PositiveValue { + @Override + public Boolean notValid(Long value) { + return value <= 0; + } + } + + /** + * Validates a positive number. For type Double + */ + public static class PositiveDouble extends PositiveValue { + @Override + public Boolean notValid(Double value) { + return value <= 0; + } + } + + /** + * Validates a positive number. For type Integer + */ + public static class PositiveInteger extends PositiveValue { + @Override + public Boolean notValid(Integer value) { + return value <= 0; + } + } + + /** + * Ensures that the string is not empty. Will strip spaces. + */ + public static class NotEmptyString implements IParameterValidator { + @Override + public void validate(String name, String value) throws ParameterException { + if (value.replace(" ", "").isEmpty()) { + throw new ParameterException("Parameter " + name + " cannot be empty"); + } + } + } +} diff --git a/commons/src/main/java/org/apache/mesos/elasticsearch/common/zookeeper/ZookeeperCLIParameter.java b/commons/src/main/java/org/apache/mesos/elasticsearch/common/zookeeper/ZookeeperCLIParameter.java new file mode 100644 index 00000000..8a9697a0 --- /dev/null +++ b/commons/src/main/java/org/apache/mesos/elasticsearch/common/zookeeper/ZookeeperCLIParameter.java @@ -0,0 +1,23 @@ +package org.apache.mesos.elasticsearch.common.zookeeper; + +import com.beust.jcommander.Parameter; +import org.apache.mesos.elasticsearch.common.cli.validators.CLIValidators; + +/** + * Class to reuse ZooKeeper CLI Parameters + */ +public class ZookeeperCLIParameter { + public static final String ZOOKEEPER_URL = "--zookeeperUrl"; + @Parameter(names = {ZOOKEEPER_URL}, required = true, description = "Zookeeper urls in the format zk://IP:PORT,IP:PORT,...)", validateWith = CLIValidators.NotEmptyString.class) + private String zookeeperUrl = "zk://mesos.master:2181"; + public String getZookeeperUrl() { + return zookeeperUrl; + } + + public static final String ZOOKEEPER_TIMEOUT = "--zookeeperTimeout"; + @Parameter(names = {ZOOKEEPER_TIMEOUT}, description = "The timeout for connecting to zookeeper (ms).", validateValueWith = CLIValidators.PositiveLong.class) + private long zookeeperTimeout = 20000L; + public long getZookeeperTimeout() { + return zookeeperTimeout; + } +} diff --git a/executor/build.gradle b/executor/build.gradle index e17b9853..977094c8 100644 --- a/executor/build.gradle +++ b/executor/build.gradle @@ -29,6 +29,7 @@ dependencies { compile "log4j:log4j:1.2.16" compile "org.apache.zookeeper:zookeeper:3.4.6" compile "com.github.containersolutions:elasticsearch-zookeeper:v${elasticsearchVersion}" + compile "com.beust:jcommander:1.48" } shadowJar { diff --git a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/Configuration.java b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/Configuration.java new file mode 100644 index 00000000..1c098afc --- /dev/null +++ b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/Configuration.java @@ -0,0 +1,58 @@ +package org.apache.mesos.elasticsearch.executor; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import org.apache.log4j.Logger; +import org.apache.mesos.elasticsearch.common.cli.validators.CLIValidators; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; +import org.apache.mesos.elasticsearch.common.zookeeper.formatter.ElasticsearchZKFormatter; +import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser; + +import java.net.URISyntaxException; + +/** + * Executor configuration + */ +public class Configuration { + private static final Logger LOGGER = Logger.getLogger(Configuration.class); + public static final String ELASTICSEARCH_YML = "elasticsearch.yml"; + + // **** ZOOKEEPER + private final ZookeeperCLIParameter zookeeperCLI = new ZookeeperCLIParameter(); + + public Configuration(String[] args) { + final JCommander jCommander = new JCommander(); + jCommander.addObject(zookeeperCLI); + jCommander.addObject(this); + try { + jCommander.parse(args); // Parse command line args into configuration class. + } catch (com.beust.jcommander.ParameterException ex) { + System.out.println(ex); + jCommander.setProgramName("(Options preceded by an asterisk are required)"); + jCommander.usage(); + throw ex; + } + } + + // ******* ELASTICSEARCH + public static final String ELASTICSEARCH_SETTINGS_LOCATION = "--elasticsearchSettingsLocation"; + @Parameter(names = {ELASTICSEARCH_SETTINGS_LOCATION}, description = "Local path to custom elasticsearch.yml settings file", validateWith = CLIValidators.NotEmptyString.class) + private String elasticsearchSettingsLocation = getElasticsearchSettingsPath(); + public String getElasticsearchSettingsLocation() { + return elasticsearchSettingsLocation; + } + private String getElasticsearchSettingsPath() { + String path = ""; + try { + path = getClass().getClassLoader().getResource(ELASTICSEARCH_YML).toURI().toString(); + } catch (NullPointerException | URISyntaxException ex) { + LOGGER.error("Unable to read default settings file from resources", ex); + } + return path; + } + + public String getElasticsearchZKURL() { + ElasticsearchZKFormatter zkFormatter = new ElasticsearchZKFormatter(new ZKAddressParser()); + return zkFormatter.format(zookeeperCLI.getZookeeperUrl()); + } +} diff --git a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchSettings.java b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchSettings.java index 7f4a430a..c8adc11c 100644 --- a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchSettings.java +++ b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchSettings.java @@ -6,15 +6,8 @@ * Builds the ES settings from the provided settings. */ public class ElasticsearchSettings { - // Todo: Make ES settings, settings. public ImmutableSettings.Builder defaultSettings() { return ImmutableSettings.settingsBuilder() - .put("node.local", false) - .put("cluster.name", "mesos-elasticsearch") - .put("node.master", true) - .put("node.data", true) - .put("index.number_of_shards", 5) - .put("index.number_of_replicas", 1) .put("discovery.type", "com.sonian.elasticsearch.zookeeper.discovery.ZooKeeperDiscoveryModule") .put("sonian.elasticsearch.zookeeper.settings.enabled", true) .put("sonian.elasticsearch.zookeeper.discovery.state_publishing.enabled", true); diff --git a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutor.java b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutor.java index 6c0bc799..263fd8d9 100644 --- a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutor.java +++ b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutor.java @@ -5,14 +5,19 @@ import org.apache.mesos.Executor; import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; +import org.apache.mesos.elasticsearch.executor.Configuration; import org.apache.mesos.elasticsearch.executor.elasticsearch.Launcher; import org.apache.mesos.elasticsearch.executor.model.PortsModel; import org.apache.mesos.elasticsearch.executor.model.RunTimeSettings; import org.apache.mesos.elasticsearch.executor.model.ZooKeeperModel; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.node.Node; +import java.net.MalformedURLException; +import java.net.URL; import java.security.InvalidParameterException; import java.util.Arrays; +import java.util.List; /** * Executor for Elasticsearch. @@ -21,6 +26,7 @@ public class ElasticsearchExecutor implements Executor { private final Launcher launcher; public static final Logger LOGGER = Logger.getLogger(ElasticsearchExecutor.class.getCanonicalName()); private final TaskStatus taskStatus; + private Configuration configuration; public ElasticsearchExecutor(Launcher launcher, TaskStatus taskStatus) { this.launcher = launcher; @@ -54,12 +60,24 @@ public void launchTask(final ExecutorDriver driver, final Protos.TaskInfo task) driver.sendStatusUpdate(taskStatus.starting()); try { + // Parse CommandInfo arguments + List list = task.getExecutor().getCommand().getArgumentsList(); + String[] args = list.toArray(new String[list.size()]); + LOGGER.debug("Using arguments: " + Arrays.toString(args)); + configuration = new Configuration(args); + + // Add settings provided in es Settings file + URL elasticsearchSettingsPath = java.net.URI.create(configuration.getElasticsearchSettingsLocation()).toURL(); + LOGGER.debug("Using elasticsearch settings file: " + elasticsearchSettingsPath); + ImmutableSettings.Builder esSettings = ImmutableSettings.builder().loadFromUrl(elasticsearchSettingsPath); + launcher.addRuntimeSettings(esSettings); + // Parse ports RunTimeSettings ports = new PortsModel(task); launcher.addRuntimeSettings(ports.getRuntimeSettings()); // Parse ZooKeeper address - RunTimeSettings zk = new ZooKeeperModel(task); + RunTimeSettings zk = new ZooKeeperModel(configuration.getElasticsearchZKURL()); launcher.addRuntimeSettings(zk.getRuntimeSettings()); // Launch Node @@ -76,7 +94,7 @@ public void run() { // Send status update, running driver.sendStatusUpdate(taskStatus.running()); - } catch (InvalidParameterException e) { + } catch (InvalidParameterException | MalformedURLException e) { driver.sendStatusUpdate(taskStatus.failed()); LOGGER.error(e); } diff --git a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModel.java b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModel.java index 6891d36f..b07bc87b 100644 --- a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModel.java +++ b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModel.java @@ -1,8 +1,5 @@ package org.apache.mesos.elasticsearch.executor.model; -import org.apache.mesos.Protos; -import org.apache.mesos.elasticsearch.executor.parser.ParseZooKeeper; -import org.apache.mesos.elasticsearch.executor.parser.TaskParser; import org.elasticsearch.common.settings.ImmutableSettings; /** @@ -10,11 +7,10 @@ */ public class ZooKeeperModel implements RunTimeSettings { public static final String ZOOKEEPER_ADDRESS_KEY = "sonian.elasticsearch.zookeeper.client.host"; - private final TaskParser parser = new ParseZooKeeper(); private final String address; - public ZooKeeperModel(Protos.TaskInfo taskInfo) { - address = parser.parse(taskInfo); + public ZooKeeperModel(String address) { + this.address = address; } private ImmutableSettings.Builder getAddress() { diff --git a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/parser/ParseZooKeeper.java b/executor/src/main/java/org/apache/mesos/elasticsearch/executor/parser/ParseZooKeeper.java deleted file mode 100644 index ade1fdaf..00000000 --- a/executor/src/main/java/org/apache/mesos/elasticsearch/executor/parser/ParseZooKeeper.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.apache.mesos.elasticsearch.executor.parser; - -import org.apache.log4j.Logger; -import org.apache.mesos.Protos; -import org.apache.mesos.elasticsearch.common.zookeeper.ZooKeeper; -import org.apache.mesos.elasticsearch.common.zookeeper.formatter.ElasticsearchZKFormatter; -import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser; - -import java.security.InvalidParameterException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -/** - * Parses ZooKeeper information - */ -public class ParseZooKeeper implements TaskParser { - - private static final Logger LOGGER = Logger.getLogger(ParseZooKeeper.class.getCanonicalName()); - - @Override - public String parse(Protos.TaskInfo taskInfo) throws InvalidParameterException { - int nargs = taskInfo.getExecutor().getCommand().getArgumentsCount(); - LOGGER.info("Using arguments [" + nargs + "]: " + taskInfo.getExecutor().getCommand().getArgumentsList().toString()); - Map argMap = new HashMap<>(1); - Iterator itr = taskInfo.getExecutor().getCommand().getArgumentsList().iterator(); - String lastKey = ""; - while (itr.hasNext()) { - String value = itr.next(); - // If it is a argument command - if (value.charAt(0) == '-') { - lastKey = value; - } else { // Else it must be an argument parameter - argMap.put(lastKey, value); - } - } - String address = argMap.get(ZooKeeper.ZOOKEEPER_ARG); - if (address == null) { - throw new InvalidParameterException("The task must pass a ZooKeeper address argument using " + ZooKeeper.ZOOKEEPER_ARG + "."); - } - - ElasticsearchZKFormatter zkFormatter = new ElasticsearchZKFormatter(new ZKAddressParser()); - - return zkFormatter.format(address); - } -} diff --git a/executor/src/main/resources/elasticsearch.yml b/executor/src/main/resources/elasticsearch.yml new file mode 100644 index 00000000..c73573ea --- /dev/null +++ b/executor/src/main/resources/elasticsearch.yml @@ -0,0 +1,334 @@ +##################### ElasticSearch Configuration Example ##################### + +# This file contains an overview of various configuration settings, +# targeted at operations staff. Application developers should +# consult the guide at . +# +# The installation procedure is covered at +# . +# +# ElasticSearch comes with reasonable defaults for most settings, +# so you can try it out without bothering with configuration. +# +# Most of the time, these defaults are just fine for running a production +# cluster. If you're fine-tuning your cluster, or wondering about the +# effect of certain configuration option, please _do ask_ on the +# mailing list or IRC channel [http://elasticsearch.org/community]. + +# Any element in the configuration can be replaced with environment variables +# by placing them in ${...} notation. For example: +# +# node.rack: ${RACK_ENV_VAR} + +# For information on supported formats and syntax for the config file, see +# + + +################################### Cluster ################################### + +# Cluster name identifies your cluster for auto-discovery. If you're running +# multiple clusters on the same network, make sure you're using unique names. +# +# cluster.name: SET_BY_FRAMEWORK + + +#################################### Node ##################################### + +# Node names are generated dynamically on startup, so you're relieved +# from configuring them manually. You can tie this node to a specific name: +# +# cluster.name: SET_BY_FRAMEWORK + +# Every node can be configured to allow or deny being eligible as the master, +# and to allow or deny to store the data. +# +# Allow this node to be eligible as a master node (enabled by default): +# +node.master: true +# +# Allow this node to store data (enabled by default): +# +node.data: true + +# You can exploit these settings to design advanced cluster topologies. +# +# 1. You want this node to never become a master node, only to hold data. +# This will be the "workhorse" of your cluster. +# +# node.master: false +# node.data: true +# +# 2. You want this node to only serve as a master: to not store any data and +# to have free resources. This will be the "coordinator" of your cluster. +# +# node.master: true +# node.data: false +# +# 3. You want this node to be neither master nor data node, but +# to act as a "search load balancer" (fetching data from nodes, +# aggregating results, etc.) +# +# node.master: false +# node.data: false + +# Use the Cluster Health API [http://localhost:9200/_cluster/health], the +# Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools +# such as and +# to inspect the cluster state. + +# A node can have generic attributes associated with it, which can later be used +# for customized shard allocation filtering, or allocation awareness. An attribute +# is a simple key value pair, similar to node.key: value, here is an example: +# +# node.rack: rack314 + +# By default, multiple nodes are allowed to start from the same installation location +# to disable it, set the following: +# node.max_local_storage_nodes: 1 + + +#################################### Index #################################### + +# You can set a number of options (such as shard/replica options, mapping +# or analyzer definitions, translog settings, ...) for indices globally, +# in this file. +# +# Note, that it makes more sense to configure index settings specifically for +# a certain index, either when creating it or by using the index templates API. +# +# See and +# +# for more information. + +# Set the number of shards (splits) of an index (5 by default): +# +index.number_of_shards: 5 + +# Set the number of replicas (additional copies) of an index (1 by default): +# +index.number_of_replicas: 2 + +# Note, that for development on a local machine, with small indices, it usually +# makes sense to "disable" the distributed features: +# +# index.number_of_shards: 1 +# index.number_of_replicas: 0 + +# These settings directly affect the performance of index and search operations +# in your cluster. Assuming you have enough machines to hold shards and +# replicas, the rule of thumb is: +# +# 1. Having more *shards* enhances the _indexing_ performance and allows to +# _distribute_ a big index across machines. +# 2. Having more *replicas* enhances the _search_ performance and improves the +# cluster _availability_. +# +# The "number_of_shards" is a one-time setting for an index. +# +# The "number_of_replicas" can be increased or decreased anytime, +# by using the Index Update Settings API. +# +# ElasticSearch takes care about load balancing, relocating, gathering the +# results from nodes, etc. Experiment with different settings to fine-tune +# your setup. + +# Use the Index Status API () to inspect +# the index status. + + +#################################### Paths #################################### + +# Path to directory containing configuration (this file and logging.yml): +# +# path.conf: /path/to/conf + +# Path to directory where to store index data allocated for this node. +# +# path.data: /path/to/data +# +# Can optionally include more than one location, causing data to be striped across +# the locations (a la RAID 0) on a file level, favouring locations with most free +# space on creation. For example: +# +# path.data: /path/to/data1,/path/to/data2 + +# Path to temporary files: +# +# path.work: /path/to/work + +# Path to log files: +# +# path.logs: /path/to/logs + +# Path to where plugins are installed: +# +# path.plugins: /path/to/plugins + + +#################################### Plugin ################################### + +# If a plugin listed here is not installed for current node, the node will not start. +# +# plugin.mandatory: mapper-attachments,lang-groovy + + +################################### Memory #################################### + +# ElasticSearch performs poorly when JVM starts swapping: you should ensure that +# it _never_ swaps. +# +# Set this property to true to lock the memory: +# +bootstrap.mlockall: true + +# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set +# to the same value, and that the machine has enough memory to allocate +# for ElasticSearch, leaving enough memory for the operating system itself. +# +# You should also make sure that the ElasticSearch process is allowed to lock +# the memory, eg. by using `ulimit -l unlimited`. + + +############################## Network And HTTP ############################### + +# ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). + +# Set the bind address specifically (IPv4 or IPv6): +# +# network.bind_host: 192.168.0.1 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. +# +# network.publish_host: 192.168.0.1 + +# Set both 'bind_host' and 'publish_host': +# +# network.host: 192.168.0.1 + +# Set a custom port for the node to node communication (9300 by default): +# +# transport.tcp.port: SET_BY_FRAMEWORK + +# Enable compression for all communication between nodes (disabled by default): +# +# transport.tcp.compress: true + +# Set a custom port to listen for HTTP traffic: +# +# http.port: SET_BY_FRAMEWORK + +# Set a custom allowed content length: +# +# http.max_content_length: 100mb + +# Disable HTTP completely: +# +# http.enabled: false + + +################################### Gateway ################################### + +# The gateway allows for persisting the cluster state between full cluster +# restarts. Every change to the state (such as adding an index) will be stored +# in the gateway, and when the cluster starts up for the first time, +# it will read its state from the gateway. + +# There are several types of gateway implementations. For more information, see +# . + +# The default gateway type is the "local" gateway (recommended): +# +# gateway.type: local + +# Settings below control how and when to start the initial recovery process on +# a full cluster restart (to reuse as much local data as possible when using shared +# gateway). + +# Allow recovery process after N nodes in a cluster are up: +# +gateway.recover_after_nodes: 1 + +# Set the timeout to initiate the recovery process, once the N nodes +# from previous setting are up (accepts time value): +# +gateway.recover_after_time: 10m + +# Set how many nodes are expected in this cluster. Once these N nodes +# are up (and recover_after_nodes is met), begin recovery process immediately +# (without waiting for recover_after_time to expire): +# +gateway.expected_nodes: 2 + +# Require explicit index creation +# action.auto_create_index: false + +# Protect against accidental close/delete operations +# on all indices. You can still close/delete individual +# indices +action.disable_close_all_indices: true +action.disable_delete_all_indices: true +action.disable_shutdown: true + +############################# Recovery Throttling ############################# + +# These settings allow to control the process of shards allocation between +# nodes during initial recovery, replica allocation, rebalancing, +# or when adding and removing nodes. + +# Set the number of concurrent recoveries happening on a node: +# +# 1. During the initial recovery +# +# cluster.routing.allocation.node_initial_primaries_recoveries: 4 +# +# 2. During adding/removing nodes, rebalancing, etc +# +# cluster.routing.allocation.node_concurrent_recoveries: 2 + +# Set to throttle throughput when recovering (eg. 100mb, by default 20mb): +# +indices.recovery.max_bytes_per_sec: 100mb + +# Set to limit the number of open concurrent streams when +# recovering a shard from a peer: +# +# indices.recovery.concurrent_streams: 5 + + +################################## Discovery ################################## + +# SET BY FRAMEWORK - USES ZOOKEEPER + + +################################## Slow Log ################################## + +# Shard level query and fetch threshold logging. + +#index.search.slowlog.threshold.query.warn: 10s +#index.search.slowlog.threshold.query.info: 5s +#index.search.slowlog.threshold.query.debug: 2s +#index.search.slowlog.threshold.query.trace: 500ms + +#index.search.slowlog.threshold.fetch.warn: 1s +#index.search.slowlog.threshold.fetch.info: 800ms +#index.search.slowlog.threshold.fetch.debug: 500ms +#index.search.slowlog.threshold.fetch.trace: 200ms + +#index.indexing.slowlog.threshold.index.warn: 10s +#index.indexing.slowlog.threshold.index.info: 5s +#index.indexing.slowlog.threshold.index.debug: 2s +#index.indexing.slowlog.threshold.index.trace: 500ms + +################################## GC Logging ################################ + +#monitor.jvm.gc.ParNew.warn: 1000ms +#monitor.jvm.gc.ParNew.info: 700ms +#monitor.jvm.gc.ParNew.debug: 400ms + +#monitor.jvm.gc.ConcurrentMarkSweep.warn: 10s +#monitor.jvm.gc.ConcurrentMarkSweep.info: 5s +#monitor.jvm.gc.ConcurrentMarkSweep.debug: 2s \ No newline at end of file diff --git a/executor/src/main/resources/log4j.xml b/executor/src/main/resources/log4j.xml index 5108c406..21119249 100644 --- a/executor/src/main/resources/log4j.xml +++ b/executor/src/main/resources/log4j.xml @@ -9,7 +9,12 @@ + + + + + diff --git a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchLauncherTest.java b/executor/src/test/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchLauncherTest.java index 37534321..291fa010 100644 --- a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchLauncherTest.java +++ b/executor/src/test/java/org/apache/mesos/elasticsearch/executor/elasticsearch/ElasticsearchLauncherTest.java @@ -6,6 +6,7 @@ import org.junit.runner.RunWith; import org.mockito.runners.MockitoJUnitRunner; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.*; @@ -64,6 +65,12 @@ public void shouldBeAbleToAddRunTimeSettings() { verify(settings, times(1)).put(runtimeSettings.build()); } + @Test + public void shouldBeAbleToLoadSettingsFromResources() { + ImmutableSettings.Builder esSettings = ImmutableSettings.builder().loadFromClasspath("elasticsearch.yml"); + assertNotNull(esSettings); + } + private ImmutableSettings.Builder getClientPort() { return ImmutableSettings.settingsBuilder().put(PortsModel.HTTP_PORT_KEY, "1234"); diff --git a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutorTest.java b/executor/src/test/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutorTest.java index 0e833fcd..200fac4a 100644 --- a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutorTest.java +++ b/executor/src/test/java/org/apache/mesos/elasticsearch/executor/mesos/ElasticsearchExecutorTest.java @@ -3,6 +3,7 @@ import org.apache.mesos.ExecutorDriver; import org.apache.mesos.Protos; import org.apache.mesos.elasticsearch.common.Discovery; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.executor.elasticsearch.Launcher; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.node.Node; @@ -102,7 +103,7 @@ private Protos.DiscoveryInfo.Builder getDefaultDiscoveryInfo() { private Protos.ExecutorInfo.Builder getDefaultExecutorInfo() { return Protos.ExecutorInfo.newBuilder() - .setCommand(Protos.CommandInfo.newBuilder().addArguments("-zk").addArguments("zk://master:2181/mesos")) + .setCommand(Protos.CommandInfo.newBuilder().addArguments(ZookeeperCLIParameter.ZOOKEEPER_URL).addArguments("zk://master:2181/mesos")) .setExecutorId(Protos.ExecutorID.newBuilder().setValue("0")); } } \ No newline at end of file diff --git a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModelTest.java b/executor/src/test/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModelTest.java deleted file mode 100644 index 064d3181..00000000 --- a/executor/src/test/java/org/apache/mesos/elasticsearch/executor/model/ZooKeeperModelTest.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.apache.mesos.elasticsearch.executor.model; - -import org.apache.mesos.Protos; -import org.apache.mesos.elasticsearch.common.zookeeper.exception.ZKAddressException; -import org.junit.Test; - -import java.security.InvalidParameterException; - -import static org.junit.Assert.assertEquals; - -/** - * Tests - */ -@SuppressWarnings({"PMD.AvoidUsingHardCodedIP"}) -public class ZooKeeperModelTest { - @Test(expected = NullPointerException.class) - public void shouldExceptionIfPassedNull() { - new ZooKeeperModel(null); - } - - @Test(expected = InvalidParameterException.class) - public void shouldExceptionIfNoZKInfo() { - Protos.TaskInfo taskInfo = Protos.TaskInfo.getDefaultInstance(); - new ZooKeeperModel(taskInfo); - } - - @Test(expected = InvalidParameterException.class) - public void shouldExceptionIfOnlyParameterCommand() { - Protos.ExecutorInfo.Builder executorInfo = getDefaultExecutorInfo(Protos.CommandInfo.newBuilder().addArguments("-zk")); - Protos.TaskInfo.Builder taskInfo = getDefaultTaskInfo(executorInfo); - new ZooKeeperModel(taskInfo.build()); - } - - private Protos.ExecutorInfo.Builder getDefaultExecutorInfo(Protos.CommandInfo.Builder commandInfo) { - return Protos.ExecutorInfo.newBuilder() - .setCommand(commandInfo) - .setExecutorId(Protos.ExecutorID.newBuilder().setValue("0")); - } - - private Protos.TaskInfo.Builder getDefaultTaskInfo(Protos.ExecutorInfo.Builder executorInfo) { - return Protos.TaskInfo.newBuilder() - .setName("") - .setTaskId(Protos.TaskID.newBuilder().setValue("0")) - .setSlaveId(Protos.SlaveID.newBuilder().setValue("0")) - .setExecutor(executorInfo); - } -} \ No newline at end of file diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java index 4c285c2d..24ea348c 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java @@ -1,8 +1,12 @@ package org.apache.mesos.elasticsearch.scheduler; -import com.beust.jcommander.*; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; import org.apache.log4j.Logger; import org.apache.mesos.Protos; +import org.apache.mesos.elasticsearch.common.cli.validators.CLIValidators; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.common.zookeeper.formatter.MesosStateZKFormatter; import org.apache.mesos.elasticsearch.common.zookeeper.formatter.MesosZKFormatter; import org.apache.mesos.elasticsearch.common.zookeeper.formatter.ZKFormatter; @@ -20,9 +24,13 @@ @SuppressWarnings("PMD.TooManyFields") public class Configuration { private static final Logger LOGGER = Logger.getLogger(Configuration.class); + // **** ZOOKEEPER + private final ZookeeperCLIParameter zookeeperCLI = new ZookeeperCLIParameter(); public Configuration(String[] args) { - final JCommander jCommander = new JCommander(this); + final JCommander jCommander = new JCommander(); + jCommander.addObject(zookeeperCLI); + jCommander.addObject(this); try { jCommander.parse(args); // Parse command line args into configuration class. } catch (com.beust.jcommander.ParameterException ex) { @@ -33,39 +41,24 @@ public Configuration(String[] args) { } } - // **** ZOOKEEPER - public static final String ZOOKEEPER_TIMEOUT = "--zookeeperTimeout"; - @Parameter(names = {ZOOKEEPER_TIMEOUT}, description = "The timeout for connecting to zookeeper (ms).", validateValueWith = PositiveLong.class) - private long zookeeperTimeout = 20000L; - public long getZookeeperTimeout() { - return zookeeperTimeout; - } - - public static final String ZOOKEEPER_URL = "--zookeeperUrl"; - @Parameter(names = {ZOOKEEPER_URL}, required = true, description = "Zookeeper urls in the format zk://IP:PORT,IP:PORT,...)", validateWith = NotEmptyString.class) - private String zookeeperUrl = "zk://mesos.master:2181"; - private String getZookeeperUrl() { - return zookeeperUrl; - } - // **** ELASTICSEARCH public static final String ELASTICSEARCH_CPU = "--elasticsearchCpu"; - @Parameter(names = {ELASTICSEARCH_CPU}, description = "The amount of CPU resource to allocate to the elasticsearch instance.", validateValueWith = PositiveDouble.class) + @Parameter(names = {ELASTICSEARCH_CPU}, description = "The amount of CPU resource to allocate to the elasticsearch instance.", validateValueWith = CLIValidators.PositiveDouble.class) private double cpus = 1.0; public double getCpus() { return cpus; } public static final String ELASTICSEARCH_RAM = "--elasticsearchRam"; - @Parameter(names = {ELASTICSEARCH_RAM}, description = "The amount of ram resource to allocate to the elasticsearch instance (MB).", validateValueWith = PositiveDouble.class) + @Parameter(names = {ELASTICSEARCH_RAM}, description = "The amount of ram resource to allocate to the elasticsearch instance (MB).", validateValueWith = CLIValidators.PositiveDouble.class) private double mem = 256; public double getMem() { return mem; } public static final String ELASTICSEARCH_DISK = "--elasticsearchDisk"; - @Parameter(names = {ELASTICSEARCH_DISK}, description = "The amount of Disk resource to allocate to the elasticsearch instance (MB).", validateValueWith = PositiveDouble.class) + @Parameter(names = {ELASTICSEARCH_DISK}, description = "The amount of Disk resource to allocate to the elasticsearch instance (MB).", validateValueWith = CLIValidators.PositiveDouble.class) private double disk = 1024; public double getDisk() { return disk; @@ -79,15 +72,22 @@ public int getElasticsearchNodes() { } public static final String ELASTICSEARCH_CLUSTER_NAME = "--elasticsearchClusterName"; - @Parameter(names = {ELASTICSEARCH_CLUSTER_NAME}, description = "Name of the elasticsearch cluster", validateWith = NotEmptyString.class) + @Parameter(names = {ELASTICSEARCH_CLUSTER_NAME}, description = "Name of the elasticsearch cluster", validateWith = CLIValidators.NotEmptyString.class) private String elasticsearchClusterName = "mesos-ha"; public String getElasticsearchClusterName() { return elasticsearchClusterName; } + public static final String ELASTICSEARCH_SETTINGS_LOCATION = "--elasticsearchSettingsLocation"; + @Parameter(names = {ELASTICSEARCH_SETTINGS_LOCATION}, description = "Local path to custom elasticsearch.yml settings file", validateWith = CLIValidators.NotEmptyString.class) + private String elasticsearchSettingsLocation = ""; + public String getElasticsearchSettingsLocation() { + return elasticsearchSettingsLocation; + } + // **** WEB UI public static final String WEB_UI_PORT = "--webUiPort"; - @Parameter(names = {WEB_UI_PORT}, description = "TCP port for web ui interface.", validateValueWith = PositiveInteger.class) + @Parameter(names = {WEB_UI_PORT}, description = "TCP port for web ui interface.", validateValueWith = CLIValidators.PositiveInteger.class) private int webUiPort = 31100; // Default is more likely to work on a default Mesos installation public int getWebUiPort() { return webUiPort; @@ -101,14 +101,14 @@ public String getVersion() { } public static final String FRAMEWORK_NAME = "--frameworkName"; - @Parameter(names = {FRAMEWORK_NAME}, description = "The name given to the framework.", validateWith = NotEmptyString.class) + @Parameter(names = {FRAMEWORK_NAME}, description = "The name given to the framework.", validateWith = CLIValidators.NotEmptyString.class) private String frameworkName = "elasticsearch"; public String getFrameworkName() { return frameworkName; } public static final String EXECUTOR_NAME = "--executorName"; - @Parameter(names = {EXECUTOR_NAME}, description = "The name given to the executor task.", validateWith = NotEmptyString.class) + @Parameter(names = {EXECUTOR_NAME}, description = "The name given to the executor task.", validateWith = CLIValidators.NotEmptyString.class) private String executorName = "elasticsearch-executor"; public String getTaskName() { return executorName; @@ -116,14 +116,14 @@ public String getTaskName() { // DCOS Certification requirement 01 public static final String FRAMEWORK_FAILOVER_TIMEOUT = "--frameworkFailoverTimeout"; - @Parameter(names = {FRAMEWORK_FAILOVER_TIMEOUT}, description = "The time before Mesos kills a scheduler and tasks if it has not recovered (ms).", validateValueWith = PositiveDouble.class) + @Parameter(names = {FRAMEWORK_FAILOVER_TIMEOUT}, description = "The time before Mesos kills a scheduler and tasks if it has not recovered (ms).", validateValueWith = CLIValidators.PositiveDouble.class) private double frameworkFailoverTimeout = 2592000; // Mesos will kill framework after 1 month if marathon does not restart. public double getFailoverTimeout() { return frameworkFailoverTimeout; } public static final String EXECUTOR_HEALTH_DELAY = "--executorHealthDelay"; - @Parameter(names = {EXECUTOR_HEALTH_DELAY}, description = "The delay between executor healthcheck requests (ms).", validateValueWith = PositiveLong.class) + @Parameter(names = {EXECUTOR_HEALTH_DELAY}, description = "The delay between executor healthcheck requests (ms).", validateValueWith = CLIValidators.PositiveLong.class) private static Long executorHealthDelay = 30000L; public Long getExecutorHealthDelay() { return executorHealthDelay; @@ -140,7 +140,7 @@ public Long getExecutorTimeout() { public static final String EXECUTOR_IMAGE = "--executorImage"; public static final String DEFAULT_EXECUTOR_IMAGE = "mesos/elasticsearch-executor"; - @Parameter(names = {EXECUTOR_IMAGE}, description = "The docker executor image to use.", validateWith = NotEmptyString.class) + @Parameter(names = {EXECUTOR_IMAGE}, description = "The docker executor image to use.", validateWith = CLIValidators.NotEmptyString.class) private String executorImage = DEFAULT_EXECUTOR_IMAGE; public String getEexecutorImage() { return executorImage; @@ -172,7 +172,7 @@ public SerializableState getState() { if (state == null) { org.apache.mesos.state.State zkState = new ZooKeeperState( getMesosStateZKURL(), - getZookeeperTimeout(), + zookeeperCLI.getZookeeperTimeout(), TimeUnit.MILLISECONDS, "/" + getFrameworkName() + "/" + getElasticsearchClusterName()); state = new SerializableZookeeperState(zkState); @@ -182,63 +182,30 @@ public SerializableState getState() { public String getMesosStateZKURL() { ZKFormatter mesosStateZKFormatter = new MesosStateZKFormatter(new ZKAddressParser()); - return mesosStateZKFormatter.format(getZookeeperUrl()); + return mesosStateZKFormatter.format(zookeeperCLI.getZookeeperUrl()); } public String getMesosZKURL() { ZKFormatter mesosZKFormatter = new MesosZKFormatter(new ZKAddressParser()); - return mesosZKFormatter.format(getZookeeperUrl()); + return mesosZKFormatter.format(zookeeperCLI.getZookeeperUrl()); } /** - * Abstract class to validate a number. - * @param A numeric type + * Ensures that the number is > than the EXECUTOR_HEALTH_DELAY */ - public abstract static class PositiveValue implements IValueValidator { + public static class GreaterThanHealthDelay extends CLIValidators.PositiveLong { @Override - public void validate(String name, T value) throws ParameterException { - if (notValid(value)) { - throw new ParameterException("Parameter " + name + " should be greater than zero (found " + value + ")"); + public void validate(String name, Long value) throws ParameterException { + if (notValid(value) || value <= Configuration.executorHealthDelay) { + throw new ParameterException("Parameter " + name + " should be greater than " + EXECUTOR_HEALTH_DELAY + " (found " + value + ")"); } } - - public abstract Boolean notValid(T value); - } - - /** - * Validates a positive number. For type Long - */ - public static class PositiveLong extends PositiveValue { - @Override - public Boolean notValid(Long value) { - return value <= 0; - } - } - - /** - * Validates a positive number. For type Double - */ - public static class PositiveDouble extends PositiveValue { - @Override - public Boolean notValid(Double value) { - return value <= 0; - } - } - - /** - * Validates a positive number. For type Integer - */ - public static class PositiveInteger extends PositiveValue { - @Override - public Boolean notValid(Integer value) { - return value <= 0; - } } /** * Adds a warning message if an even number is encountered */ - public static class OddNumberOfNodes extends PositiveInteger { + public static class OddNumberOfNodes extends CLIValidators.PositiveInteger { @Override public Boolean notValid(Integer value) { if (value % 2 == 0) { @@ -247,28 +214,4 @@ public Boolean notValid(Integer value) { return super.notValid(value); } } - - /** - * Ensures that the number is > than the EXECUTOR_HEALTH_DELAY - */ - public static class GreaterThanHealthDelay extends PositiveLong { - @Override - public void validate(String name, Long value) throws ParameterException { - if (notValid(value) || value <= Configuration.executorHealthDelay) { - throw new ParameterException("Parameter " + name + " should be greater than " + EXECUTOR_HEALTH_DELAY + " (found " + value + ")"); - } - } - } - - /** - * Ensures that the string is not empty. Will strip spaces. - */ - public static class NotEmptyString implements IParameterValidator { - @Override - public void validate(String name, String value) throws ParameterException { - if (value.replace(" ", "").isEmpty()) { - throw new ParameterException("Parameter " + name + " cannot be empty"); - } - } - } } diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java index c1cb30ff..858cee8c 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/ElasticsearchScheduler.java @@ -172,7 +172,6 @@ public void slaveLost(SchedulerDriver driver, Protos.SlaveID slaveId) { LOGGER.info("Slave lost: " + slaveId.getValue()); } - // Todo, we still don't perform reconciliation @Override public void executorLost(SchedulerDriver driver, Protos.ExecutorID executorId, Protos.SlaveID slaveId, int status) { // This is never called by Mesos, so we have to call it ourselves via a healthcheck diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactory.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactory.java index 14f2b978..01080da0 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactory.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactory.java @@ -3,6 +3,7 @@ import org.apache.log4j.Logger; import org.apache.mesos.Protos; import org.apache.mesos.elasticsearch.common.Discovery; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.scheduler.configuration.ExecutorEnvironmentalVariables; import java.text.SimpleDateFormat; @@ -71,9 +72,13 @@ private Protos.ExecutorInfo.Builder newExecutorInfo(Configuration configuration) private Protos.CommandInfo.Builder newCommandInfo(Configuration configuration) { ExecutorEnvironmentalVariables executorEnvironmentalVariables = new ExecutorEnvironmentalVariables(configuration); + List args = new ArrayList<>(asList(ZookeeperCLIParameter.ZOOKEEPER_URL, configuration.getMesosZKURL())); + if (!configuration.getElasticsearchSettingsLocation().isEmpty()) { + args.addAll(asList(Configuration.ELASTICSEARCH_SETTINGS_LOCATION, configuration.getElasticsearchSettingsLocation())); + } return Protos.CommandInfo.newBuilder() .setShell(false) - .addAllArguments(asList("-zk", configuration.getMesosZKURL())) + .addAllArguments(args) .setEnvironment(Protos.Environment.newBuilder().addAllVariables(executorEnvironmentalVariables.getList())) .setContainer(Protos.CommandInfo.ContainerInfo.newBuilder().setImage(configuration.getEexecutorImage()).build()); } diff --git a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/CLITest.java b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/CLITest.java index f7c545d7..7f813b4d 100644 --- a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/CLITest.java +++ b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/CLITest.java @@ -1,5 +1,6 @@ package org.apache.mesos.elasticsearch.scheduler; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -26,7 +27,7 @@ public void mainShouldBombOutOfMainIfInvalidParams() { @Test public void shouldPassIfOnlyRequiredParams() { - String[] args = {Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; new Configuration(args); } @@ -38,32 +39,32 @@ public void shouldCrashIfNoRequiredParams() { @Test public void randomParamTest() { - String[] args = {Configuration.ELASTICSEARCH_RAM, "512", Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {Configuration.ELASTICSEARCH_RAM, "512", ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; Configuration configuration = new Configuration(args); assertEquals(512.0, configuration.getMem(), 0.1); } @Test(expected = com.beust.jcommander.ParameterException.class) public void shouldRejectNumbersEqualTo0() { - String[] args = {Configuration.ZOOKEEPER_TIMEOUT, "0", Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {ZookeeperCLIParameter.ZOOKEEPER_TIMEOUT, "0", ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; new Configuration(args); } @Test(expected = com.beust.jcommander.ParameterException.class) public void shouldRejectNumbersLessThan0() { - String[] args = {Configuration.ZOOKEEPER_TIMEOUT, "-1", Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {ZookeeperCLIParameter.ZOOKEEPER_TIMEOUT, "-1", ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; new Configuration(args); } @Test(expected = com.beust.jcommander.ParameterException.class) public void shouldFailIfExecutorTimeoutLessThanHealthDelay() { - String[] args = {Configuration.EXECUTOR_HEALTH_DELAY, "1000", Configuration.EXECUTOR_TIMEOUT, "10", Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {Configuration.EXECUTOR_HEALTH_DELAY, "1000", Configuration.EXECUTOR_TIMEOUT, "10", ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; new Configuration(args); } @Test(expected = com.beust.jcommander.ParameterException.class) public void shouldFailIfParamIsEmpty() { - String[] args = {Configuration.ELASTICSEARCH_CLUSTER_NAME, " ", Configuration.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; + String[] args = {Configuration.ELASTICSEARCH_CLUSTER_NAME, " ", ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://dummyIPAddress:2181"}; new Configuration(args); } } \ No newline at end of file diff --git a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactoryTest.java b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactoryTest.java index cde02dc8..ff0b0276 100644 --- a/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactoryTest.java +++ b/scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactoryTest.java @@ -36,6 +36,7 @@ public void testCreateTaskInfo() { when(configuration.getTaskName()).thenReturn("esdemo"); when(configuration.getMesosZKURL()).thenReturn("zk://zookeeper:2181/mesos"); when(configuration.getEexecutorImage()).thenReturn(Configuration.DEFAULT_EXECUTOR_IMAGE); + when(configuration.getElasticsearchSettingsLocation()).thenReturn("/var/null"); Protos.Offer offer = Protos.Offer.newBuilder() .setId(Protos.OfferID.newBuilder().setValue(UUID.randomUUID().toString())) diff --git a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java index 491f82bd..00bb8524 100644 --- a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java +++ b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java @@ -2,6 +2,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.scheduler.Configuration; import org.apache.mesos.mini.container.AbstractContainer; @@ -37,7 +38,7 @@ protected CreateContainerCmd dockerCommand() { .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") .withExtraHosts(IntStream.rangeClosed(1, 3).mapToObj(value -> "slave" + value + ":" + mesosIp).toArray(String[]::new)) .withCmd( - Configuration.ZOOKEEPER_URL, "zk://" + mesosIp + ":2181/mesos", + ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://" + mesosIp + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256", Configuration.WEB_UI_PORT, "8080", diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/SchedulerMainSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/SchedulerMainSystemTest.java index 91497d99..321182bf 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/SchedulerMainSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/SchedulerMainSystemTest.java @@ -6,6 +6,7 @@ import com.github.dockerjava.api.model.Container; import com.jayway.awaitility.Awaitility; import org.apache.commons.io.IOUtils; +import org.apache.mesos.elasticsearch.common.zookeeper.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.scheduler.Configuration; import org.apache.mesos.mini.mesos.MesosClusterConfig; import org.junit.Test; @@ -31,7 +32,7 @@ public void ensureMainFailsIfNoHeap() throws Exception { final String schedulerImage = "mesos/elasticsearch-scheduler"; CreateContainerCmd createCommand = CONFIG.dockerClient .createContainerCmd(schedulerImage) - .withCmd(Configuration.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); + .withCmd(ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); CreateContainerResponse r = createCommand.exec(); String containerId = r.getId(); @@ -53,7 +54,7 @@ public void ensureMainFailsIfInvalidHeap() throws Exception { CreateContainerCmd createCommand = CONFIG.dockerClient .createContainerCmd(schedulerImage) .withEnv("JAVA_OPTS=-Xms128s1m -Xmx256f5m") - .withCmd(Configuration.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); + .withCmd(ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); CreateContainerResponse r = createCommand.exec(); String containerId = r.getId(); @@ -76,7 +77,7 @@ public void ensureMainWorksIfValidHeap() throws Exception { CreateContainerCmd createCommand = CONFIG.dockerClient .createContainerCmd(schedulerImage) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withCmd(Configuration.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); + .withCmd(ZookeeperCLIParameter.ZOOKEEPER_URL, "zk://" + "noIP" + ":2181/mesos", Configuration.ELASTICSEARCH_NODES, "3", Configuration.ELASTICSEARCH_RAM, "256"); CreateContainerResponse r = createCommand.exec(); String containerId = r.getId();