From be4dcb056cb2b560084fdd8e86fb1f3d1afa91c0 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 11:22:52 +0000 Subject: [PATCH 1/8] Bug with java home. --- .../apache/mesos/elasticsearch/scheduler/Configuration.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java index ae9bdd5b..4b31e23b 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java @@ -240,7 +240,11 @@ public void setFrameworkFileServerAddress(InetSocketAddress addr) { } public String getJavaHome() { - return javaHome.replaceAll("java$", "").replaceAll("/$", "") + "/"; + if (!javaHome.isEmpty()) { + return javaHome.replaceAll("java$", "").replaceAll("/$", "") + "/"; + } else { + return ""; + } } /** From 825ceaeb0ba686082fac0a112e6259ceb6ff6b85 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 11:24:23 +0000 Subject: [PATCH 2/8] Working jar system test. Exposes 31000 on localhost and hacks hostname to get access to executors. --- system-test/build.gradle | 4 +- .../systemtest/RunAsJarSystemTest.java | 206 ++++++++++++++++++ 2 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java diff --git a/system-test/build.gradle b/system-test/build.gradle index 03df2738..1c94a8d3 100644 --- a/system-test/build.gradle +++ b/system-test/build.gradle @@ -20,14 +20,14 @@ dependencies { compile 'com.github.docker-java:docker-java:1.3.0' compile 'com.mashape.unirest:unirest-java:1.4.5' compile 'com.jayway.awaitility:awaitility:1.6.3' - compile 'com.github.ContainerSolutions:mini-mesos:bb53e37569' + compile 'com.github.ContainerSolutions:mini-mesos:a20d094f65' systemTestCompile project(':scheduler') systemTestCompile 'junit:junit:4.12' systemTestCompile 'com.github.docker-java:docker-java:1.3.0' systemTestCompile 'com.mashape.unirest:unirest-java:1.4.5' systemTestCompile 'com.jayway.awaitility:awaitility:1.6.3' - systemTestCompile 'com.github.ContainerSolutions:mini-mesos:bb53e37569' + systemTestCompile 'com.github.ContainerSolutions:mini-mesos:a20d094f65' } task main(type:JavaExec, dependsOn: 'compileJava') { diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java new file mode 100644 index 00000000..a5770dce --- /dev/null +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -0,0 +1,206 @@ +package org.apache.mesos.elasticsearch.systemtest; + +import com.containersol.minimesos.MesosCluster; +import com.containersol.minimesos.container.AbstractContainer; +import com.containersol.minimesos.mesos.*; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.Bind; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.Link; +import com.github.dockerjava.api.model.Ports; +import org.apache.log4j.Logger; +import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter; +import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter; +import org.apache.mesos.elasticsearch.scheduler.Configuration; +import org.apache.mesos.elasticsearch.systemtest.callbacks.ElasticsearchNodesResponse; +import org.apache.mesos.elasticsearch.systemtest.util.ContainerLifecycleManagement; +import org.apache.mesos.elasticsearch.systemtest.util.DockerUtil; +import org.junit.*; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import java.io.File; +import java.io.IOException; +import java.security.SecureRandom; +import java.util.TreeMap; +import java.util.stream.IntStream; + +import static org.junit.Assert.assertTrue; + +/** + * A system test to ensure that the framework can run as a JAR, not using docker. + */ +public class RunAsJarSystemTest { + private static final Logger LOGGER = Logger.getLogger(RunAsJarSystemTest.class); + private static final ContainerLifecycleManagement CONTAINER_MANAGER = new ContainerLifecycleManagement(); + protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); + private static final int NUMBER_OF_TEST_TASKS = 1; + + // Need full control over the cluster, so need to do all the lifecycle stuff. + private static MesosCluster CLUSTER; + private static DockerClient dockerClient = DockerClientFactory.build(); + private JarScheduler scheduler; + + @Before + public void before() { + ClusterArchitecture.Builder builder = new ClusterArchitecture.Builder() + .withZooKeeper() + .withMaster(zooKeeper -> new MesosMaster22(dockerClient, zooKeeper)) + .withSlave(zooKeeper -> new MesosSlave22(dockerClient, zooKeeper)) // Have to have a slave before we build. This offer wont get accepted. + .withContainer(zkContainer -> new JarScheduler(dockerClient, zkContainer), ClusterContainers.Filter.zooKeeper()); + scheduler = (JarScheduler) builder.build().getClusterContainers().getOne(container -> container instanceof JarScheduler).get(); + IntStream.range(0,NUMBER_OF_TEST_TASKS).forEach(dummy -> + builder.withSlave(zooKeeper -> new MesosSlaveWithSchedulerLink(dockerClient, zooKeeper, scheduler)) + ); + CLUSTER = new MesosCluster(builder.build()); + + CLUSTER.start(); + + } + + @After + public void stopContainer() { + CONTAINER_MANAGER.stopAll(); + CLUSTER.stop(); + } + + @ClassRule + public static final TestWatcher WATCHER = new TestWatcher() { + @Override + protected void failed(Throwable e, Description description) { + CLUSTER.stop(); + } + }; + + @BeforeClass + public static void prepareCleanDockerEnvironment() { + new DockerUtil(dockerClient).killAllSchedulers(); + new DockerUtil(dockerClient).killAllExecutors(); + } + + @AfterClass + public static void killAllContainers() { + new DockerUtil(dockerClient).killAllExecutors(); + } + + @Test + public void shouldStartScheduler() throws IOException, InterruptedException { + ESTasks esTasks = new ESTasks(TEST_CONFIG, scheduler.getIpAddress()); + new TasksResponse(esTasks, NUMBER_OF_TEST_TASKS); + + ElasticsearchNodesResponse nodesResponse = new ElasticsearchNodesResponse(esTasks, NUMBER_OF_TEST_TASKS); + assertTrue("Elasticsearch nodes did not discover each other within 5 minutes", nodesResponse.isDiscoverySuccessful()); + } + + private class JarScheduler extends AbstractContainer { + private final ZooKeeper zooKeeperContainer; + + protected JarScheduler(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + super(dockerClient); + this.zooKeeperContainer = zooKeeperContainer; + } + + @Override + public void pullImage() { + dockerClient.pullImageCmd(TEST_CONFIG.getSchedulerImageName()); + } + + @Override + protected CreateContainerCmd dockerCommand() { + return dockerClient + .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) + .withName(TEST_CONFIG.getSchedulerName() + "_" + MesosCluster.getClusterId() + "_" + new SecureRandom().nextInt()) + .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") + .withExtraHosts("hostnamehack:" + ElasticsearchSchedulerContainer.DOCKER0_ADAPTOR_IP_ADDRESS) // Hack to get past offer refusal + .withCmd( + ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), + Configuration.FRAMEWORK_USE_DOCKER, "false", + ElasticsearchCLIParameter.ELASTICSEARCH_NODES, String.valueOf(NUMBER_OF_TEST_TASKS), + Configuration.ELASTICSEARCH_CPU, "0.1", + Configuration.ELASTICSEARCH_RAM, "128", + Configuration.ELASTICSEARCH_DISK, "10", + Configuration.JAVA_HOME, "/usr/bin" + ); + } + public String getZookeeperMesosUrl() { + return "zk://" + zooKeeperContainer.getIpAddress() + ":2181/mesos"; + } + } + + private class MesosMaster22 extends MesosMaster { + + public MesosMaster22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + super(dockerClient, zooKeeperContainer); + } + + @Override + protected CreateContainerCmd dockerCommand() { + CreateContainerCmd createContainerCmd = super.dockerCommand(); + return createContainerCmd.withImage("containersol/mesos-master:" + Main.MESOS_IMAGE_TAG); + } + } + + private class MesosSlave22 extends MesosSlave { + + public MesosSlave22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + super(dockerClient, zooKeeperContainer); + } + + @Override + protected CreateContainerCmd dockerCommand() { + CreateContainerCmd createContainerCmd = super.dockerCommand(); + return createContainerCmd.withImage("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG); + } + } + + private class MesosSlaveWithSchedulerLink extends MesosSlave { + + private final JarScheduler scheduler; + + protected MesosSlaveWithSchedulerLink(DockerClient dockerClient, ZooKeeper zooKeeperContainer, JarScheduler scheduler) { + super(dockerClient, zooKeeperContainer); + this.scheduler = scheduler; + } + + @Override + protected CreateContainerCmd dockerCommand() { + String dockerBin1 = "/usr/bin/docker"; + File dockerBinFile1 = new File(dockerBin1); + if(!dockerBinFile1.exists() || !dockerBinFile1.canExecute()) { + dockerBin1 = "/usr/local/bin/docker"; + dockerBinFile1 = new File(dockerBin1); + if(!dockerBinFile1.exists() || !dockerBinFile1.canExecute()) { + LOGGER.error("Docker binary not found in /usr/local/bin or /usr/bin. Creating containers will most likely fail."); + } + } + Ports ports = new Ports(); + ports.bind(ExposedPort.tcp(31000), Ports.Binding(31000)); + CreateContainerCmd createContainerCmd = this.dockerClient + .createContainerCmd("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG) + .withName("minimesos-agent-" + MesosCluster.getClusterId() + "-" + this.getRandomId()) + .withPrivileged(true) + .withEnv(this.createMesosLocalEnvironment()) + .withPid("host") + .withBinds(new Bind[]{Bind.parse("/var/lib/docker:/var/lib/docker"), Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup"), Bind.parse(String.format("%s:/usr/bin/docker", new Object[]{dockerBin1})), Bind.parse("/var/run/docker.sock:/var/run/docker.sock")}) + .withHostName("hostnamehack") // Hack to get past offer refusal + .withNetworkMode("bridge") + .withLinks(new Link(scheduler.getContainerId(), scheduler.getContainerId())) + .withExposedPorts(new ExposedPort(31000), new ExposedPort(5051)) + .withPortBindings(ports) + ; + return createContainerCmd; + } + + @Override + protected String[] createMesosLocalEnvironment() { + TreeMap map = this.getDefaultEnvVars(); + map.putAll(this.getSharedEnvVars()); + return (String[])map.entrySet().stream().map((e) -> { + return (String)e.getKey() + "=" + (String)e.getValue(); + }).toArray((x$0) -> { + return new String[x$0]; + }); + } + } +} From c7473b23715cd1ff13156ea7d0cd2b5862b1b10f Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 11:41:04 +0000 Subject: [PATCH 3/8] Removed unnecessary code. --- .../systemtest/RunAsJarSystemTest.java | 84 ++++++------------- 1 file changed, 26 insertions(+), 58 deletions(-) diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java index a5770dce..45a679ae 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -5,25 +5,20 @@ import com.containersol.minimesos.mesos.*; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.model.Bind; import com.github.dockerjava.api.model.ExposedPort; import com.github.dockerjava.api.model.Link; import com.github.dockerjava.api.model.Ports; -import org.apache.log4j.Logger; import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter; import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter; import org.apache.mesos.elasticsearch.scheduler.Configuration; import org.apache.mesos.elasticsearch.systemtest.callbacks.ElasticsearchNodesResponse; -import org.apache.mesos.elasticsearch.systemtest.util.ContainerLifecycleManagement; import org.apache.mesos.elasticsearch.systemtest.util.DockerUtil; import org.junit.*; import org.junit.rules.TestWatcher; import org.junit.runner.Description; -import java.io.File; import java.io.IOException; import java.security.SecureRandom; -import java.util.TreeMap; import java.util.stream.IntStream; import static org.junit.Assert.assertTrue; @@ -32,8 +27,6 @@ * A system test to ensure that the framework can run as a JAR, not using docker. */ public class RunAsJarSystemTest { - private static final Logger LOGGER = Logger.getLogger(RunAsJarSystemTest.class); - private static final ContainerLifecycleManagement CONTAINER_MANAGER = new ContainerLifecycleManagement(); protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); private static final int NUMBER_OF_TEST_TASKS = 1; @@ -47,7 +40,7 @@ public void before() { ClusterArchitecture.Builder builder = new ClusterArchitecture.Builder() .withZooKeeper() .withMaster(zooKeeper -> new MesosMaster22(dockerClient, zooKeeper)) - .withSlave(zooKeeper -> new MesosSlave22(dockerClient, zooKeeper)) // Have to have a slave before we build. This offer wont get accepted. + .withSlave(zooKeeper -> new MesosSlave22(dockerClient, zooKeeper)) // Have to have a slave before we build. Bit of a minimesos bug. This offer wont get accepted. .withContainer(zkContainer -> new JarScheduler(dockerClient, zkContainer), ClusterContainers.Filter.zooKeeper()); scheduler = (JarScheduler) builder.build().getClusterContainers().getOne(container -> container instanceof JarScheduler).get(); IntStream.range(0,NUMBER_OF_TEST_TASKS).forEach(dummy -> @@ -56,12 +49,10 @@ public void before() { CLUSTER = new MesosCluster(builder.build()); CLUSTER.start(); - } @After public void stopContainer() { - CONTAINER_MANAGER.stopAll(); CLUSTER.stop(); } @@ -112,7 +103,7 @@ protected CreateContainerCmd dockerCommand() { .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) .withName(TEST_CONFIG.getSchedulerName() + "_" + MesosCluster.getClusterId() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts("hostnamehack:" + ElasticsearchSchedulerContainer.DOCKER0_ADAPTOR_IP_ADDRESS) // Hack to get past offer refusal + .withExtraHosts("hostnamehack:" + ElasticsearchSchedulerContainer.DOCKER0_ADAPTOR_IP_ADDRESS) // Will redirect hostnamehack to host IP address (where ports are bound) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), Configuration.FRAMEWORK_USE_DOCKER, "false", @@ -128,79 +119,56 @@ public String getZookeeperMesosUrl() { } } - private class MesosMaster22 extends MesosMaster { + private class MesosSlaveWithSchedulerLink extends MesosSlave22 { - public MesosMaster22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + private final JarScheduler scheduler; + + protected MesosSlaveWithSchedulerLink(DockerClient dockerClient, ZooKeeper zooKeeperContainer, JarScheduler scheduler) { super(dockerClient, zooKeeperContainer); + this.scheduler = scheduler; } @Override protected CreateContainerCmd dockerCommand() { CreateContainerCmd createContainerCmd = super.dockerCommand(); - return createContainerCmd.withImage("containersol/mesos-master:" + Main.MESOS_IMAGE_TAG); + + Ports ports = new Ports(); + ports.bind(ExposedPort.tcp(31000), Ports.Binding(31000)); + createContainerCmd + .withHostName("hostnamehack") // Hack to get past offer refusal + .withLinks(new Link(scheduler.getContainerId(), scheduler.getContainerId())) + .withExposedPorts(new ExposedPort(31000), new ExposedPort(5051)) + .withPortBindings(ports) + ; + return createContainerCmd; } } - private class MesosSlave22 extends MesosSlave { + // TODO (pnw): Remove when upgrading ES to 25 + private class MesosMaster22 extends MesosMaster { - public MesosSlave22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + public MesosMaster22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient, zooKeeperContainer); } @Override protected CreateContainerCmd dockerCommand() { CreateContainerCmd createContainerCmd = super.dockerCommand(); - return createContainerCmd.withImage("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG); + return createContainerCmd.withImage("containersol/mesos-master:" + Main.MESOS_IMAGE_TAG); } } - private class MesosSlaveWithSchedulerLink extends MesosSlave { - - private final JarScheduler scheduler; + // TODO (pnw): Remove when upgrading ES to 25 + private class MesosSlave22 extends MesosSlave { - protected MesosSlaveWithSchedulerLink(DockerClient dockerClient, ZooKeeper zooKeeperContainer, JarScheduler scheduler) { + public MesosSlave22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient, zooKeeperContainer); - this.scheduler = scheduler; } @Override protected CreateContainerCmd dockerCommand() { - String dockerBin1 = "/usr/bin/docker"; - File dockerBinFile1 = new File(dockerBin1); - if(!dockerBinFile1.exists() || !dockerBinFile1.canExecute()) { - dockerBin1 = "/usr/local/bin/docker"; - dockerBinFile1 = new File(dockerBin1); - if(!dockerBinFile1.exists() || !dockerBinFile1.canExecute()) { - LOGGER.error("Docker binary not found in /usr/local/bin or /usr/bin. Creating containers will most likely fail."); - } - } - Ports ports = new Ports(); - ports.bind(ExposedPort.tcp(31000), Ports.Binding(31000)); - CreateContainerCmd createContainerCmd = this.dockerClient - .createContainerCmd("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG) - .withName("minimesos-agent-" + MesosCluster.getClusterId() + "-" + this.getRandomId()) - .withPrivileged(true) - .withEnv(this.createMesosLocalEnvironment()) - .withPid("host") - .withBinds(new Bind[]{Bind.parse("/var/lib/docker:/var/lib/docker"), Bind.parse("/sys/fs/cgroup:/sys/fs/cgroup"), Bind.parse(String.format("%s:/usr/bin/docker", new Object[]{dockerBin1})), Bind.parse("/var/run/docker.sock:/var/run/docker.sock")}) - .withHostName("hostnamehack") // Hack to get past offer refusal - .withNetworkMode("bridge") - .withLinks(new Link(scheduler.getContainerId(), scheduler.getContainerId())) - .withExposedPorts(new ExposedPort(31000), new ExposedPort(5051)) - .withPortBindings(ports) - ; - return createContainerCmd; - } - - @Override - protected String[] createMesosLocalEnvironment() { - TreeMap map = this.getDefaultEnvVars(); - map.putAll(this.getSharedEnvVars()); - return (String[])map.entrySet().stream().map((e) -> { - return (String)e.getKey() + "=" + (String)e.getValue(); - }).toArray((x$0) -> { - return new String[x$0]; - }); + CreateContainerCmd createContainerCmd = super.dockerCommand(); + return createContainerCmd.withImage("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG); } } } From 7e543adaa71290a144857e35bcab576f070d12e7 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 12:52:13 +0000 Subject: [PATCH 4/8] Fixed gradle check errors. --- .../systemtest/RunAsJarSystemTest.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java index 45a679ae..aed61149 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -26,12 +26,13 @@ /** * A system test to ensure that the framework can run as a JAR, not using docker. */ +@SuppressWarnings({"PMD.TooManyMethods"}) public class RunAsJarSystemTest { protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); private static final int NUMBER_OF_TEST_TASKS = 1; // Need full control over the cluster, so need to do all the lifecycle stuff. - private static MesosCluster CLUSTER; + private MesosCluster cluster; private static DockerClient dockerClient = DockerClientFactory.build(); private JarScheduler scheduler; @@ -43,24 +44,24 @@ public void before() { .withSlave(zooKeeper -> new MesosSlave22(dockerClient, zooKeeper)) // Have to have a slave before we build. Bit of a minimesos bug. This offer wont get accepted. .withContainer(zkContainer -> new JarScheduler(dockerClient, zkContainer), ClusterContainers.Filter.zooKeeper()); scheduler = (JarScheduler) builder.build().getClusterContainers().getOne(container -> container instanceof JarScheduler).get(); - IntStream.range(0,NUMBER_OF_TEST_TASKS).forEach(dummy -> + IntStream.range(0, NUMBER_OF_TEST_TASKS).forEach(dummy -> builder.withSlave(zooKeeper -> new MesosSlaveWithSchedulerLink(dockerClient, zooKeeper, scheduler)) ); - CLUSTER = new MesosCluster(builder.build()); + cluster = new MesosCluster(builder.build()); - CLUSTER.start(); + cluster.start(); } @After public void stopContainer() { - CLUSTER.stop(); + cluster.stop(); } - @ClassRule - public static final TestWatcher WATCHER = new TestWatcher() { + @Rule + public final TestWatcher WATCHER = new TestWatcher() { @Override protected void failed(Throwable e, Description description) { - CLUSTER.stop(); + cluster.stop(); } }; @@ -72,6 +73,7 @@ public static void prepareCleanDockerEnvironment() { @AfterClass public static void killAllContainers() { + new DockerUtil(dockerClient).killAllSchedulers(); new DockerUtil(dockerClient).killAllExecutors(); } @@ -84,7 +86,7 @@ public void shouldStartScheduler() throws IOException, InterruptedException { assertTrue("Elasticsearch nodes did not discover each other within 5 minutes", nodesResponse.isDiscoverySuccessful()); } - private class JarScheduler extends AbstractContainer { + private static class JarScheduler extends AbstractContainer { private final ZooKeeper zooKeeperContainer; protected JarScheduler(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { @@ -119,7 +121,7 @@ public String getZookeeperMesosUrl() { } } - private class MesosSlaveWithSchedulerLink extends MesosSlave22 { + private static class MesosSlaveWithSchedulerLink extends MesosSlave22 { private final JarScheduler scheduler; @@ -145,7 +147,7 @@ protected CreateContainerCmd dockerCommand() { } // TODO (pnw): Remove when upgrading ES to 25 - private class MesosMaster22 extends MesosMaster { + private static class MesosMaster22 extends MesosMaster { public MesosMaster22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient, zooKeeperContainer); @@ -159,7 +161,7 @@ protected CreateContainerCmd dockerCommand() { } // TODO (pnw): Remove when upgrading ES to 25 - private class MesosSlave22 extends MesosSlave { + private static class MesosSlave22 extends MesosSlave { public MesosSlave22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient, zooKeeperContainer); From b1e6d13f9c478815a542fa8f883f568ded98a184 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 14:21:13 +0000 Subject: [PATCH 5/8] PMD fix. --- .../systemtest/RunAsJarSystemTest.java | 40 ++++--------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java index aed61149..7ab24e92 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -26,7 +26,7 @@ /** * A system test to ensure that the framework can run as a JAR, not using docker. */ -@SuppressWarnings({"PMD.TooManyMethods"}) +@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidUsingHardCodedIP"}) public class RunAsJarSystemTest { protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); private static final int NUMBER_OF_TEST_TASKS = 1; @@ -40,8 +40,8 @@ public class RunAsJarSystemTest { public void before() { ClusterArchitecture.Builder builder = new ClusterArchitecture.Builder() .withZooKeeper() - .withMaster(zooKeeper -> new MesosMaster22(dockerClient, zooKeeper)) - .withSlave(zooKeeper -> new MesosSlave22(dockerClient, zooKeeper)) // Have to have a slave before we build. Bit of a minimesos bug. This offer wont get accepted. + .withMaster() + .withSlave() // Have to have a slave before we build. Bit of a minimesos bug. This offer wont get accepted. .withContainer(zkContainer -> new JarScheduler(dockerClient, zkContainer), ClusterContainers.Filter.zooKeeper()); scheduler = (JarScheduler) builder.build().getClusterContainers().getOne(container -> container instanceof JarScheduler).get(); IntStream.range(0, NUMBER_OF_TEST_TASKS).forEach(dummy -> @@ -88,10 +88,12 @@ public void shouldStartScheduler() throws IOException, InterruptedException { private static class JarScheduler extends AbstractContainer { private final ZooKeeper zooKeeperContainer; + private String docker0AdaptorIpAddress; protected JarScheduler(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient); this.zooKeeperContainer = zooKeeperContainer; + docker0AdaptorIpAddress = dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; } @Override @@ -105,7 +107,7 @@ protected CreateContainerCmd dockerCommand() { .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) .withName(TEST_CONFIG.getSchedulerName() + "_" + MesosCluster.getClusterId() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts("hostnamehack:" + ElasticsearchSchedulerContainer.DOCKER0_ADAPTOR_IP_ADDRESS) // Will redirect hostnamehack to host IP address (where ports are bound) + .withExtraHosts("hostnamehack:" + docker0AdaptorIpAddress) // Will redirect hostnamehack to host IP address (where ports are bound) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), Configuration.FRAMEWORK_USE_DOCKER, "false", @@ -121,7 +123,7 @@ public String getZookeeperMesosUrl() { } } - private static class MesosSlaveWithSchedulerLink extends MesosSlave22 { + private static class MesosSlaveWithSchedulerLink extends MesosSlave { private final JarScheduler scheduler; @@ -145,32 +147,4 @@ protected CreateContainerCmd dockerCommand() { return createContainerCmd; } } - - // TODO (pnw): Remove when upgrading ES to 25 - private static class MesosMaster22 extends MesosMaster { - - public MesosMaster22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { - super(dockerClient, zooKeeperContainer); - } - - @Override - protected CreateContainerCmd dockerCommand() { - CreateContainerCmd createContainerCmd = super.dockerCommand(); - return createContainerCmd.withImage("containersol/mesos-master:" + Main.MESOS_IMAGE_TAG); - } - } - - // TODO (pnw): Remove when upgrading ES to 25 - private static class MesosSlave22 extends MesosSlave { - - public MesosSlave22(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { - super(dockerClient, zooKeeperContainer); - } - - @Override - protected CreateContainerCmd dockerCommand() { - CreateContainerCmd createContainerCmd = super.dockerCommand(); - return createContainerCmd.withImage("containersol/mesos-agent:" + Main.MESOS_IMAGE_TAG); - } - } } From b3433447e77a134af64512c5aec1a0ec4ba49be3 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 14:26:31 +0000 Subject: [PATCH 6/8] Update to latest minimesos master with new API. --- system-test/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system-test/build.gradle b/system-test/build.gradle index 1c94a8d3..92a7a1c6 100644 --- a/system-test/build.gradle +++ b/system-test/build.gradle @@ -20,14 +20,14 @@ dependencies { compile 'com.github.docker-java:docker-java:1.3.0' compile 'com.mashape.unirest:unirest-java:1.4.5' compile 'com.jayway.awaitility:awaitility:1.6.3' - compile 'com.github.ContainerSolutions:mini-mesos:a20d094f65' + compile 'com.github.ContainerSolutions:mini-mesos:91c6dddbbd' systemTestCompile project(':scheduler') systemTestCompile 'junit:junit:4.12' systemTestCompile 'com.github.docker-java:docker-java:1.3.0' systemTestCompile 'com.mashape.unirest:unirest-java:1.4.5' systemTestCompile 'com.jayway.awaitility:awaitility:1.6.3' - systemTestCompile 'com.github.ContainerSolutions:mini-mesos:a20d094f65' + systemTestCompile 'com.github.ContainerSolutions:mini-mesos:91c6dddbbd' } task main(type:JavaExec, dependsOn: 'compileJava') { From e0104ff35f383e52e2ed9bcc5c5a677047108ee4 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 14:42:10 +0000 Subject: [PATCH 7/8] Attempt to fix intermittent system test failure on executor system test. Should not fail. --- .../mesos/elasticsearch/systemtest/ExecutorSystemTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java index b381dbe5..7129e29b 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java @@ -15,8 +15,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * System test for the executor @@ -56,7 +56,7 @@ public void ensureEnvVarPointsToLibMesos() throws IOException { // Get MESOS_NATIVE_JAVA_LIBRARY from env List env = Arrays.asList(result.split("\n")).stream().filter(s -> s.contains("MESOS_NATIVE_JAVA_LIBRARY")).collect(Collectors.toList()); - assertEquals("env does not have MESOS_NATIVE_JAVA_LIBRARY: " + result, 1, env.size()); + assertTrue("env does not have MESOS_NATIVE_JAVA_LIBRARY: " + result, env.size() > 0); // Remote execute the ENV var to make sure it points to a real file String path = env.get(0).split("=")[1].replace("\r", "").replace("\n", ""); From da592112fcdf7e3f44ebcea85e7771acd718491d Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Fri, 6 Nov 2015 15:17:43 +0000 Subject: [PATCH 8/8] Fix issues after peer review. Added documentation, --- .../systemtest/Configuration.java | 7 +++ .../ElasticsearchSchedulerContainer.java | 13 +++-- .../systemtest/ReconciliationSystemTest.java | 2 +- .../systemtest/RunAsJarSystemTest.java | 51 +++++++++++++------ 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java index 31fb6d0c..04a199c9 100644 --- a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java +++ b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java @@ -1,8 +1,11 @@ package org.apache.mesos.elasticsearch.systemtest; +import com.github.dockerjava.api.DockerClient; + /** * SystemTest configuration object */ +@SuppressWarnings({"PMD.AvoidUsingHardCodedIP"}) public class Configuration { private String schedulerImageName = "mesos/elasticsearch-scheduler"; private String schedulerName = "elasticsearch-scheduler"; @@ -11,6 +14,10 @@ public class Configuration { private int elasticsearchMemorySize = 512; private String elasticsearchJobName = "esdemo"; + public static String getDocker0AdaptorIpAddress(DockerClient dockerClient) { + return dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; + } + public String getSchedulerImageName() { return schedulerImageName; } diff --git a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java index 341a9070..072d8524 100644 --- a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java +++ b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java @@ -5,8 +5,6 @@ import com.containersol.minimesos.mesos.MesosSlave; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.PortBinding; import org.apache.commons.lang.StringUtils; import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter; import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter; @@ -16,14 +14,15 @@ import java.util.Arrays; import java.util.List; +import static org.apache.mesos.elasticsearch.systemtest.Configuration.getDocker0AdaptorIpAddress; + /** * Container for the Elasticsearch scheduler */ -@SuppressWarnings("PMD.AvoidUsingHardCodedIP") public class ElasticsearchSchedulerContainer extends AbstractContainer { private static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); - protected final String DOCKER0_ADAPTOR_IP_ADDRESS; + protected final String docker0AdaptorIpAddress; private final String zkIp; @@ -43,7 +42,7 @@ public ElasticsearchSchedulerContainer(DockerClient dockerClient, String zkIp, S this.frameworkRole = frameworkRole; this.cluster = cluster; - DOCKER0_ADAPTOR_IP_ADDRESS = dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; + docker0AdaptorIpAddress = getDocker0AdaptorIpAddress(dockerClient); } @Override @@ -55,14 +54,14 @@ public void pullImage() { protected CreateContainerCmd dockerCommand() { List slaves = Arrays.asList(cluster.getSlaves()); - // Note we are redirecting each slave host to the static docker0 adaptor address (DOCKER0_ADAPTOR_IP_ADDRESS). + // Note we are redirecting each slave host to the static docker0 adaptor address (docker0AdaptorIpAddress). // The executors expose ports and when running system tests these are exposed on the single docker daemon machine // (localhost for linux, virtual machine for mac users). However, the docker0 ip address *always* points to the host. return dockerClient .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) .withName(TEST_CONFIG.getSchedulerName() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + DOCKER0_ADAPTOR_IP_ADDRESS).toArray(String[]::new)) + .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + docker0AdaptorIpAddress).toArray(String[]::new)) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), ZookeeperCLIParameter.ZOOKEEPER_FRAMEWORK_URL, getZookeeperFrameworkUrl(), diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java index 85644128..f30bf3ce 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java @@ -128,7 +128,7 @@ protected CreateContainerCmd dockerCommand() { .createContainerCmd(getTestConfig().getSchedulerImageName()) .withName(getTestConfig().getSchedulerName() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + DOCKER0_ADAPTOR_IP_ADDRESS).toArray(String[]::new)) + .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + docker0AdaptorIpAddress).toArray(String[]::new)) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), Configuration.EXECUTOR_HEALTH_DELAY, "99", diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java index 7ab24e92..910152bc 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -21,12 +21,31 @@ import java.security.SecureRandom; import java.util.stream.IntStream; +import static org.apache.mesos.elasticsearch.systemtest.Configuration.getDocker0AdaptorIpAddress; import static org.junit.Assert.assertTrue; /** * A system test to ensure that the framework can run as a JAR, not using docker. + *

+ * Test method + *

+ * The goal is to start the framework in jar mode, and ensure that the system can still be discovered. + * We still start the scheduler in the container (but with jar mode enabled) because the system running the system + * test may not have Java 8 installed. But this is no issue; the test is exactly the same. + *

+ * Next, a custom MesosSlave is provided so that it can resolve the IP address of the scheduler (which is hosting + * the executor jar). Its own hostname is set to "hostnamehack" so that it can be resolved on the scheduler side. + * The ES port (31000 in this case) is bound to the host, where the host is the VM or linux OS. This will allow us + * to curl the ES endpoint without knowing the IP address of the slave container. + *

+ * The scheduler has an extra host called "hostnamehack" that resolves to the docker0 adaptor address. This is + * equivalent to the VM/linux localhost, is known ahead of time and is system independent. I.e. both Mac's and linux + * boxes will be able to resolve this address. The port is the same. + *

+ * So finally, the ES system tests can cary on as normal and resolve the ES hosts. Nice! + *

*/ -@SuppressWarnings({"PMD.TooManyMethods", "PMD.AvoidUsingHardCodedIP"}) +@SuppressWarnings({"PMD.TooManyMethods"}) public class RunAsJarSystemTest { protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); private static final int NUMBER_OF_TEST_TASKS = 1; @@ -36,6 +55,20 @@ public class RunAsJarSystemTest { private static DockerClient dockerClient = DockerClientFactory.build(); private JarScheduler scheduler; + @BeforeClass + public static void prepareCleanDockerEnvironment() { + new DockerUtil(dockerClient).killAllSchedulers(); + new DockerUtil(dockerClient).killAllExecutors(); + } + + @Rule + public final TestWatcher WATCHER = new TestWatcher() { + @Override + protected void failed(Throwable e, Description description) { + cluster.stop(); + } + }; + @Before public void before() { ClusterArchitecture.Builder builder = new ClusterArchitecture.Builder() @@ -57,20 +90,6 @@ public void stopContainer() { cluster.stop(); } - @Rule - public final TestWatcher WATCHER = new TestWatcher() { - @Override - protected void failed(Throwable e, Description description) { - cluster.stop(); - } - }; - - @BeforeClass - public static void prepareCleanDockerEnvironment() { - new DockerUtil(dockerClient).killAllSchedulers(); - new DockerUtil(dockerClient).killAllExecutors(); - } - @AfterClass public static void killAllContainers() { new DockerUtil(dockerClient).killAllSchedulers(); @@ -93,7 +112,7 @@ private static class JarScheduler extends AbstractContainer { protected JarScheduler(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { super(dockerClient); this.zooKeeperContainer = zooKeeperContainer; - docker0AdaptorIpAddress = dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; + docker0AdaptorIpAddress = getDocker0AdaptorIpAddress(dockerClient); } @Override