diff --git a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java index ae9bdd5b..4b31e23b 100644 --- a/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java +++ b/scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java @@ -240,7 +240,11 @@ public void setFrameworkFileServerAddress(InetSocketAddress addr) { } public String getJavaHome() { - return javaHome.replaceAll("java$", "").replaceAll("/$", "") + "/"; + if (!javaHome.isEmpty()) { + return javaHome.replaceAll("java$", "").replaceAll("/$", "") + "/"; + } else { + return ""; + } } /** diff --git a/system-test/build.gradle b/system-test/build.gradle index 3cefbb92..92a7a1c6 100644 --- a/system-test/build.gradle +++ b/system-test/build.gradle @@ -20,14 +20,14 @@ dependencies { compile 'com.github.docker-java:docker-java:1.3.0' compile 'com.mashape.unirest:unirest-java:1.4.5' compile 'com.jayway.awaitility:awaitility:1.6.3' - compile 'com.github.ContainerSolutions:minimesos:a53e783f88' + compile 'com.github.ContainerSolutions:mini-mesos:91c6dddbbd' systemTestCompile project(':scheduler') systemTestCompile 'junit:junit:4.12' systemTestCompile 'com.github.docker-java:docker-java:1.3.0' systemTestCompile 'com.mashape.unirest:unirest-java:1.4.5' systemTestCompile 'com.jayway.awaitility:awaitility:1.6.3' - systemTestCompile 'com.github.ContainerSolutions:minimesos:a53e783f88' + systemTestCompile 'com.github.ContainerSolutions:mini-mesos:91c6dddbbd' } task main(type:JavaExec, dependsOn: 'compileJava') { diff --git a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java index 31fb6d0c..04a199c9 100644 --- a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java +++ b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/Configuration.java @@ -1,8 +1,11 @@ package org.apache.mesos.elasticsearch.systemtest; +import com.github.dockerjava.api.DockerClient; + /** * SystemTest configuration object */ +@SuppressWarnings({"PMD.AvoidUsingHardCodedIP"}) public class Configuration { private String schedulerImageName = "mesos/elasticsearch-scheduler"; private String schedulerName = "elasticsearch-scheduler"; @@ -11,6 +14,10 @@ public class Configuration { private int elasticsearchMemorySize = 512; private String elasticsearchJobName = "esdemo"; + public static String getDocker0AdaptorIpAddress(DockerClient dockerClient) { + return dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; + } + public String getSchedulerImageName() { return schedulerImageName; } diff --git a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java index 341a9070..072d8524 100644 --- a/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java +++ b/system-test/src/main/java/org/apache/mesos/elasticsearch/systemtest/ElasticsearchSchedulerContainer.java @@ -5,8 +5,6 @@ import com.containersol.minimesos.mesos.MesosSlave; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.CreateContainerCmd; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.PortBinding; import org.apache.commons.lang.StringUtils; import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter; import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter; @@ -16,14 +14,15 @@ import java.util.Arrays; import java.util.List; +import static org.apache.mesos.elasticsearch.systemtest.Configuration.getDocker0AdaptorIpAddress; + /** * Container for the Elasticsearch scheduler */ -@SuppressWarnings("PMD.AvoidUsingHardCodedIP") public class ElasticsearchSchedulerContainer extends AbstractContainer { private static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); - protected final String DOCKER0_ADAPTOR_IP_ADDRESS; + protected final String docker0AdaptorIpAddress; private final String zkIp; @@ -43,7 +42,7 @@ public ElasticsearchSchedulerContainer(DockerClient dockerClient, String zkIp, S this.frameworkRole = frameworkRole; this.cluster = cluster; - DOCKER0_ADAPTOR_IP_ADDRESS = dockerClient.versionCmd().exec().getVersion().startsWith("1.9.") ? "172.17.0.1" : "172.17.42.1"; + docker0AdaptorIpAddress = getDocker0AdaptorIpAddress(dockerClient); } @Override @@ -55,14 +54,14 @@ public void pullImage() { protected CreateContainerCmd dockerCommand() { List slaves = Arrays.asList(cluster.getSlaves()); - // Note we are redirecting each slave host to the static docker0 adaptor address (DOCKER0_ADAPTOR_IP_ADDRESS). + // Note we are redirecting each slave host to the static docker0 adaptor address (docker0AdaptorIpAddress). // The executors expose ports and when running system tests these are exposed on the single docker daemon machine // (localhost for linux, virtual machine for mac users). However, the docker0 ip address *always* points to the host. return dockerClient .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) .withName(TEST_CONFIG.getSchedulerName() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + DOCKER0_ADAPTOR_IP_ADDRESS).toArray(String[]::new)) + .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + docker0AdaptorIpAddress).toArray(String[]::new)) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), ZookeeperCLIParameter.ZOOKEEPER_FRAMEWORK_URL, getZookeeperFrameworkUrl(), diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java index b381dbe5..7129e29b 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ExecutorSystemTest.java @@ -15,8 +15,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * System test for the executor @@ -56,7 +56,7 @@ public void ensureEnvVarPointsToLibMesos() throws IOException { // Get MESOS_NATIVE_JAVA_LIBRARY from env List env = Arrays.asList(result.split("\n")).stream().filter(s -> s.contains("MESOS_NATIVE_JAVA_LIBRARY")).collect(Collectors.toList()); - assertEquals("env does not have MESOS_NATIVE_JAVA_LIBRARY: " + result, 1, env.size()); + assertTrue("env does not have MESOS_NATIVE_JAVA_LIBRARY: " + result, env.size() > 0); // Remote execute the ENV var to make sure it points to a real file String path = env.get(0).split("=")[1].replace("\r", "").replace("\n", ""); diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java index 85644128..f30bf3ce 100644 --- a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/ReconciliationSystemTest.java @@ -128,7 +128,7 @@ protected CreateContainerCmd dockerCommand() { .createContainerCmd(getTestConfig().getSchedulerImageName()) .withName(getTestConfig().getSchedulerName() + "_" + new SecureRandom().nextInt()) .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") - .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + DOCKER0_ADAPTOR_IP_ADDRESS).toArray(String[]::new)) + .withExtraHosts(slaves.stream().map(mesosSlave -> mesosSlave.getHostname() + ":" + docker0AdaptorIpAddress).toArray(String[]::new)) .withCmd( ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), Configuration.EXECUTOR_HEALTH_DELAY, "99", diff --git a/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java new file mode 100644 index 00000000..910152bc --- /dev/null +++ b/system-test/src/systemTest/java/org/apache/mesos/elasticsearch/systemtest/RunAsJarSystemTest.java @@ -0,0 +1,169 @@ +package org.apache.mesos.elasticsearch.systemtest; + +import com.containersol.minimesos.MesosCluster; +import com.containersol.minimesos.container.AbstractContainer; +import com.containersol.minimesos.mesos.*; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerCmd; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.Link; +import com.github.dockerjava.api.model.Ports; +import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter; +import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter; +import org.apache.mesos.elasticsearch.scheduler.Configuration; +import org.apache.mesos.elasticsearch.systemtest.callbacks.ElasticsearchNodesResponse; +import org.apache.mesos.elasticsearch.systemtest.util.DockerUtil; +import org.junit.*; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import java.io.IOException; +import java.security.SecureRandom; +import java.util.stream.IntStream; + +import static org.apache.mesos.elasticsearch.systemtest.Configuration.getDocker0AdaptorIpAddress; +import static org.junit.Assert.assertTrue; + +/** + * A system test to ensure that the framework can run as a JAR, not using docker. + *

+ * Test method + *

+ * The goal is to start the framework in jar mode, and ensure that the system can still be discovered. + * We still start the scheduler in the container (but with jar mode enabled) because the system running the system + * test may not have Java 8 installed. But this is no issue; the test is exactly the same. + *

+ * Next, a custom MesosSlave is provided so that it can resolve the IP address of the scheduler (which is hosting + * the executor jar). Its own hostname is set to "hostnamehack" so that it can be resolved on the scheduler side. + * The ES port (31000 in this case) is bound to the host, where the host is the VM or linux OS. This will allow us + * to curl the ES endpoint without knowing the IP address of the slave container. + *

+ * The scheduler has an extra host called "hostnamehack" that resolves to the docker0 adaptor address. This is + * equivalent to the VM/linux localhost, is known ahead of time and is system independent. I.e. both Mac's and linux + * boxes will be able to resolve this address. The port is the same. + *

+ * So finally, the ES system tests can cary on as normal and resolve the ES hosts. Nice! + *

+ */ +@SuppressWarnings({"PMD.TooManyMethods"}) +public class RunAsJarSystemTest { + protected static final org.apache.mesos.elasticsearch.systemtest.Configuration TEST_CONFIG = new org.apache.mesos.elasticsearch.systemtest.Configuration(); + private static final int NUMBER_OF_TEST_TASKS = 1; + + // Need full control over the cluster, so need to do all the lifecycle stuff. + private MesosCluster cluster; + private static DockerClient dockerClient = DockerClientFactory.build(); + private JarScheduler scheduler; + + @BeforeClass + public static void prepareCleanDockerEnvironment() { + new DockerUtil(dockerClient).killAllSchedulers(); + new DockerUtil(dockerClient).killAllExecutors(); + } + + @Rule + public final TestWatcher WATCHER = new TestWatcher() { + @Override + protected void failed(Throwable e, Description description) { + cluster.stop(); + } + }; + + @Before + public void before() { + ClusterArchitecture.Builder builder = new ClusterArchitecture.Builder() + .withZooKeeper() + .withMaster() + .withSlave() // Have to have a slave before we build. Bit of a minimesos bug. This offer wont get accepted. + .withContainer(zkContainer -> new JarScheduler(dockerClient, zkContainer), ClusterContainers.Filter.zooKeeper()); + scheduler = (JarScheduler) builder.build().getClusterContainers().getOne(container -> container instanceof JarScheduler).get(); + IntStream.range(0, NUMBER_OF_TEST_TASKS).forEach(dummy -> + builder.withSlave(zooKeeper -> new MesosSlaveWithSchedulerLink(dockerClient, zooKeeper, scheduler)) + ); + cluster = new MesosCluster(builder.build()); + + cluster.start(); + } + + @After + public void stopContainer() { + cluster.stop(); + } + + @AfterClass + public static void killAllContainers() { + new DockerUtil(dockerClient).killAllSchedulers(); + new DockerUtil(dockerClient).killAllExecutors(); + } + + @Test + public void shouldStartScheduler() throws IOException, InterruptedException { + ESTasks esTasks = new ESTasks(TEST_CONFIG, scheduler.getIpAddress()); + new TasksResponse(esTasks, NUMBER_OF_TEST_TASKS); + + ElasticsearchNodesResponse nodesResponse = new ElasticsearchNodesResponse(esTasks, NUMBER_OF_TEST_TASKS); + assertTrue("Elasticsearch nodes did not discover each other within 5 minutes", nodesResponse.isDiscoverySuccessful()); + } + + private static class JarScheduler extends AbstractContainer { + private final ZooKeeper zooKeeperContainer; + private String docker0AdaptorIpAddress; + + protected JarScheduler(DockerClient dockerClient, ZooKeeper zooKeeperContainer) { + super(dockerClient); + this.zooKeeperContainer = zooKeeperContainer; + docker0AdaptorIpAddress = getDocker0AdaptorIpAddress(dockerClient); + } + + @Override + public void pullImage() { + dockerClient.pullImageCmd(TEST_CONFIG.getSchedulerImageName()); + } + + @Override + protected CreateContainerCmd dockerCommand() { + return dockerClient + .createContainerCmd(TEST_CONFIG.getSchedulerImageName()) + .withName(TEST_CONFIG.getSchedulerName() + "_" + MesosCluster.getClusterId() + "_" + new SecureRandom().nextInt()) + .withEnv("JAVA_OPTS=-Xms128m -Xmx256m") + .withExtraHosts("hostnamehack:" + docker0AdaptorIpAddress) // Will redirect hostnamehack to host IP address (where ports are bound) + .withCmd( + ZookeeperCLIParameter.ZOOKEEPER_MESOS_URL, getZookeeperMesosUrl(), + Configuration.FRAMEWORK_USE_DOCKER, "false", + ElasticsearchCLIParameter.ELASTICSEARCH_NODES, String.valueOf(NUMBER_OF_TEST_TASKS), + Configuration.ELASTICSEARCH_CPU, "0.1", + Configuration.ELASTICSEARCH_RAM, "128", + Configuration.ELASTICSEARCH_DISK, "10", + Configuration.JAVA_HOME, "/usr/bin" + ); + } + public String getZookeeperMesosUrl() { + return "zk://" + zooKeeperContainer.getIpAddress() + ":2181/mesos"; + } + } + + private static class MesosSlaveWithSchedulerLink extends MesosSlave { + + private final JarScheduler scheduler; + + protected MesosSlaveWithSchedulerLink(DockerClient dockerClient, ZooKeeper zooKeeperContainer, JarScheduler scheduler) { + super(dockerClient, zooKeeperContainer); + this.scheduler = scheduler; + } + + @Override + protected CreateContainerCmd dockerCommand() { + CreateContainerCmd createContainerCmd = super.dockerCommand(); + + Ports ports = new Ports(); + ports.bind(ExposedPort.tcp(31000), Ports.Binding(31000)); + createContainerCmd + .withHostName("hostnamehack") // Hack to get past offer refusal + .withLinks(new Link(scheduler.getContainerId(), scheduler.getContainerId())) + .withExposedPorts(new ExposedPort(31000), new ExposedPort(5051)) + .withPortBindings(ports) + ; + return createContainerCmd; + } + } +}