Infra/436 jenkins failure

commons/build.gradle

@@ -11,6 +11,9 @@ task taskCopyFilesForDocker(type: Copy) {
 
 dependencies {
     compile "com.beust:jcommander:1.48"
-    compile 'commons-lang:commons-lang:2.6'
     compile "log4j:log4j:1.2.16"
+    compile "org.apache.commons:commons-exec:1.3"
+    compile "org.apache.commons:commons-lang3:3.4"
+
+    testCompile "commons-validator:commons-validator:1.5.0"
 }

commons/src/main/java/org/apache/mesos/elasticsearch/common/util/NetworkUtils.java

@@ -0,0 +1,126 @@
+package org.apache.mesos.elasticsearch.common.util;
+
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.*;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.Map;
+
+/**
+ * Utilities to help with networking
+ */
+@SuppressWarnings("PMD.AvoidUsingHardCodedIP")
+public class NetworkUtils {
+    private static final Logger LOG = Logger.getLogger(NetworkUtils.class);
+    public static final String DOCKER_MACHINE_IP = "docker-machine ip";
+    public static final String LOCALHOST = "127.0.0.1";
+    public static final String DOCKER_MACHINE_NAME = "DOCKER_MACHINE_NAME";
+
+    public static InetAddress hostAddress() {
+        try {
+            return InetAddress.getLocalHost();
+        } catch (UnknownHostException e) {
+            LOG.error("", e);
+            throw new RuntimeException("Unable to bind to local host.");
+        }
+    }
+
+    public static InetSocketAddress hostSocket(int port) {
+        return new InetSocketAddress(hostAddress(), port);
+    }
+
+    public static String addressToString(InetSocketAddress address, Boolean useIpAddress) {
+        if (useIpAddress) {
+            return "http://" + address.getAddress().getHostAddress() + ":" + address.getPort();
+        } else {
+            return "http://" + address.getAddress().getHostName() + ":" + address.getPort();
+        }
+    }
+
+    public static String getDockerMachineName(Map<String, String> environment) {
+        String envVar = DOCKER_MACHINE_NAME;
+        String dockerMachineName = environment.getOrDefault(envVar, "");
+        if (dockerMachineName == null || dockerMachineName.isEmpty()) {
+            LOG.debug("The environmental variable DOCKER_MACHINE_NAME was not found. Using docker0 address.");
+        }
+        return dockerMachineName;
+    }
+
+    public static String getDockerHostIpAddress(Map<String, String> environment) {
+        String ipAddress = LOCALHOST; // Default of localhost
+        String dockerMachineName = getDockerMachineName(environment);
+
+        if (!dockerMachineName.isEmpty()) {
+            LOG.debug("Docker machine name = " + dockerMachineName);
+            CommandLine commandline = CommandLine.parse(DOCKER_MACHINE_IP);
+            commandline.addArgument(dockerMachineName);
+            LOG.debug("Running exec: " + commandline.toString());
+            try {
+                ipAddress = StringUtils.strip(runCommand(commandline));
+            } catch (IOException e) {
+                LOG.error("Unable to run exec command to find ip address.", e);
+            }
+        } else {
+            ipAddress = getDocker0AdapterIPAddress();
+        }
+        LOG.debug("Returned IP address: " + ipAddress);
+        return ipAddress;
+    }
+
+    public static Map<String, String> getEnvironment() {
+        Map<String, String> env = Collections.emptyMap();
+        try {
+            env = EnvironmentUtils.getProcEnvironment();
+        } catch (IOException e) {
+            LOG.error("Unable to get environmental variables", e);
+        }
+        return env;
+    }
+
+    public static String runCommand(CommandLine commandline) throws IOException {
+        DefaultExecutor exec = new DefaultExecutor();
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream);
+        exec.setStreamHandler(streamHandler);
+        exec.execute(commandline);
+        return outputStream.toString(Charset.defaultCharset().name());
+    }
+
+    public static String getDocker0AdapterIPAddress() {
+        InetAddress docker0 = getLocalAddress("docker0");
+        if (docker0 == null) {
+            LOG.error("Could not get address for docker0");
+            return LOCALHOST;
+        } else {
+            return docker0.getHostAddress();
+        }
+    }
+
+    private static InetAddress getLocalAddress(String adaptorName){
+        try {
+            Enumeration<NetworkInterface> b = NetworkInterface.getNetworkInterfaces();
+            while (b.hasMoreElements()) {
+                NetworkInterface networkInterface = b.nextElement();
+                if (networkInterface.getName().equals(adaptorName)) {
+                    for (InterfaceAddress f : networkInterface.getInterfaceAddresses()) {
+                        if (f.getAddress().isSiteLocalAddress()) {
+                            return f.getAddress();
+                        }
+                    }
+                }
+            }
+        } catch (SocketException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+}

commons/src/test/java/org/apache/mesos/elasticsearch/common/util/NetworkUtilsTest.java

@@ -0,0 +1,59 @@
+package org.apache.mesos.elasticsearch.common.util;
+
+import org.apache.commons.validator.routines.InetAddressValidator;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+
+import static org.junit.Assert.*;
+
+/**
+ */
+public class NetworkUtilsTest {
+
+    public static boolean validate(final String ip) {
+        return InetAddressValidator.getInstance().isValid(ip);
+    }
+
+    @Test
+    // Note: On OSX, when not connected to a network, it will return a IPv6 address, which will not validate properly.
+    // Please connect to a network to obtain a IPv4 address.
+    public void shouldProvideIPAddress() {
+        int port = 1234;
+        String string = NetworkUtils.addressToString(NetworkUtils.hostSocket(port), true);
+        assertTrue(validate(string.replace("http://", "").replace(":" + port, "")));
+    }
+
+
+    @Test
+    public void shouldProvideHostname() {
+        int port = 1234;
+        String string = NetworkUtils.addressToString(NetworkUtils.hostSocket(port), false);
+        assertFalse(validate(string.replace("http://", "").replace(":" + port, "")));
+    }
+
+    @Test
+    public void shouldGetDockerIPAddress() throws IOException {
+        // Should always be either a valid IP or 127.0.0.1
+        assertTrue(validate(NetworkUtils.getDockerHostIpAddress(NetworkUtils.getEnvironment())));
+    }
+
+    @Test
+    public void shouldReturnLocahostOrDocker0AddressWhenNoEnvVar() {
+        if (NetworkUtils.getDockerHostIpAddress(Collections.emptyMap()).equals(NetworkUtils.LOCALHOST)) {
+            assertEquals(NetworkUtils.LOCALHOST, NetworkUtils.getDockerHostIpAddress(Collections.emptyMap()));
+        } else {
+            assertEquals(NetworkUtils.getDocker0AdapterIPAddress(), NetworkUtils.getDockerHostIpAddress(Collections.emptyMap()));
+        }
+    }
+
+    @Test
+    public void shouldReturnDockerMachineNameWhenIncluded() {
+        HashMap<String, String> map = new HashMap<>();
+        String dev = "dev";
+        map.put(NetworkUtils.DOCKER_MACHINE_NAME, dev);
+        assertEquals(dev, NetworkUtils.getDockerMachineName(map));
+    }
+}

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Configuration.java

@@ -7,14 +7,13 @@
 import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter;
 import org.apache.mesos.elasticsearch.common.cli.ZookeeperCLIParameter;
 import org.apache.mesos.elasticsearch.common.cli.validators.CLIValidators;
+import org.apache.mesos.elasticsearch.common.util.NetworkUtils;
 import org.apache.mesos.elasticsearch.common.zookeeper.formatter.IpPortsListZKFormatter;
 import org.apache.mesos.elasticsearch.common.zookeeper.formatter.MesosZKFormatter;
 import org.apache.mesos.elasticsearch.common.zookeeper.formatter.ZKFormatter;
 import org.apache.mesos.elasticsearch.common.zookeeper.parser.ZKAddressParser;
-import org.apache.mesos.elasticsearch.scheduler.util.NetworkUtils;
 
 import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -107,8 +106,6 @@ public class Configuration {
     private Boolean isUseIpAddress = false;
 
     // ****************** Runtime configuration **********************
-    private final NetworkUtils networkUtils = new NetworkUtils();
-
     public Configuration(String... args) {
         final JCommander jCommander = new JCommander();
         jCommander.addObject(zookeeperCLI);
@@ -242,13 +239,13 @@ public Boolean isFrameworkUseDocker() {
     public String getFrameworkFileServerAddress() {
         String result = "";
         if (frameworkFileServerAddress != null) {
-            return networkUtils.addressToString(frameworkFileServerAddress, getIsUseIpAddress());
+            return NetworkUtils.addressToString(frameworkFileServerAddress, getIsUseIpAddress());
         }
         return result;
     }
 
     public String webUiAddress() {
-        return networkUtils.addressToString(networkUtils.hostSocket(getWebUiPort()), getIsUseIpAddress());
+        return NetworkUtils.addressToString(NetworkUtils.hostSocket(getWebUiPort()), getIsUseIpAddress());
     }
 
     public void setFrameworkFileServerAddress(InetSocketAddress addr) {

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/Main.java

@@ -7,7 +7,6 @@
 import org.apache.mesos.elasticsearch.scheduler.state.ClusterState;
 import org.apache.mesos.elasticsearch.scheduler.state.FrameworkState;
 import org.apache.mesos.elasticsearch.scheduler.state.SerializableZookeeperState;
-import org.apache.mesos.elasticsearch.scheduler.util.NetworkUtils;
 import org.apache.mesos.state.ZooKeeperState;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 
@@ -38,7 +37,7 @@ public void run(String[] args) {
         configuration = new Configuration(args);
 
         if (!configuration.isFrameworkUseDocker()) {
-            final SimpleFileServer simpleFileServer = new SimpleFileServer(new NetworkUtils(), Configuration.ES_EXECUTOR_JAR);
+            final SimpleFileServer simpleFileServer = new SimpleFileServer(Configuration.ES_EXECUTOR_JAR);
             simpleFileServer.run();
             configuration.setFrameworkFileServerAddress(simpleFileServer.getAddress());
         }

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/SimpleFileServer.java

@@ -6,7 +6,7 @@
 import com.sun.net.httpserver.HttpServer;
 import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Logger;
-import org.apache.mesos.elasticsearch.scheduler.util.NetworkUtils;
+import org.apache.mesos.elasticsearch.common.util.NetworkUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -19,11 +19,9 @@
 public class SimpleFileServer implements Runnable {
     private static final Logger LOGGER = Logger.getLogger(SimpleFileServer.class);
     private HttpServer server;
-    private final NetworkUtils networkUtils;
     private final String file;
 
-    public SimpleFileServer(NetworkUtils networkUtils, String file) {
-        this.networkUtils = networkUtils;
+    public SimpleFileServer(String file) {
         this.file = file;
     }
 
@@ -52,7 +50,7 @@ public InetSocketAddress getAddress() {
         if (server == null) {
             throw new IllegalStateException("Fileserver is not running. Cannot get address.");
         } else {
-            return networkUtils.hostSocket(server.getAddress().getPort());
+            return NetworkUtils.hostSocket(server.getAddress().getPort());
         }
     }
 

scheduler/src/main/java/org/apache/mesos/elasticsearch/scheduler/TaskInfoFactory.java

@@ -6,11 +6,11 @@
 import org.apache.mesos.elasticsearch.common.Discovery;
 import org.apache.mesos.elasticsearch.common.cli.ElasticsearchCLIParameter;
 import org.apache.mesos.elasticsearch.common.cli.HostsCLIParameter;
+import org.apache.mesos.elasticsearch.common.util.NetworkUtils;
 import org.apache.mesos.elasticsearch.scheduler.configuration.ExecutorEnvironmentalVariables;
 import org.apache.mesos.elasticsearch.scheduler.state.ClusterState;
 import org.apache.mesos.elasticsearch.scheduler.state.FrameworkState;
 import org.apache.mesos.elasticsearch.scheduler.util.Clock;
-import org.apache.mesos.elasticsearch.scheduler.util.NetworkUtils;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -37,7 +37,6 @@ public class TaskInfoFactory {
 
     private FrameworkState frameworkState;
     private final ClusterState clusterState;
-    private final NetworkUtils networkUtils = new NetworkUtils();
 
     public TaskInfoFactory(ClusterState clusterState) {
         this.clusterState = clusterState;
@@ -145,7 +144,7 @@ private Protos.CommandInfo.Builder newCommandInfo(Configuration configuration) {
             Protos.TaskInfo taskInfo = taskList.get(0);
             String taskId = taskInfo.getTaskId().getValue();
             InetSocketAddress transportAddress = clusterState.getGuiTaskList().get(taskId).getTransportAddress();
-            hostAddress = networkUtils.addressToString(transportAddress, configuration.getIsUseIpAddress()).replace("http://", "");
+            hostAddress = NetworkUtils.addressToString(transportAddress, configuration.getIsUseIpAddress()).replace("http://", "");
         }
         addIfNotEmpty(args, HostsCLIParameter.ELASTICSEARCH_HOST, hostAddress);
 

scheduler/src/test/java/org/apache/mesos/elasticsearch/scheduler/SimpleFileServerTest.java

@@ -3,7 +3,7 @@
 import com.mashape.unirest.http.HttpResponse;
 import com.mashape.unirest.http.Unirest;
 import com.mashape.unirest.http.exceptions.UnirestException;
-import org.apache.mesos.elasticsearch.scheduler.util.NetworkUtils;
+import org.apache.mesos.elasticsearch.common.util.NetworkUtils;
 import org.junit.Test;
 
 import java.net.InetSocketAddress;
@@ -17,22 +17,21 @@
  */
 public class SimpleFileServerTest {
     public static final String TEST_FILE = "test.file";
-    private final NetworkUtils networkUtils = new NetworkUtils();
 
     @Test
     public void shouldStartAndServeFile() throws UnknownHostException, UnirestException, InterruptedException {
-        final SimpleFileServer simpleFileServer = new SimpleFileServer(networkUtils, TEST_FILE);
+        final SimpleFileServer simpleFileServer = new SimpleFileServer(TEST_FILE);
         simpleFileServer.run();
         InetSocketAddress address = simpleFileServer.getAddress();
-        String serverAddress = networkUtils.addressToString(address, true);
+        String serverAddress = NetworkUtils.addressToString(address, true);
         HttpResponse<String> response = Unirest.get(serverAddress + "/get").asString();
         assertEquals(200, response.getStatus());
         assertTrue(response.getBody().contains("This is a test file"));
     }
 
     @Test(expected = IllegalStateException.class)
     public void shouldErrorIfGettingAddressBeforeStart() {
-        final SimpleFileServer simpleFileServer = new SimpleFileServer(networkUtils, TEST_FILE);
+        final SimpleFileServer simpleFileServer = new SimpleFileServer(TEST_FILE);
         simpleFileServer.getAddress();
     }
 }