From ca1cce323277e67c188e9e85b34f85f1f1921c9a Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Mon, 16 Nov 2015 15:10:48 +0000 Subject: [PATCH 1/6] Makefile should use DOCKER_HOST env var --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index cb468e93c7..b0dc5acacf 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,7 @@ $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go go build $(BUILD_FLAGS) -o $@ ./$(@D) $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) - $(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver touch $@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) @@ -116,7 +116,7 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER cp $(NETCHECK_EXE) prog/weaveexec/netcheck cp $(DOCKERTLSARGS_EXE) prog/weaveexec/docker_tls_args cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz - $(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec touch $@ $(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE) @@ -124,7 +124,7 @@ $(DOCKERPLUGIN_UPTODATE): prog/plugin/Dockerfile $(DOCKERPLUGIN_EXE) touch $@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE) - $(SUDO) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@ + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@ $(DOCKER_DISTRIB): curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL) @@ -142,16 +142,16 @@ $(RUNNER_EXE): tools/.git make -C tools/runner $(PUBLISH): publish_%: $(IMAGES_UPTODATE) - $(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) - $(SUDO) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) ifneq ($(UPDATE_LATEST),false) - $(SUDO) docker push $(DOCKERHUB_USER)/$*:latest + $(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:latest endif publish: $(PUBLISH) clean-bin: - -$(SUDO) docker rmi $(IMAGES) + -$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) go clean -r $(addprefix ./,$(dir $(EXES))) rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) From 0afc3db383802217fe8bfbfe0935386ac906ed8a Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Mon, 16 Nov 2015 15:16:14 +0000 Subject: [PATCH 2/6] First pass at passing DOCKER_HOST=unix://* through to the weave and proxy --- prog/weaveproxy/main.go | 8 +++++++- proxy/common.go | 1 + proxy/proxy.go | 22 ++++++++++++++++------ weave | 22 +++++++++++++++++----- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/prog/weaveproxy/main.go b/prog/weaveproxy/main.go index 6de9a8b45e..0cd13eeda8 100644 --- a/prog/weaveproxy/main.go +++ b/prog/weaveproxy/main.go @@ -12,7 +12,8 @@ import ( ) var ( - version = "(unreleased version)" + version = "(unreleased version)" + defaultDockerHost = "unix:///var/run/docker.sock" ) func main() { @@ -61,6 +62,11 @@ func main() { Log.Infoln("weave proxy", version) Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " ")) + c.DockerHost = defaultDockerHost + if dockerHost := os.Getenv("DOCKER_HOST"); dockerHost != "" { + c.DockerHost = dockerHost + } + p, err := proxy.NewProxy(c) if err != nil { Log.Fatalf("Could not start proxy: %s", err) diff --git a/proxy/common.go b/proxy/common.go index 9a927cca70..a4810c66ce 100644 --- a/proxy/common.go +++ b/proxy/common.go @@ -39,6 +39,7 @@ func callWeave(args ...string) ([]byte, []byte, error) { } propagateEnv("DOCKER_BRIDGE") + propagateEnv("DOCKER_HOST") // Propogage WEAVE_DEBUG, to make debugging easier. propagateEnv("WEAVE_DEBUG") diff --git a/proxy/proxy.go b/proxy/proxy.go index 9c8b15f706..6771d964f5 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -25,8 +25,6 @@ const ( defaultCaFile = "ca.pem" defaultKeyFile = "key.pem" defaultCertFile = "cert.pem" - dockerSock = "/var/run/docker.sock" - dockerSockUnix = "unix://" + dockerSock weaveSock = "/var/run/weave/weave.sock" weaveSockUnix = "unix://" + weaveSock @@ -61,6 +59,7 @@ type Config struct { WithDNS bool WithoutDNS bool NoMulticastRoute bool + DockerHost string } type wait struct { @@ -94,7 +93,7 @@ func NewProxy(c Config) (*Proxy, error) { // to insulate ourselves from breaking changes to the API, as // happened in 1.20 (Docker 1.8.0) when the presentation of // volumes changed in `inspect`. - client, err := weavedocker.NewVersionedClient(dockerSockUnix, "1.18") + client, err := weavedocker.NewVersionedClient(c.DockerHost, "1.18") if err != nil { return nil, err } @@ -140,7 +139,16 @@ func (proxy *Proxy) AttachExistingContainers() { } func (proxy *Proxy) Dial() (net.Conn, error) { - return net.Dial("unix", dockerSock) + proto := "tcp" + addr := proxy.Config.DockerHost + switch { + case strings.HasPrefix(addr, "unix://"): + proto = "unix" + addr = strings.TrimPrefix(addr, "unix://") + case strings.HasPrefix(addr, "tcp://"): + addr = strings.TrimPrefix(addr, "tcp://") + } + return net.Dial(proto, addr) } func (proxy *Proxy) findWeaveWaitVolumes() error { @@ -321,8 +329,10 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) { if err != nil { return nil, "", err } - if err = copyOwnerAndPermissions(dockerSock, addr); err != nil { - return nil, "", err + if strings.HasPrefix(proxy.Config.DockerHost, "unix://") { + if err = copyOwnerAndPermissions(strings.TrimPrefix(proxy.Config.DockerHost, "unix://"), addr); err != nil { + return nil, "", err + } } default: diff --git a/weave b/weave index aa76b409be..27abdf1fd2 100755 --- a/weave +++ b/weave @@ -96,9 +96,21 @@ usage() { exit 1 } +docker_sock() { + if echo "$DOCKER_HOST" | grep -q "^unix://" >/dev/null; then + echo "${DOCKER_HOST#unix://}" + else + echo "/var/run/docker.sock" + fi +} + +docker_sock_volume_mount() { + echo "-v $(docker_sock):$(docker_sock) -e DOCKER_HOST=unix://$(docker_sock)" +} + exec_remote() { docker $DOCKER_CLIENT_ARGS run --rm --privileged --net=host \ - -v /var/run/docker.sock:/var/run/docker.sock \ + $(docker_sock_volume_mount) \ -v /proc:/hostproc \ -e PROCFS=/hostproc \ -e DOCKERHUB_USER="$DOCKERHUB_USER" \ @@ -1609,7 +1621,7 @@ launch_router() { # additional parameters, such as resource limits, to docker # when launching the weave container. ROUTER_CONTAINER=$(docker run --privileged -d --name=$CONTAINER_NAME \ - -v /var/run/docker.sock:/var/run/docker.sock \ + $(docker_sock_volume_mount) \ -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp \ ${NETHOST_OPT:-$DNS_PORT_MAPPING} \ -e WEAVE_PASSWORD \ @@ -1621,7 +1633,7 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ --http-addr $HTTP_IP:$HTTP_PORT \ - --docker-api "unix:///var/run/docker.sock" "$@") + --docker-api "unix://$(docker_sock)" "$@") with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE attach_router } @@ -1666,7 +1678,7 @@ launch_proxy() { mkdir -p /var/run/weave PROXY_CONTAINER=$(docker run --privileged -d --name=$PROXY_CONTAINER_NAME --net=host \ $PROXY_VOLUMES \ - -v /var/run/docker.sock:/var/run/docker.sock \ + $(docker_sock_volume_mount) \ -v /var/run/weave:/var/run/weave \ -v /proc:/hostproc \ -e PROCFS=/hostproc \ @@ -1689,7 +1701,7 @@ launch_plugin() { PLUGIN_CONTAINER=$(docker run --privileged -d --name=$PLUGIN_CONTAINER_NAME \ --restart=always \ --net=host \ - -v /var/run/docker.sock:/var/run/docker.sock \ + $(docker_sock_volume_mount) \ -v /run/docker/plugins:/run/docker/plugins \ $PLUGIN_IMAGE "$@") } From f28ad350bdd82c7b60a990b37a513233b30b92f7 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Fri, 20 Nov 2015 12:39:57 +0000 Subject: [PATCH 3/6] make weave and the proxy work with tcp-only dockers via DOCKER_HOST --- common/docker/client.go | 8 ++++++++ prog/weaver/main.go | 8 +++++++- weave | 16 ++++++++++++---- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/common/docker/client.go b/common/docker/client.go index bd4b78f63d..0a823690eb 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -2,6 +2,8 @@ package docker import ( "errors" + "strings" + "github.com/fsouza/go-dockerclient" . "github.com/weaveworks/weave/common" @@ -19,6 +21,9 @@ type Client struct { // NewClient creates a new Docker client and checks we can talk to Docker func NewClient(apiPath string) (*Client, error) { + if apiPath != "" && !strings.Contains(apiPath, "://") { + apiPath = "tcp://" + apiPath + } dc, err := docker.NewClient(apiPath) if err != nil { return nil, err @@ -29,6 +34,9 @@ func NewClient(apiPath string) (*Client, error) { } func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) { + if !strings.Contains(apiPath, "://") { + apiPath = "tcp://" + apiPath + } dc, err := docker.NewVersionedClient(apiPath, apiVersionString) if err != nil { return nil, err diff --git a/prog/weaver/main.go b/prog/weaver/main.go index 51acd0c33f..a4ff46e8df 100644 --- a/prog/weaver/main.go +++ b/prog/weaver/main.go @@ -70,8 +70,14 @@ func main() { iface *net.Interface datapathName string trustedSubnetStr string + + defaultDockerHost = "unix:///var/run/docker.sock" ) + if val := os.Getenv("DOCKER_HOST"); val != "" { + defaultDockerHost = val + } + mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit") mflag.BoolVar(&createDatapath, []string{"-create-datapath"}, false, "create ODP datapath and exit") mflag.BoolVar(&deleteDatapath, []string{"-delete-datapath"}, false, "delete ODP datapath and exit") @@ -93,7 +99,7 @@ func main() { mflag.StringVar(&iprangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation") mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation") mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)") - mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, "", "Docker API endpoint, e.g. unix:///var/run/docker.sock") + mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint") mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server") mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for") mflag.StringVar(&dnsListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests") diff --git a/weave b/weave index 27abdf1fd2..01becaf2f5 100755 --- a/weave +++ b/weave @@ -97,15 +97,19 @@ usage() { } docker_sock() { + if [ -z "$DOCKER_HOST" ]; then + echo "/var/run/docker.sock" + return + fi if echo "$DOCKER_HOST" | grep -q "^unix://" >/dev/null; then echo "${DOCKER_HOST#unix://}" - else - echo "/var/run/docker.sock" fi } docker_sock_volume_mount() { - echo "-v $(docker_sock):$(docker_sock) -e DOCKER_HOST=unix://$(docker_sock)" + if [ -n "$(docker_sock)" ]; then + echo "-v $(docker_sock):$(docker_sock)" + fi } exec_remote() { @@ -125,6 +129,7 @@ exec_remote() { -e WEAVE_NO_FASTDP \ -e WEAVE_NO_BRIDGED_FASTDP \ -e WEAVE_NO_PLUGIN \ + -e DOCKER_HOST \ -e DOCKER_BRIDGE \ -e DOCKER_CLIENT_HOST="$DOCKER_CLIENT_HOST" \ -e DOCKER_CLIENT_ARGS \ @@ -1624,6 +1629,7 @@ launch_router() { $(docker_sock_volume_mount) \ -p $PORT:$CONTAINER_PORT/tcp -p $PORT:$CONTAINER_PORT/udp \ ${NETHOST_OPT:-$DNS_PORT_MAPPING} \ + -e DOCKER_HOST \ -e WEAVE_PASSWORD \ -e WEAVE_CIDR=none \ $WEAVE_DOCKER_ARGS $IMAGE $COVERAGE_ARGS \ @@ -1633,7 +1639,7 @@ launch_router() { --dns-effective-listen-address $DOCKER_BRIDGE_IP \ ${NETHOST_OPT:+$DNS_ROUTER_OPTS} $NO_DNS_OPT \ --http-addr $HTTP_IP:$HTTP_PORT \ - --docker-api "unix://$(docker_sock)" "$@") + "$@") with_container_netns_or_die $CONTAINER_NAME setup_router_iface_$BRIDGE_TYPE attach_router } @@ -1683,6 +1689,7 @@ launch_proxy() { -v /proc:/hostproc \ -e PROCFS=/hostproc \ -e WEAVE_CIDR=none \ + -e DOCKER_HOST \ -e DOCKER_BRIDGE \ -e WEAVE_DEBUG \ -e COVERAGE \ @@ -1703,6 +1710,7 @@ launch_plugin() { --net=host \ $(docker_sock_volume_mount) \ -v /run/docker/plugins:/run/docker/plugins \ + -e DOCKER_HOST \ $PLUGIN_IMAGE "$@") } From a4ef96690efb3302b3c1994e9fb378f2eef87208 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Mon, 30 Nov 2015 16:33:53 +0000 Subject: [PATCH 4/6] support alternate DOCKER_HOST settings when launching the plugin --- common/docker/client.go | 18 ++++++++++++++---- prog/plugin/main.go | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/common/docker/client.go b/common/docker/client.go index 0a823690eb..d2b5dab054 100644 --- a/common/docker/client.go +++ b/common/docker/client.go @@ -30,7 +30,7 @@ func NewClient(apiPath string) (*Client, error) { } client := &Client{dc} - return client, client.checkWorking(apiPath) + return client, client.checkWorking() } func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) { @@ -43,15 +43,25 @@ func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error } client := &Client{dc} - return client, client.checkWorking(apiPath) + return client, client.checkWorking() } -func (c *Client) checkWorking(apiPath string) error { +func NewVersionedClientFromEnv(apiVersionString string) (*Client, error) { + dc, err := docker.NewVersionedClientFromEnv(apiVersionString) + if err != nil { + return nil, err + } + client := &Client{dc} + + return client, client.checkWorking() +} + +func (c *Client) checkWorking() error { env, err := c.Version() if err != nil { return err } - Log.Infof("[docker] Using Docker API on %s: %v", apiPath, env) + Log.Infof("[docker] Using Docker API on %s: %v", c.Endpoint(), env) return nil } diff --git a/prog/plugin/main.go b/prog/plugin/main.go index 4da4c71e0a..b9a5a581af 100644 --- a/prog/plugin/main.go +++ b/prog/plugin/main.go @@ -49,7 +49,7 @@ func main() { Log.Println("Weave plugin", version, "Command line options:", os.Args[1:]) // API 1.21 is the first version that supports docker network commands - dockerClient, err := docker.NewVersionedClient("unix:///var/run/docker.sock", "1.21") + dockerClient, err := docker.NewVersionedClientFromEnv("1.21") if err != nil { Log.Fatalf("unable to connect to docker: %s", err) } From 88fad251aac18ab193c196e7092f39d65e5ac415 Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Tue, 1 Dec 2015 15:42:32 +0000 Subject: [PATCH 5/6] add test for proxy against docker on non-standard unix socket Means we need to rebuild the gce images (also small improvement re tmpdir) --- test/690_proxy_config_test.sh | 11 ++++++++--- test/gce.sh | 4 ++-- vagrant-common.rb | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/690_proxy_config_test.sh b/test/690_proxy_config_test.sh index 6293a3f56b..11ddcbda2a 100755 --- a/test/690_proxy_config_test.sh +++ b/test/690_proxy_config_test.sh @@ -5,7 +5,7 @@ start_suite "Various launch-proxy configurations" # Booting it over unix socket listens on unix socket -run_on $HOST1 COVERAGE=$COVERAGE sudo -E weave launch-proxy +run_on $HOST1 COVERAGE=$COVERAGE weave launch-proxy assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" assert_raises "proxy docker_on $HOST1 ps" 1 weave_on $HOST1 stop-proxy @@ -17,17 +17,22 @@ assert_raises "proxy docker_on $HOST1 ps" weave_on $HOST1 stop-proxy # Booting it over tcp (no prefix) listens on tcp -DOCKER_CLIENT_ARGS="-H $HOST1:$DOCKER_PORT" $WEAVE launch-proxy +DOCKER_HOST=tcp://$HOST1:$DOCKER_PORT $WEAVE launch-proxy assert_raises "run_on $HOST1 sudo docker -H unix:///var/run/weave/weave.sock ps" 1 assert_raises "proxy docker_on $HOST1 ps" weave_on $HOST1 stop-proxy # Booting it with -H outside /var/run/weave, still works -socket="$(mktemp -d)/weave.sock" +socket="$($SSH $HOST1 mktemp -d)/weave.sock" weave_on $HOST1 launch-proxy -H unix://$socket assert_raises "run_on $HOST1 sudo docker -H unix:///$socket ps" 0 weave_on $HOST1 stop-proxy +# Booting it against non-standard docker unix sock +run_on $HOST1 "DOCKER_HOST=unix:///var/run/alt-docker.sock COVERAGE=$COVERAGE weave launch-proxy -H tcp://0.0.0.0:12375" +assert_raises "proxy docker_on $HOST1 ps" +weave_on $HOST1 stop-proxy + # Booting it over tls errors assert_raises "DOCKER_CLIENT_ARGS='--tls' weave_on $HOST1 launch-proxy" 1 assert_raises "DOCKER_CERT_PATH='./tls' DOCKER_TLS_VERIFY=1 weave_on $HOST1 launch-proxy" 1 diff --git a/test/gce.sh b/test/gce.sh index ae22739ab3..e42b9209c8 100755 --- a/test/gce.sh +++ b/test/gce.sh @@ -11,7 +11,7 @@ set -e : ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key} : ${PROJECT:=positive-cocoa-90213} : ${IMAGE:=ubuntu-14-04} -: ${TEMPLATE_NAME:=test-template-8} +: ${TEMPLATE_NAME:=test-template-9} : ${ZONE:=us-central1-a} : ${NUM_HOSTS:=5} SUFFIX="" @@ -73,7 +73,7 @@ curl -sSL https://get.docker.com/ | sh apt-get update -qq; apt-get install -q -y --force-yes --no-install-recommends ethtool; usermod -a -G docker vagrant; -echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker; +echo 'DOCKER_OPTS="-H unix:///var/run/docker.sock -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay"' >> /etc/default/docker; service docker restart EOF # It seems we need a short delay for docker to start up, so I put this in diff --git a/vagrant-common.rb b/vagrant-common.rb index 15efa7ac41..1cfeb599d0 100644 --- a/vagrant-common.rb +++ b/vagrant-common.rb @@ -45,7 +45,7 @@ def tweak_user_env(vm) def tweak_docker_daemon(vm) vm.provision :shell, :inline => <