vagrant: Implement gcr.io proxy

This introduces the ability to configure an nginx proxy that will
redirect traffic destined for gcr.io to a custom registry. This allows
the user to push copies of the container images from gcr.io to their
custom registry and then pull them in subsequent deployments.

Signed-off-by: Jose A. Rivera <[email protected]>

vagrant/README.md

@@ -42,6 +42,14 @@ scripts are available to facilitate this feature:
    ./docker-cache.sh 192.168.121.1:5000 master
    ```
 
+**GCR.IO PROXY:** An addition to the custom registry, this environment allows
+you to set up an nginx proxy on the master node VM that can redirect gcr.io
+traffic to your custom registry. This allows you to store any images from
+gcr.io in your custom registry and then have things like kubeadm pull from
+there instead of the actual gcr.io. Just specify `custom_registry_gcr=true` in
+`global_vars.yml`. The `gcr-proxy-state.sh` script is available to set the
+proxy redirect on or off at runtime.
+
 A typical workflow to start using this would look like:
 
  1. Run `docker-registry-run.sh`.
@@ -53,5 +61,10 @@ A typical workflow to start using this would look like:
  5. Run `docker-cache.sh <host IP>:5000 node0`
     * **OPTIONAL:** Run `docker pull gcr.io/google_containers/nginx-slim:0.8`
       on `node0` before caching, since it is used in testing.
+ 6. Tear down the vagrant environment: `vagrant destroy`
+ 7. Set `custom_registry_gcr=true` in `global_vars.yml`
 
-Now Docker will check your custom registry before pulling from Docker Hub.
+Now Docker will pull gcr.io images and custom images from your custom registry,
+and check your custom registry before pulling from Docker Hub. You will want to
+periodically set `custom_registry_add=false` and `custom_registry_gcr=false` to
+pull updated images and then cache them with `docker-cache.sh`.

vagrant/Vagrantfile

@@ -40,7 +40,8 @@ Vagrant.configure("2") do |config|
             }
             ansible.extra_vars = {
                 "vagrant_home"  => ENV['VAGRANT_HOME'] ? ENV['VAGRANT_HOME'] : "~/.vagrant.d",
-                "vagrant_cache" => ENV['VAGRANT_CACHE'] ? ENV['VAGRANT_CACHE'] : CACHE
+                "vagrant_cache" => ENV['VAGRANT_CACHE'] ? ENV['VAGRANT_CACHE'] : CACHE,
+                "vagrant_master"  => "192.168.10.90"
             }
         end
 

vagrant/gcr-proxy-state.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ "${1}" != "present" ]] && [[ "${1}" != "absent" ]]; then
+  echo "Error: must supply state, either 'present' or 'absent'"
+  echo "  Example: ./${0} absent"
+  exit 1
+fi
+
+ansible-playbook -i /vagrant/ansible-inventory -e gcr_proxy_state="${1}" gcr_proxy.yml

vagrant/gcr_proxy.yml

@@ -0,0 +1,21 @@
+- hosts: master
+  become: yes
+  become_method: sudo
+  vars_files:
+    - "global_vars.yml"
+  tasks:
+    - include_role:
+        name: master
+        tasks_from: gcr_proxy.yml
+      when: custom_registry_gcr | default(false)
+
+- hosts: nodes
+  become: yes
+  become_method: sudo
+  vars_files:
+    - "global_vars.yml"
+  tasks:
+    - include_role:
+        name: nodes
+        tasks_from: gcr_proxy.yml
+      when: custom_registry_gcr | default(false)

vagrant/global_vars.yml

@@ -25,6 +25,19 @@ install_pkgs:
 # * custom_registry_add: Adds the custom registry as the first registry, making
 #   it the default registry to search for images if no registry is specified.
 #   (default true)
+# * custom_registry_gcr: Sets up an nginx proxy on the master node VM to allow
+#   redirection of requests to gcr.io to your custom registry. This allows you
+#   to pull Google's images without changing image names/URLs. (default false)
+#   It is recommended to increase your master node's RAM to at least 2048 MB
+#   in the Vagrantfile for better performance.
+# * gcr_proxy_state: Whether or not the VMs' /etc/hosts will be updated to
+#   redirect gcr.io traffic to the nginx proxy running on the master node VM.
+#   Valid values are 'present' and 'absent'. (default 'present')
+# * gcr_proxy_nginx: The version of nginx-slim to use for the gcr.io proxy.
+#   (default '0.8')
 #custom_registry: 192.168.121.1:5000
 #custom_registry_insecure: false
 #custom_registry_add: false
+#custom_registry_gcr: true
+#gcr_proxy_state: absent
+#gcr_proxy_nginx: 0.23

vagrant/roles/common/templates/docker.j2

@@ -7,5 +7,5 @@ fi
 ADD_REGISTRY='--add-registry {{ custom_registry }}'
 {% endif %}
 {% if custom_registry_insecure | default(true) -%}
-INSECURE_REGISTRY='--insecure-registry {{ custom_registry }}'
+INSECURE_REGISTRY='--insecure-registry {{ custom_registry }}{% if custom_registry_gcr | default(false) %} --insecure-registry gcr.io{% endif %}'
 {% endif %}

vagrant/roles/master/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+custom_registry_add: false
+custom_registry_gcr: false
+gcr_proxy_nginx: 0.8

vagrant/roles/master/tasks/gcr_proxy.yml

@@ -0,0 +1,58 @@
+---
+- name: install ansible-docker depencendies
+  yum:
+    name: "{{ item }}"
+    state: present
+    disable_gpg_check: yes
+  with_items:
+  - docker-python
+
+- name: check for nginx-slim image
+  uri:
+    url: "http://{{ custom_registry }}/v2/google_containers/nginx-slim/manifests/{{ gcr_proxy_nginx }}"
+    status_code: "200,404"
+  register: nginx_exists
+
+- name: pull nginx-slim image (custom)
+  docker_image:
+    repository: "{{ custom_registry }}"
+    name: google_containers/nginx-slim
+    tag: "{{ gcr_proxy_nginx }}"
+  when:
+  - nginx_exists.status == "200"
+
+- name: pull nginx-slim image (gcr.io)
+  docker_image:
+    repository: gcr.io
+    name: google_containers/nginx-slim
+    tag: "{{ gcr_proxy_nginx }}"
+  when:
+  - nginx_exists.status == "404"
+
+- name: push nginx-slim image
+  docker_image:
+    repository: "{{ custom_registry }}"
+    name: google_containers/nginx-slim
+    tag: "{{ gcr_proxy_nginx }}"
+    push: yes
+  when:
+  - nginx_exists.status == "404"
+
+- name: copy nginx.conf
+  template: src=nginx.conf.j2 dest=/vagrant/nginx.conf force=yes mode=0644
+
+- name: start proxy container
+  docker_container:
+    name: gcr_proxy
+    image: "google_containers/nginx-slim:{{ gcr_proxy_nginx }}"
+    ports:
+    - "80:80"
+    state: started
+    restart_policy: always
+    volumes:
+    - /vagrant/nginx.conf:/etc/nginx/nginx.conf:ro
+
+- lineinfile:
+    path: /etc/hosts
+    state: "{{ gcr_proxy_state | default('present') }}"
+    line: "{{ vagrant_master }} gcr.io"

vagrant/roles/master/tasks/main.yml

@@ -1,3 +1,8 @@
+- include: gcr_proxy.yml
+  when:
+  - custom_registry is defined
+  - custom_registry_gcr | default(false)
+
 - name: install s3-curl depencendies
   yum:
     name: "{{ item }}"

vagrant/roles/master/templates/nginx.conf.j2

@@ -0,0 +1,17 @@
+events {
+    worker_connections 1024;
+}
+http {
+  upstream my_gcr {
+    server {{ custom_registry }};
+  }
+  server {
+    listen *:80;
+    location / {
+      proxy_http_version 1.1;
+      proxy_buffering off;
+      proxy_set_header Proxy "";
+      proxy_pass http://my_gcr;
+    }
+  }
+}

vagrant/roles/nodes/tasks/gcr_proxy.yml

@@ -0,0 +1,5 @@
+---
+- lineinfile:
+    path: /etc/hosts
+    state: "{{ gcr_proxy_state | default('present') }}"
+    line: "{{ vagrant_master }} gcr.io"

vagrant/roles/nodes/tasks/main.yml

@@ -16,6 +16,11 @@
 - name: Open ports 49152-49251 (GlusterFS bricks)
   firewalld: port=49152-49251/tcp zone=trusted permanent=true state=enabled immediate=true
 
+- include: gcr_proxy.yml
+  when:
+  - custom_registry is defined
+  - custom_registry_gcr | default(false)
+
 - name: Pull GlusterFS Docker image
   command: docker pull gluster/gluster-centos:latest
   register: task_result