Merge pull request #35 from tmjd/wip-auto-migrate

auto migrate scripts and snippets and supporting commands

Makefile

@@ -17,13 +17,15 @@ VERSIONS_FILE?=$(CALICO_UPGRADE_DIR)../_data/versions.yml
 # overriden (by the environment).
 CALICOCTL_VER?=master
 CALICOCTL_V2_VER?=v1.6.x-series
+K8S_VERSION?=v1.8.1
 
 # Construct the calico/ctl names we'll use to download calicoctl and extract the
 # binaries.
 $(info $(shell printf "%-21s = %-10s\n" "CALICOCTL_VER" $(CALICOCTL_VER)))
 $(info $(shell printf "%-21s = %-10s\n" "CALICOCTL_V2_VER" $(CALICOCTL_V2_VER)))
 CTL_CONTAINER_NAME?=calico/ctl:$(CALICOCTL_VER)
 CTL_CONTAINER_V2_NAME?=calico/ctl:$(CALICOCTL_V2_VER)
+KUBECTL_URL=https://dl.k8s.io/$(K8S_VERSION)/kubernetes-client-linux-amd64.tar.gz
 
 ###############################################################################
 # calico-upgrade build
@@ -55,6 +57,9 @@ LDFLAGS=-ldflags "-X $(PACKAGE_NAME)/pkg/commands.VERSION=$(CALICO_UPGRADE_VERSI
 
 LIBCALICOGO_PATH?=none
 
+# curl should failed on 404
+CURL=curl -sSf
+
 calico/upgrade: $(CALICO_UPGRADE_CONTAINER_CREATED)      ## Create the calico/upgrade image
 
 .PHONY: clean-calico-upgrade
@@ -80,10 +85,17 @@ vendor: glide.yaml
       glide install -strip-vendor'
 
 # build calico_upgrade image
-$(CALICO_UPGRADE_CONTAINER_CREATED): pkg/Dockerfile.calico_upgrade dist/calico-upgrade
+$(CALICO_UPGRADE_CONTAINER_CREATED): pkg/Dockerfile.calico_upgrade dist/calico-upgrade dist/kubectl
 	docker build -t $(CALICO_UPGRADE_CONTAINER_NAME) -f pkg/Dockerfile.calico_upgrade .
 	touch $@
 
+# Download kubectl instead of copying from hyperkube because it is 4x smaller
+# this way
+dist/kubectl:
+	$(CURL) -L $(KUBECTL_URL) -o - | tar -zxvf - -C dist --strip-components=3
+	chmod +x $(@D)/*
+
+
 ## Build calico-upgrade
 binary: $(CALICO_UPGRADE_FILES) vendor
 	# Don't try to "install" the intermediate build files (.a .o) when not on linux

glide.yaml

@@ -3,7 +3,7 @@ import:
 - package: github.com/sirupsen/logrus
   version: v1.0.4
 - package: github.com/projectcalico/libcalico-go
-  version: e3351395c934cee118999b9d29508e9280fa75ec
+  version: 9047dcb0ecb57fbfc9924c5aac09db4e019a7588
 - package: github.com/coreos/etcd
   version: v3.3.0
 - package: github.com/spf13/pflag

integration-snippets/README.md

@@ -0,0 +1,81 @@
+# Purpose
+
+The scripts and snippets here are intended to support automating the upgrade
+of a cluster through the addition of two initContainers and a Job. The scripts
+are added to the calico/upgrade container image so they can be ran as
+containers.
+
+# Configuration
+
+Each of the scripts need some configuration, here are the common options
+between the scripts.
+
+## V3 datastore configuration
+The following can be used to configure access to the V3 datastore.
+
+  CALICO_ETCD_ENDPOINTS
+  CALICO_ETCD_USERNAME
+  CALCIO_ETCD_PASSWORD
+  CALICO_ETCD_KEY_FILE
+  CALICO_ETCD_CERT_FILE
+  CALICO_ETCD_CA_CERT_FILE
+
+## V1 datastore configuration
+The following can be used to configure access to the V1 datastore.
+
+  CALICO_APIV1_ETCD_SCHEME
+  CALICO_APIV1_ETCD_AUTHORITY
+  CALICO_APIV1_ETCD_ENDPOINTS
+  CALICO_APIV1_ETCD_USERNAME
+  CALICO_APIV1_ETCD_PASSWORD
+  CALICO_APIV1_ETCD_KEY_FILE
+  CALICO_APIV1_ETCD_CERT_FILE
+  CALICO_APIV1_ETCD_CA_CERT_FILE
+
+## calico-upgrade configuration
+
+It may be useful to provide some additional options or args to calico-upgrade.
+The following are provided for that purpose:
+* `UPGRADE_OPTIONS` can be used to to pass log level to the commands.
+* `UPGRADE_ARGS` could be used to to pass apiconfig if config files are
+  available instead of environment variables.
+
+# Node Daemonset initContainer
+
+This container will be responsible for checking if the datastore needs to be
+migrated and if needed will start the migration. If the datastore migration is
+in progress this container will block startup until it is complete.
+The [script](node-init-container.sh) which does the above can be added to the
+calico-node Daemonset with something like
+[this snippet](node-init-container.yaml).
+
+# calico/kube-controller initContainer
+
+This container will be responsible for checking if the datastore needs to be
+migrated and block startup until the migration is finished.
+The [script](controller-init.sh) which does the above can be added to the
+calico-kube-controllers Deployment with something like
+[this snippet](controller-init.yaml).
+
+# Calico upgrade completion Job
+
+This container will be responsible for checking if the calico-node Daemonset
+has rolled out before completing the upgrade. It also checks that the Daemonset
+has the correct calico/node image as a check that the proper Daemonset
+rolled out.
+The [script](completion-job.sh) which does the above can be deployed with
+something like [this snippet](completion-job.yaml).
+
+## Daemonset monitor configuration
+
+The monitoring of the Daemonset has some values that may change in some
+deployments so the following are exposed to allow their configuration.
+* `DS_NAME`: This should be the name of the 'new' Calico Daemonset.
+  (Default: "calico-node")
+* `DS_SELECTOR`: This is the selector used select the rollout that is being
+  monitored.  (Default: "k8s-app=$DS_NAME")
+* `DS_IMAGE_SELECTOR`: This is the jsonpath used to select the name of the
+  image for the calico-node container. (Default: "{.items[?(@.metadata.name=='$DS_NAME')].spec.template.spec.containers[?(@.name=='calico-node')].image}")
+* `EXPECTED_NODE_IMAGE`: This is the image that is expected in the Daemonset,
+  this is to ensure the correct Daemonset is rolled out.
+  (Default: "quay.io/calico/node:v3.1.1")

integration-snippets/completion-job.sh

@@ -0,0 +1,74 @@
+#!/bin/sh
+
+# Monitors the status of the rollout of the calico-node Daemonset, the script
+# will poll until the Rollout is complete and verify it is the correct Rollout
+# by checking the calico/node images is the expected version.
+#
+# Requirements:
+#  - calico-upgrade is available in the PATH
+#  - kubectl is available in the PATH
+
+
+DS_NAME=${DS_NAME:-"calico-node"}
+DS_SELECTOR=${DS_SELECTOR:-"k8s-app=$DS_NAME"}
+DS_IMAGE_SELECTOR="{.items[?(@.metadata.name=='$DS_NAME')].spec.template.spec.containers[?(@.name=='calico-node')].image}"
+EXPECTED_NODE_IMAGE=${EXPECTED_NODE_IMAGE:-"quay.io/calico/node:v3.1.1"}
+
+echo "Ensure that the Daemonset $DS_NAME is rolled out, and the calico-node"
+echo "container is running $EXPECTED_NODE_IMAGE before completing the upgrade"
+
+IsDsImageCorrect()
+{
+	image=$(kubectl get daemonset --selector=$DS_SELECTOR -n kube-system \
+		-o jsonpath="$DS_IMAGE_SELECTOR")
+	if [ $? -ne 0 ]; then
+		return 1
+	fi
+	echo "$image" | grep "$EXPECTED_NODE_IMAGE"
+}
+
+IsDsRollOutFinished()
+{
+	rollout_status=$(kubectl rollout status daemonset/$DS_NAME)
+	if [ $? -ne 0 ]; then
+		return 1
+	fi
+	echo "$rollout_status" | grep "successfully rolled out"
+}
+
+echo "=== Current $DS_NAME Daemonset ==="
+kubectl get daemonset --selector=$DS_SELECTOR -n kube-system
+
+# Wait for calico-node daemonset to have a v3 calico-node image
+while ! IsDsImageCorrect; do
+	echo "Waiting for the image $EXPECTED_NODE_IMAGE to be in the Daemonset $DS_NAME"
+	sleep 5
+done
+
+echo "=== Current $DS_NAME Daemonset ==="
+kubectl get daemonset --selector=$DS_SELECTOR -n kube-system
+
+# Wait for daemonset to finish rollout
+while ! IsDsRollOutFinished; do
+    echo "Waiting for Daemonset $DS_NAME to finish rolling out"
+	sleep 5
+done
+
+# Verify daemonset still has v3 calico-node image, in case they've done a rollback
+if ! IsDsImageCorrect; then
+    echo "=== Current $DS_NAME Daemonset ==="
+    kubectl get daemonset --selector=$DS_SELECTOR -n kube-system \
+		-o jsonpath="$DS_IMAGE_SELECTOR"
+    echo ""
+	echo "After waiting for $DS_NAME to finish rolling out it does not have the expected"
+	echo "calico/node image version.  If a rollback was done on the calico/node daemonset"
+	echo "this Job should be deleted?"
+	exit 1
+fi
+
+# Complete upgrade
+calico-upgrade $UPGRADE_OPTIONS complete --no-prompts $UPGRADE_ARGS
+if [ $? -ne 0 ]; then
+	echo "Completing the upgrade failed,"
+	exit 1
+fi

integration-snippets/completion-job.yaml

@@ -0,0 +1,87 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: complete-upgrade
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      hostNetwork: true
+      serviceAccountName: calico-upgrade-job
+      restartPolicy: Never
+      containers:
+        - name: migrate-completion
+          image: $UPGRADE_CONTAINER
+          command: ['/bin/sh', '-c', '/completion-job.sh']
+          env:
+            - name: EXPECTED_NODE_IMAGE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: node_image
+            # The location of the Calico etcd cluster.
+            - name: CALICO_ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            - name: CALICO_ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: CALICO_ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: CALICO_ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+            # The location of the Calico etcd cluster.
+            - name: CALICO_APIV1_ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            - name: CALICO_APIV1_ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: CALICO_APIV1_ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: CALICO_APIV1_ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+            - name: CALICO_APIV1_DATASTORE_TYPE
+              value: "etcdv2"
+          volumeMounts:
+            - mountPath: /calico-secrets
+              name: etcd-certs
+      volumes:
+        # Mount in the etcd TLS secrets with mode 400.
+        # See https://kubernetes.io/docs/concepts/configuration/secret/
+        - name: etcd-certs
+          secret:
+            secretName: calico-etcd-secrets
+            defaultMode: 0400
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-upgrade-job
+  namespace: kube-system

integration-snippets/completion-rbac.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-upgrade-job
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-upgrade-job
+subjects:
+- kind: ServiceAccount
+  name: calico-upgrade-job
+  namespace: kube-system
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-upgrade-job
+rules:
+  - apiGroups:
+      - extensions
+    resources:
+      - daemonsets
+      - daemonsets/status
+    verbs:
+      - get
+      - list
+