An attempt to force a new IP address on minikube

jetstack · Apr 13, 2018 · 246f25c · 246f25c
1 parent e2d911f
commit 246f25c
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 39 deletions.
diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -351,7 +351,7 @@ function test_cassandracluster() {
     fi
 
     # Increment the replica count
-    export CASS_REPLICAS=3
+    export CASS_REPLICAS=2
     kubectl apply \
         --namespace "${namespace}" \
         --filename \
@@ -366,7 +366,7 @@ function test_cassandracluster() {
         fail_test "A ScaleOut event was not recorded"
     fi
 
-    if ! retry TIMEOUT=600 stdout_equals 3 kubectl \
+    if ! retry TIMEOUT=600 stdout_equals 2 kubectl \
          --namespace "${namespace}" \
          get cassandracluster \
          "${CASS_NAME}" \
@@ -418,34 +418,7 @@ function test_cassandracluster() {
     kubectl --namespace "${namespace}" get pods \
             --output 'jsonpath={range .items[*]}{.spec.hostname}: {.status.podIP} {"\n"}{end}'
 
-    echo "Get the IP of node 2 of 3"
-    original_ip=$(kubectl --namespace "${namespace}" get pod \
-                          "cass-${CASS_NAME}-${CASS_NODEPOOL1_NAME}-1" \
-                          --output "jsonpath={ .status.podIP }")
-
-    echo "Delete a pod 2 of 3"
-    kubectl --namespace "${namespace}" delete pod \
-            "cass-${CASS_NAME}-${CASS_NODEPOOL1_NAME}-1" --force --grace-period=0
-
-    echo "Waiting for pod IP to change"
-    local start_time
-    start_time="$(date +"%s")"
-    local end_time
-    end_time="$(($start_time + 60))"
-    while true
-    do
-        if [[ "$(date +"%s")" -lt "${end_time}" ]]; then
-            fail_test "IP address did not change"
-            return 1
-        fi
-        new_ip=$(kubectl --namespace "${namespace}" get pod \
-                         "cass-${CASS_NAME}-${CASS_NODEPOOL1_NAME}-1" \
-                         --output "jsonpath={ .status.podIP }")
-        if [[ "${new_ip}" && "${new_ip}" != "${original_ip}" ]]; then
-            break
-        fi
-        sleep 1
-    done
+    retry TIMEOUT=120 kube_delete_pod_and_test_for_new_ip "${namespace}" "${pod}"
 
     echo "Get a map of pod name to IP address after pod IP changes"
     kubectl --namespace "${namespace}" get pods \

diff --git a/hack/libe2e.sh b/hack/libe2e.sh
@@ -160,7 +160,7 @@ function stdout_matches() {
     shift
     local actual
     actual=$("${@}")
-    grep --quiet "${expected}" <<<"${actual}"
+    grep "${expected}" <<<"${actual}"
 }
 
 function stdout_gt() {
@@ -253,12 +253,15 @@ function kube_create_pv() {
     local name="${1}"
     local capacity="${2}"
     local storage_class="${3}"
-    local schedulable_nodes=$(
+    local schedulable_nodes
+    schedulable_nodes=$(
         kubectl get nodes \
                 --output \
                 'jsonpath={range $.items[*]}{.metadata.name} {.spec.taints[*].effect}{"\n"}{end}' \
             | grep -v NoSchedule)
-    local node_name=$(python -c 'import random,sys; print(random.choice(sys.argv[1:]))' $schedulable_nodes)
+    local node_name
+    node_name=$(python -c 'import random,sys; print(random.choice(sys.argv[1:]))' $schedulable_nodes)
+
     local path="hostpath_pvs/${name}/"
 
     kubectl create --filename - <<EOF
@@ -328,6 +331,48 @@ spec:
       - name: host-tmp
         hostPath:
           path: /tmp
-  backoffLimit: 4
 EOF
 }
+
+function kube_get_pod_uid() {
+    local namespace="${1}"
+    local pod="${2}"
+
+    kubectl --namespace "${namespace}" get pod \
+            "${pod}" \
+            --output "jsonpath={ .metadata.uid }" \
+        | egrep '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+}
+
+function kube_get_pod_ip() {
+    local namespace="${1}"
+    local pod="${2}"
+
+    kubectl --namespace "${namespace}" get pod \
+            "${pod}" \
+            --output "jsonpath={ .status.podIP }" | \
+        egrep '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
+}
+
+function kube_delete_pod_and_test_for_new_ip() {
+    local namespace="${1}"
+    local pod="${2}"
+
+    local original_ip
+    original_ip=$(retry kube_get_pod_ip "${namespace}" "${pod}")
+
+    local original_uid
+    original_uid=$(retry kube_get_pod_uid "${namespace}" "${pod}")
+
+    # Delete the pod without grace period
+    kubectl --namespace "${namespace}" delete pod "${pod}" --grace-period=0 --force || true
+    # Run another pod immediately, to hopefully take the IP address of the deleted pod
+    in_cluster_command "${namespace}" "busybox:latest" "/bin/sleep" "2"
+    retry not stdout_equals "${original_uid}" \
+          retry kube_get_pod_uid "${namespace}" "${pod}"
+
+    local new_ip
+    new_ip=$(retry kube_get_pod_ip "${namespace}" "${pod}")
+
+    [[ "${original_ip}" != "${new_ip}" ]]
+}
diff --git a/hack/testdata/cass-cluster-test.template.yaml b/hack/testdata/cass-cluster-test.template.yaml
@@ -16,11 +16,11 @@ spec:
     nodeSelector: {}
     resources:
       requests:
-        cpu: "500m"
-        memory: "2Gi"
+        cpu: "250m"
+        memory: "1Gi"
       limits:
-        cpu: "500m"
-        memory: "2Gi"
+        cpu: "250m"
+        memory: "1Gi"
   pilotImage:
     repository: "${NAVIGATOR_IMAGE_REPOSITORY}/navigator-pilot-cassandra"
     tag: "${NAVIGATOR_IMAGE_TAG}"

diff --git a/hack/testdata/cassandra_test_database1.cql b/hack/testdata/cassandra_test_database1.cql
@@ -1,3 +1,3 @@
-CREATE KEYSPACE space1 WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 };
+CREATE KEYSPACE space1 WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 2 };
 USE space1;
 CREATE TABLE testtable1 (key varchar, value varchar, PRIMARY KEY(key));