Fail cluster validation if a master missing kube-controller-manager

kubernetes · Feb 21, 2020 · eea95d6 · eea95d6
1 parent 206782a
commit eea95d6
Show file tree

Hide file tree

Showing 2 changed files with 125 additions and 3 deletions.
diff --git a/pkg/validation/validate_cluster.go b/pkg/validation/validate_cluster.go
@@ -167,7 +167,7 @@ func (v *clusterValidatorImpl) Validate() (*ValidationCluster, error) {
 		return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err)
 	}
 
-	if err = validation.collectPodFailures(v.k8sClient); err != nil {
+	if err := validation.collectPodFailures(v.k8sClient, nodeList.Items); err != nil {
 		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
 	}
 
@@ -194,12 +194,24 @@ func (v *ValidationCluster) collectComponentFailures(client kubernetes.Interface
 	return nil
 }
 
-func (v *ValidationCluster) collectPodFailures(client kubernetes.Interface) error {
+func (v *ValidationCluster) collectPodFailures(client kubernetes.Interface, nodes []v1.Node) error {
 	pods, err := client.CoreV1().Pods("kube-system").List(metav1.ListOptions{})
 	if err != nil {
 		return fmt.Errorf("error listing Pods: %v", err)
 	}
 
+	masterWithoutManager := map[string]bool{}
+	nodeByAddress := map[string]string{}
+	for _, node := range nodes {
+		labels := node.GetLabels()
+		if labels != nil && labels["kubernetes.io/role"] == "master" {
+			masterWithoutManager[node.Name] = true
+		}
+		for _, nodeAddress := range node.Status.Addresses {
+			nodeByAddress[nodeAddress.Address] = node.Name
+		}
+	}
+
 	for _, pod := range pods.Items {
 		if pod.Status.Phase == v1.PodSucceeded {
 			continue
@@ -226,7 +238,21 @@ func (v *ValidationCluster) collectPodFailures(client kubernetes.Interface) erro
 			})
 
 		}
+
+		labels := pod.GetLabels()
+		if pod.Namespace == "kube-system" && labels != nil && labels["k8s-app"] == "kube-controller-manager" {
+			delete(masterWithoutManager, nodeByAddress[pod.Status.HostIP])
+		}
+	}
+
+	for node := range masterWithoutManager {
+		v.addError(&ValidationError{
+			Kind:    "Node",
+			Name:    node,
+			Message: fmt.Sprintf("master %q is missing kube-controller-manager pod", node),
+		})
 	}
+
 	return nil
 }
 

diff --git a/pkg/validation/validate_cluster_test.go b/pkg/validation/validate_cluster_test.go
@@ -320,6 +320,92 @@ func Test_ValidateMasterNotReady(t *testing.T) {
 	}
 }
 
+func Test_ValidateMasterNoKubeControllerManager(t *testing.T) {
+	groups := make(map[string]*cloudinstances.CloudInstanceGroup)
+	groups["node-1"] = &cloudinstances.CloudInstanceGroup{
+		InstanceGroup: &kopsapi.InstanceGroup{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "master-1",
+			},
+			Spec: kopsapi.InstanceGroupSpec{
+				Role: kopsapi.InstanceGroupRoleMaster,
+			},
+		},
+		MinSize: 1,
+		Ready: []*cloudinstances.CloudInstanceGroupMember{
+			{
+				ID: "i-00001",
+				Node: &v1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "master-1a",
+						Labels: map[string]string{"kubernetes.io/role": "master"},
+					},
+					Status: v1.NodeStatus{
+						Addresses: []v1.NodeAddress{
+							{
+								Address: "1.2.3.4",
+							},
+						},
+						Conditions: []v1.NodeCondition{
+							{Type: "Ready", Status: v1.ConditionTrue},
+						},
+					},
+				},
+			},
+		},
+		NeedUpdate: []*cloudinstances.CloudInstanceGroupMember{
+			{
+				ID: "i-00002",
+				Node: &v1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:   "master-1b",
+						Labels: map[string]string{"kubernetes.io/role": "master"},
+					},
+					Status: v1.NodeStatus{
+						Conditions: []v1.NodeCondition{
+							{Type: "Ready", Status: v1.ConditionTrue},
+						},
+						Addresses: []v1.NodeAddress{
+							{
+								Address: "5.6.7.8",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	v, err := testValidate(t, groups, makePodList(
+		[]map[string]string{
+			{
+				"name":    "pod1",
+				"ready":   "true",
+				"k8s-app": "kube-controller-manager",
+				"phase":   string(v1.PodRunning),
+				"hostip":  "1.2.3.4",
+			},
+			{
+				"name":      "pod2",
+				"namespace": "other",
+				"ready":     "true",
+				"k8s-app":   "kube-controller-manager",
+				"phase":     string(v1.PodRunning),
+				"hostip":    "5.6.7.8",
+			},
+		},
+	))
+	require.NoError(t, err)
+	if !assert.Len(t, v.Failures, 1) ||
+		!assert.Equal(t, &ValidationError{
+			Kind:    "Node",
+			Name:    "master-1b",
+			Message: "master \"master-1b\" is missing kube-controller-manager pod",
+		}, v.Failures[0]) {
+		printDebug(t, v)
+	}
+}
+
 func Test_ValidateNoComponentFailures(t *testing.T) {
 	v, err := testValidate(t, nil, []runtime.Object{
 		&v1.ComponentStatus{
@@ -443,10 +529,19 @@ func printDebug(t *testing.T, v *ValidationCluster) {
 }
 
 func dummyPod(podMap map[string]string) v1.Pod {
+	var labels map[string]string
+	if podMap["k8s-app"] != "" {
+		labels = map[string]string{"k8s-app": podMap["k8s-app"]}
+	}
+	namespace := podMap["namespace"]
+	if namespace == "" {
+		namespace = "kube-system"
+	}
 	return v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      podMap["name"],
-			Namespace: "kube-system",
+			Namespace: namespace,
+			Labels:    labels,
 		},
 		Spec: v1.PodSpec{},
 		Status: v1.PodStatus{
@@ -461,6 +556,7 @@ func dummyPod(podMap map[string]string) v1.Pod {
 					Ready: podMap["ready"] == "true",
 				},
 			},
+			HostIP: podMap["hostip"],
 		},
 	}
 }